1 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the X86MCCodeEmitter class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "MCTargetDesc/X86BaseInfo.h"
14 #include "MCTargetDesc/X86FixupKinds.h"
15 #include "MCTargetDesc/X86MCTargetDesc.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/MC/MCCodeEmitter.h"
18 #include "llvm/MC/MCContext.h"
19 #include "llvm/MC/MCExpr.h"
20 #include "llvm/MC/MCFixup.h"
21 #include "llvm/MC/MCInst.h"
22 #include "llvm/MC/MCInstrDesc.h"
23 #include "llvm/MC/MCInstrInfo.h"
24 #include "llvm/MC/MCRegisterInfo.h"
25 #include "llvm/MC/MCSubtargetInfo.h"
26 #include "llvm/MC/MCSymbol.h"
27 #include "llvm/Support/ErrorHandling.h"
28 #include "llvm/Support/raw_ostream.h"
29 #include <cassert>
30 #include <cstdint>
31 #include <cstdlib>
32
33 using namespace llvm;
34
35 #define DEBUG_TYPE "mccodeemitter"
36
37 namespace {
38
39 class X86MCCodeEmitter : public MCCodeEmitter {
40 const MCInstrInfo &MCII;
41 MCContext &Ctx;
42
43 public:
X86MCCodeEmitter(const MCInstrInfo & mcii,MCContext & ctx)44 X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
45 : MCII(mcii), Ctx(ctx) {}
46 X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
47 X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
48 ~X86MCCodeEmitter() override = default;
49
50 void emitPrefix(const MCInst &MI, raw_ostream &OS,
51 const MCSubtargetInfo &STI) const override;
52
53 void encodeInstruction(const MCInst &MI, raw_ostream &OS,
54 SmallVectorImpl<MCFixup> &Fixups,
55 const MCSubtargetInfo &STI) const override;
56
57 private:
58 unsigned getX86RegNum(const MCOperand &MO) const;
59
60 unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const;
61
62 /// \param MI a single low-level machine instruction.
63 /// \param OpNum the operand #.
64 /// \returns true if the OpNumth operand of MI require a bit to be set in
65 /// REX prefix.
66 bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const;
67
68 void emitImmediate(const MCOperand &Disp, SMLoc Loc, unsigned ImmSize,
69 MCFixupKind FixupKind, uint64_t StartByte, raw_ostream &OS,
70 SmallVectorImpl<MCFixup> &Fixups, int ImmOffset = 0) const;
71
72 void emitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
73 raw_ostream &OS) const;
74
75 void emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
76 raw_ostream &OS) const;
77
78 void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
79 uint64_t TSFlags, bool HasREX, uint64_t StartByte,
80 raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
81 const MCSubtargetInfo &STI,
82 bool ForceSIB = false) const;
83
84 bool emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
85 const MCSubtargetInfo &STI, raw_ostream &OS) const;
86
87 void emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
88 raw_ostream &OS) const;
89
90 void emitSegmentOverridePrefix(unsigned SegOperand, const MCInst &MI,
91 raw_ostream &OS) const;
92
93 bool emitOpcodePrefix(int MemOperand, const MCInst &MI,
94 const MCSubtargetInfo &STI, raw_ostream &OS) const;
95
96 bool emitREXPrefix(int MemOperand, const MCInst &MI, raw_ostream &OS) const;
97 };
98
99 } // end anonymous namespace
100
modRMByte(unsigned Mod,unsigned RegOpcode,unsigned RM)101 static uint8_t modRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
102 assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
103 return RM | (RegOpcode << 3) | (Mod << 6);
104 }
105
emitByte(uint8_t C,raw_ostream & OS)106 static void emitByte(uint8_t C, raw_ostream &OS) { OS << static_cast<char>(C); }
107
emitConstant(uint64_t Val,unsigned Size,raw_ostream & OS)108 static void emitConstant(uint64_t Val, unsigned Size, raw_ostream &OS) {
109 // Output the constant in little endian byte order.
110 for (unsigned i = 0; i != Size; ++i) {
111 emitByte(Val & 255, OS);
112 Val >>= 8;
113 }
114 }
115
116 /// Determine if this immediate can fit in a disp8 or a compressed disp8 for
117 /// EVEX instructions. \p will be set to the value to pass to the ImmOffset
118 /// parameter of emitImmediate.
isDispOrCDisp8(uint64_t TSFlags,int Value,int & ImmOffset)119 static bool isDispOrCDisp8(uint64_t TSFlags, int Value, int &ImmOffset) {
120 bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
121
122 int CD8_Scale =
123 (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
124 if (!HasEVEX || CD8_Scale == 0)
125 return isInt<8>(Value);
126
127 assert(isPowerOf2_32(CD8_Scale) && "Unexpected CD8 scale!");
128 if (Value & (CD8_Scale - 1)) // Unaligned offset
129 return false;
130
131 int CDisp8 = Value / CD8_Scale;
132 if (!isInt<8>(CDisp8))
133 return false;
134
135 // ImmOffset will be added to Value in emitImmediate leaving just CDisp8.
136 ImmOffset = CDisp8 - Value;
137 return true;
138 }
139
140 /// \returns the appropriate fixup kind to use for an immediate in an
141 /// instruction with the specified TSFlags.
getImmFixupKind(uint64_t TSFlags)142 static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
143 unsigned Size = X86II::getSizeOfImm(TSFlags);
144 bool isPCRel = X86II::isImmPCRel(TSFlags);
145
146 if (X86II::isImmSigned(TSFlags)) {
147 switch (Size) {
148 default:
149 llvm_unreachable("Unsupported signed fixup size!");
150 case 4:
151 return MCFixupKind(X86::reloc_signed_4byte);
152 }
153 }
154 return MCFixup::getKindForSize(Size, isPCRel);
155 }
156
157 /// \param Op operand # of the memory operand.
158 ///
159 /// \returns true if the specified instruction has a 16-bit memory operand.
is16BitMemOperand(const MCInst & MI,unsigned Op,const MCSubtargetInfo & STI)160 static bool is16BitMemOperand(const MCInst &MI, unsigned Op,
161 const MCSubtargetInfo &STI) {
162 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
163 const MCOperand &Index = MI.getOperand(Op + X86::AddrIndexReg);
164
165 unsigned BaseReg = Base.getReg();
166 unsigned IndexReg = Index.getReg();
167
168 if (STI.hasFeature(X86::Mode16Bit) && BaseReg == 0 && IndexReg == 0)
169 return true;
170 if ((BaseReg != 0 &&
171 X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg)) ||
172 (IndexReg != 0 &&
173 X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg)))
174 return true;
175 return false;
176 }
177
178 /// \param Op operand # of the memory operand.
179 ///
180 /// \returns true if the specified instruction has a 32-bit memory operand.
is32BitMemOperand(const MCInst & MI,unsigned Op)181 static bool is32BitMemOperand(const MCInst &MI, unsigned Op) {
182 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
183 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
184
185 if ((BaseReg.getReg() != 0 &&
186 X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
187 (IndexReg.getReg() != 0 &&
188 X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
189 return true;
190 if (BaseReg.getReg() == X86::EIP) {
191 assert(IndexReg.getReg() == 0 && "Invalid eip-based address.");
192 return true;
193 }
194 if (IndexReg.getReg() == X86::EIZ)
195 return true;
196 return false;
197 }
198
199 /// \param Op operand # of the memory operand.
200 ///
201 /// \returns true if the specified instruction has a 64-bit memory operand.
202 #ifndef NDEBUG
is64BitMemOperand(const MCInst & MI,unsigned Op)203 static bool is64BitMemOperand(const MCInst &MI, unsigned Op) {
204 const MCOperand &BaseReg = MI.getOperand(Op + X86::AddrBaseReg);
205 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
206
207 if ((BaseReg.getReg() != 0 &&
208 X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
209 (IndexReg.getReg() != 0 &&
210 X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
211 return true;
212 return false;
213 }
214 #endif
215
216 enum GlobalOffsetTableExprKind { GOT_None, GOT_Normal, GOT_SymDiff };
217
218 /// Check if this expression starts with _GLOBAL_OFFSET_TABLE_ and if it is
219 /// of the form _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on
220 /// ELF i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
221 /// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start of a
222 /// binary expression.
223 static GlobalOffsetTableExprKind
startsWithGlobalOffsetTable(const MCExpr * Expr)224 startsWithGlobalOffsetTable(const MCExpr *Expr) {
225 const MCExpr *RHS = nullptr;
226 if (Expr->getKind() == MCExpr::Binary) {
227 const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
228 Expr = BE->getLHS();
229 RHS = BE->getRHS();
230 }
231
232 if (Expr->getKind() != MCExpr::SymbolRef)
233 return GOT_None;
234
235 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
236 const MCSymbol &S = Ref->getSymbol();
237 if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
238 return GOT_None;
239 if (RHS && RHS->getKind() == MCExpr::SymbolRef)
240 return GOT_SymDiff;
241 return GOT_Normal;
242 }
243
hasSecRelSymbolRef(const MCExpr * Expr)244 static bool hasSecRelSymbolRef(const MCExpr *Expr) {
245 if (Expr->getKind() == MCExpr::SymbolRef) {
246 const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr *>(Expr);
247 return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
248 }
249 return false;
250 }
251
isPCRel32Branch(const MCInst & MI,const MCInstrInfo & MCII)252 static bool isPCRel32Branch(const MCInst &MI, const MCInstrInfo &MCII) {
253 unsigned Opcode = MI.getOpcode();
254 const MCInstrDesc &Desc = MCII.get(Opcode);
255 if ((Opcode != X86::CALL64pcrel32 && Opcode != X86::JMP_4 &&
256 Opcode != X86::JCC_4) ||
257 getImmFixupKind(Desc.TSFlags) != FK_PCRel_4)
258 return false;
259
260 unsigned CurOp = X86II::getOperandBias(Desc);
261 const MCOperand &Op = MI.getOperand(CurOp);
262 if (!Op.isExpr())
263 return false;
264
265 const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Op.getExpr());
266 return Ref && Ref->getKind() == MCSymbolRefExpr::VK_None;
267 }
268
getX86RegNum(const MCOperand & MO) const269 unsigned X86MCCodeEmitter::getX86RegNum(const MCOperand &MO) const {
270 return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
271 }
272
getX86RegEncoding(const MCInst & MI,unsigned OpNum) const273 unsigned X86MCCodeEmitter::getX86RegEncoding(const MCInst &MI,
274 unsigned OpNum) const {
275 return Ctx.getRegisterInfo()->getEncodingValue(MI.getOperand(OpNum).getReg());
276 }
277
278 /// \param MI a single low-level machine instruction.
279 /// \param OpNum the operand #.
280 /// \returns true if the OpNumth operand of MI require a bit to be set in
281 /// REX prefix.
isREXExtendedReg(const MCInst & MI,unsigned OpNum) const282 bool X86MCCodeEmitter::isREXExtendedReg(const MCInst &MI,
283 unsigned OpNum) const {
284 return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
285 }
286
emitImmediate(const MCOperand & DispOp,SMLoc Loc,unsigned Size,MCFixupKind FixupKind,uint64_t StartByte,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,int ImmOffset) const287 void X86MCCodeEmitter::emitImmediate(const MCOperand &DispOp, SMLoc Loc,
288 unsigned Size, MCFixupKind FixupKind,
289 uint64_t StartByte, raw_ostream &OS,
290 SmallVectorImpl<MCFixup> &Fixups,
291 int ImmOffset) const {
292 const MCExpr *Expr = nullptr;
293 if (DispOp.isImm()) {
294 // If this is a simple integer displacement that doesn't require a
295 // relocation, emit it now.
296 if (FixupKind != FK_PCRel_1 && FixupKind != FK_PCRel_2 &&
297 FixupKind != FK_PCRel_4) {
298 emitConstant(DispOp.getImm() + ImmOffset, Size, OS);
299 return;
300 }
301 Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
302 } else {
303 Expr = DispOp.getExpr();
304 }
305
306 // If we have an immoffset, add it to the expression.
307 if ((FixupKind == FK_Data_4 || FixupKind == FK_Data_8 ||
308 FixupKind == MCFixupKind(X86::reloc_signed_4byte))) {
309 GlobalOffsetTableExprKind Kind = startsWithGlobalOffsetTable(Expr);
310 if (Kind != GOT_None) {
311 assert(ImmOffset == 0);
312
313 if (Size == 8) {
314 FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
315 } else {
316 assert(Size == 4);
317 FixupKind = MCFixupKind(X86::reloc_global_offset_table);
318 }
319
320 if (Kind == GOT_Normal)
321 ImmOffset = static_cast<int>(OS.tell() - StartByte);
322 } else if (Expr->getKind() == MCExpr::SymbolRef) {
323 if (hasSecRelSymbolRef(Expr)) {
324 FixupKind = MCFixupKind(FK_SecRel_4);
325 }
326 } else if (Expr->getKind() == MCExpr::Binary) {
327 const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr *>(Expr);
328 if (hasSecRelSymbolRef(Bin->getLHS()) ||
329 hasSecRelSymbolRef(Bin->getRHS())) {
330 FixupKind = MCFixupKind(FK_SecRel_4);
331 }
332 }
333 }
334
335 // If the fixup is pc-relative, we need to bias the value to be relative to
336 // the start of the field, not the end of the field.
337 if (FixupKind == FK_PCRel_4 ||
338 FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
339 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
340 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
341 FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex) ||
342 FixupKind == MCFixupKind(X86::reloc_branch_4byte_pcrel)) {
343 ImmOffset -= 4;
344 // If this is a pc-relative load off _GLOBAL_OFFSET_TABLE_:
345 // leaq _GLOBAL_OFFSET_TABLE_(%rip), %r15
346 // this needs to be a GOTPC32 relocation.
347 if (startsWithGlobalOffsetTable(Expr) != GOT_None)
348 FixupKind = MCFixupKind(X86::reloc_global_offset_table);
349 }
350 if (FixupKind == FK_PCRel_2)
351 ImmOffset -= 2;
352 if (FixupKind == FK_PCRel_1)
353 ImmOffset -= 1;
354
355 if (ImmOffset)
356 Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
357 Ctx);
358
359 // Emit a symbolic constant as a fixup and 4 zeros.
360 Fixups.push_back(MCFixup::create(static_cast<uint32_t>(OS.tell() - StartByte),
361 Expr, FixupKind, Loc));
362 emitConstant(0, Size, OS);
363 }
364
emitRegModRMByte(const MCOperand & ModRMReg,unsigned RegOpcodeFld,raw_ostream & OS) const365 void X86MCCodeEmitter::emitRegModRMByte(const MCOperand &ModRMReg,
366 unsigned RegOpcodeFld,
367 raw_ostream &OS) const {
368 emitByte(modRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)), OS);
369 }
370
emitSIBByte(unsigned SS,unsigned Index,unsigned Base,raw_ostream & OS) const371 void X86MCCodeEmitter::emitSIBByte(unsigned SS, unsigned Index, unsigned Base,
372 raw_ostream &OS) const {
373 // SIB byte is in the same format as the modRMByte.
374 emitByte(modRMByte(SS, Index, Base), OS);
375 }
376
emitMemModRMByte(const MCInst & MI,unsigned Op,unsigned RegOpcodeField,uint64_t TSFlags,bool HasREX,uint64_t StartByte,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,const MCSubtargetInfo & STI,bool ForceSIB) const377 void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
378 unsigned RegOpcodeField,
379 uint64_t TSFlags, bool HasREX,
380 uint64_t StartByte, raw_ostream &OS,
381 SmallVectorImpl<MCFixup> &Fixups,
382 const MCSubtargetInfo &STI,
383 bool ForceSIB) const {
384 const MCOperand &Disp = MI.getOperand(Op + X86::AddrDisp);
385 const MCOperand &Base = MI.getOperand(Op + X86::AddrBaseReg);
386 const MCOperand &Scale = MI.getOperand(Op + X86::AddrScaleAmt);
387 const MCOperand &IndexReg = MI.getOperand(Op + X86::AddrIndexReg);
388 unsigned BaseReg = Base.getReg();
389
390 // Handle %rip relative addressing.
391 if (BaseReg == X86::RIP ||
392 BaseReg == X86::EIP) { // [disp32+rIP] in X86-64 mode
393 assert(STI.hasFeature(X86::Mode64Bit) &&
394 "Rip-relative addressing requires 64-bit mode");
395 assert(IndexReg.getReg() == 0 && !ForceSIB &&
396 "Invalid rip-relative address");
397 emitByte(modRMByte(0, RegOpcodeField, 5), OS);
398
399 unsigned Opcode = MI.getOpcode();
400 // movq loads are handled with a special relocation form which allows the
401 // linker to eliminate some loads for GOT references which end up in the
402 // same linkage unit.
403 unsigned FixupKind = [=]() {
404 switch (Opcode) {
405 default:
406 return X86::reloc_riprel_4byte;
407 case X86::ADC32rm:
408 case X86::ADD32rm:
409 case X86::AND32rm:
410 case X86::CMP32rm:
411 case X86::MOV32rm:
412 case X86::OR32rm:
413 case X86::SBB32rm:
414 case X86::SUB32rm:
415 case X86::TEST32mr:
416 case X86::XOR32rm:
417 return X86::reloc_riprel_4byte_relax;
418 case X86::MOV64rm:
419 assert(HasREX);
420 return X86::reloc_riprel_4byte_movq_load;
421 case X86::CALL64m:
422 case X86::JMP64m:
423 case X86::TAILJMPm64:
424 case X86::TEST64mr:
425 case X86::ADC64rm:
426 case X86::ADD64rm:
427 case X86::AND64rm:
428 case X86::CMP64rm:
429 case X86::OR64rm:
430 case X86::SBB64rm:
431 case X86::SUB64rm:
432 case X86::XOR64rm:
433 return HasREX ? X86::reloc_riprel_4byte_relax_rex
434 : X86::reloc_riprel_4byte_relax;
435 }
436 }();
437
438 // rip-relative addressing is actually relative to the *next* instruction.
439 // Since an immediate can follow the mod/rm byte for an instruction, this
440 // means that we need to bias the displacement field of the instruction with
441 // the size of the immediate field. If we have this case, add it into the
442 // expression to emit.
443 // Note: rip-relative addressing using immediate displacement values should
444 // not be adjusted, assuming it was the user's intent.
445 int ImmSize = !Disp.isImm() && X86II::hasImm(TSFlags)
446 ? X86II::getSizeOfImm(TSFlags)
447 : 0;
448
449 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
450 Fixups, -ImmSize);
451 return;
452 }
453
454 unsigned BaseRegNo = BaseReg ? getX86RegNum(Base) : -1U;
455
456 // 16-bit addressing forms of the ModR/M byte have a different encoding for
457 // the R/M field and are far more limited in which registers can be used.
458 if (is16BitMemOperand(MI, Op, STI)) {
459 if (BaseReg) {
460 // For 32-bit addressing, the row and column values in Table 2-2 are
461 // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
462 // some special cases. And getX86RegNum reflects that numbering.
463 // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
464 // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
465 // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
466 // while values 0-3 indicate the allowed combinations (base+index) of
467 // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
468 //
469 // R16Table[] is a lookup from the normal RegNo, to the row values from
470 // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
471 static const unsigned R16Table[] = {0, 0, 0, 7, 0, 6, 4, 5};
472 unsigned RMfield = R16Table[BaseRegNo];
473
474 assert(RMfield && "invalid 16-bit base register");
475
476 if (IndexReg.getReg()) {
477 unsigned IndexReg16 = R16Table[getX86RegNum(IndexReg)];
478
479 assert(IndexReg16 && "invalid 16-bit index register");
480 // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
481 assert(((IndexReg16 ^ RMfield) & 2) &&
482 "invalid 16-bit base/index register combination");
483 assert(Scale.getImm() == 1 &&
484 "invalid scale for 16-bit memory reference");
485
486 // Allow base/index to appear in either order (although GAS doesn't).
487 if (IndexReg16 & 2)
488 RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
489 else
490 RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
491 }
492
493 if (Disp.isImm() && isInt<8>(Disp.getImm())) {
494 if (Disp.getImm() == 0 && RMfield != 6) {
495 // There is no displacement; just the register.
496 emitByte(modRMByte(0, RegOpcodeField, RMfield), OS);
497 return;
498 }
499 // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
500 emitByte(modRMByte(1, RegOpcodeField, RMfield), OS);
501 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups);
502 return;
503 }
504 // This is the [REG]+disp16 case.
505 emitByte(modRMByte(2, RegOpcodeField, RMfield), OS);
506 } else {
507 assert(IndexReg.getReg() == 0 && "Unexpected index register!");
508 // There is no BaseReg; this is the plain [disp16] case.
509 emitByte(modRMByte(0, RegOpcodeField, 6), OS);
510 }
511
512 // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
513 emitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, StartByte, OS, Fixups);
514 return;
515 }
516
517 // Check for presence of {disp8} or {disp32} pseudo prefixes.
518 bool UseDisp8 = MI.getFlags() & X86::IP_USE_DISP8;
519 bool UseDisp32 = MI.getFlags() & X86::IP_USE_DISP32;
520
521 // We only allow no displacement if no pseudo prefix is present.
522 bool AllowNoDisp = !UseDisp8 && !UseDisp32;
523 // Disp8 is allowed unless the {disp32} prefix is present.
524 bool AllowDisp8 = !UseDisp32;
525
526 // Determine whether a SIB byte is needed.
527 if (// The SIB byte must be used if there is an index register or the
528 // encoding requires a SIB byte.
529 !ForceSIB && IndexReg.getReg() == 0 &&
530 // The SIB byte must be used if the base is ESP/RSP/R12, all of which
531 // encode to an R/M value of 4, which indicates that a SIB byte is
532 // present.
533 BaseRegNo != N86::ESP &&
534 // If there is no base register and we're in 64-bit mode, we need a SIB
535 // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
536 (!STI.hasFeature(X86::Mode64Bit) || BaseReg != 0)) {
537
538 if (BaseReg == 0) { // [disp32] in X86-32 mode
539 emitByte(modRMByte(0, RegOpcodeField, 5), OS);
540 emitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, StartByte, OS, Fixups);
541 return;
542 }
543
544 // If the base is not EBP/ESP/R12/R13 and there is no displacement, use
545 // simple indirect register encoding, this handles addresses like [EAX].
546 // The encoding for [EBP] or[R13] with no displacement means [disp32] so we
547 // handle it by emitting a displacement of 0 later.
548 if (BaseRegNo != N86::EBP) {
549 if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp) {
550 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
551 return;
552 }
553
554 // If the displacement is @tlscall, treat it as a zero.
555 if (Disp.isExpr()) {
556 auto *Sym = dyn_cast<MCSymbolRefExpr>(Disp.getExpr());
557 if (Sym && Sym->getKind() == MCSymbolRefExpr::VK_TLSCALL) {
558 // This is exclusively used by call *a@tlscall(base). The relocation
559 // (R_386_TLSCALL or R_X86_64_TLSCALL) applies to the beginning.
560 Fixups.push_back(MCFixup::create(0, Sym, FK_NONE, MI.getLoc()));
561 emitByte(modRMByte(0, RegOpcodeField, BaseRegNo), OS);
562 return;
563 }
564 }
565 }
566
567 // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
568 // Including a compressed disp8 for EVEX instructions that support it.
569 // This also handles the 0 displacement for [EBP] or [R13]. We can't use
570 // disp8 if the {disp32} pseudo prefix is present.
571 if (Disp.isImm() && AllowDisp8) {
572 int ImmOffset = 0;
573 if (isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
574 emitByte(modRMByte(1, RegOpcodeField, BaseRegNo), OS);
575 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
576 ImmOffset);
577 return;
578 }
579 }
580
581 // Otherwise, emit the most general non-SIB encoding: [REG+disp32].
582 // Displacement may be 0 for [EBP] or [R13] case if {disp32} pseudo prefix
583 // prevented using disp8 above.
584 emitByte(modRMByte(2, RegOpcodeField, BaseRegNo), OS);
585 unsigned Opcode = MI.getOpcode();
586 unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
587 : X86::reloc_signed_4byte;
588 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), StartByte, OS,
589 Fixups);
590 return;
591 }
592
593 // We need a SIB byte, so start by outputting the ModR/M byte first
594 assert(IndexReg.getReg() != X86::ESP && IndexReg.getReg() != X86::RSP &&
595 "Cannot use ESP as index reg!");
596
597 bool ForceDisp32 = false;
598 bool ForceDisp8 = false;
599 int ImmOffset = 0;
600 if (BaseReg == 0) {
601 // If there is no base register, we emit the special case SIB byte with
602 // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
603 BaseRegNo = 5;
604 emitByte(modRMByte(0, RegOpcodeField, 4), OS);
605 ForceDisp32 = true;
606 } else if (Disp.isImm() && Disp.getImm() == 0 && AllowNoDisp &&
607 // Base reg can't be EBP/RBP/R13 as that would end up with '5' as
608 // the base field, but that is the magic [*] nomenclature that
609 // indicates no base when mod=0. For these cases we'll emit a 0
610 // displacement instead.
611 BaseRegNo != N86::EBP) {
612 // Emit no displacement ModR/M byte
613 emitByte(modRMByte(0, RegOpcodeField, 4), OS);
614 } else if (Disp.isImm() && AllowDisp8 &&
615 isDispOrCDisp8(TSFlags, Disp.getImm(), ImmOffset)) {
616 // Displacement fits in a byte or matches an EVEX compressed disp8, use
617 // disp8 encoding. This also handles EBP/R13 base with 0 displacement unless
618 // {disp32} pseudo prefix was used.
619 emitByte(modRMByte(1, RegOpcodeField, 4), OS);
620 ForceDisp8 = true;
621 } else {
622 // Otherwise, emit the normal disp32 encoding.
623 emitByte(modRMByte(2, RegOpcodeField, 4), OS);
624 ForceDisp32 = true;
625 }
626
627 // Calculate what the SS field value should be...
628 static const unsigned SSTable[] = {~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3};
629 unsigned SS = SSTable[Scale.getImm()];
630
631 unsigned IndexRegNo = IndexReg.getReg() ? getX86RegNum(IndexReg) : 4;
632
633 emitSIBByte(SS, IndexRegNo, BaseRegNo, OS);
634
635 // Do we need to output a displacement?
636 if (ForceDisp8)
637 emitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, StartByte, OS, Fixups,
638 ImmOffset);
639 else if (ForceDisp32)
640 emitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
641 StartByte, OS, Fixups);
642 }
643
644 /// Emit all instruction prefixes.
645 ///
646 /// \returns true if REX prefix is used, otherwise returns false.
emitPrefixImpl(unsigned & CurOp,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const647 bool X86MCCodeEmitter::emitPrefixImpl(unsigned &CurOp, const MCInst &MI,
648 const MCSubtargetInfo &STI,
649 raw_ostream &OS) const {
650 uint64_t TSFlags = MCII.get(MI.getOpcode()).TSFlags;
651 // Determine where the memory operand starts, if present.
652 int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
653 // Emit segment override opcode prefix as needed.
654 if (MemoryOperand != -1) {
655 MemoryOperand += CurOp;
656 emitSegmentOverridePrefix(MemoryOperand + X86::AddrSegmentReg, MI, OS);
657 }
658
659 // Emit the repeat opcode prefix as needed.
660 unsigned Flags = MI.getFlags();
661 if (TSFlags & X86II::REP || Flags & X86::IP_HAS_REPEAT)
662 emitByte(0xF3, OS);
663 if (Flags & X86::IP_HAS_REPEAT_NE)
664 emitByte(0xF2, OS);
665
666 // Emit the address size opcode prefix as needed.
667 bool NeedAddressOverride;
668 uint64_t AdSize = TSFlags & X86II::AdSizeMask;
669 if ((STI.hasFeature(X86::Mode16Bit) && AdSize == X86II::AdSize32) ||
670 (STI.hasFeature(X86::Mode32Bit) && AdSize == X86II::AdSize16) ||
671 (STI.hasFeature(X86::Mode64Bit) && AdSize == X86II::AdSize32)) {
672 NeedAddressOverride = true;
673 } else if (MemoryOperand < 0) {
674 NeedAddressOverride = false;
675 } else if (STI.hasFeature(X86::Mode64Bit)) {
676 assert(!is16BitMemOperand(MI, MemoryOperand, STI));
677 NeedAddressOverride = is32BitMemOperand(MI, MemoryOperand);
678 } else if (STI.hasFeature(X86::Mode32Bit)) {
679 assert(!is64BitMemOperand(MI, MemoryOperand));
680 NeedAddressOverride = is16BitMemOperand(MI, MemoryOperand, STI);
681 } else {
682 assert(STI.hasFeature(X86::Mode16Bit));
683 assert(!is64BitMemOperand(MI, MemoryOperand));
684 NeedAddressOverride = !is16BitMemOperand(MI, MemoryOperand, STI);
685 }
686
687 if (NeedAddressOverride)
688 emitByte(0x67, OS);
689
690 // Encoding type for this instruction.
691 uint64_t Encoding = TSFlags & X86II::EncodingMask;
692 bool HasREX = false;
693 if (Encoding)
694 emitVEXOpcodePrefix(MemoryOperand, MI, OS);
695 else
696 HasREX = emitOpcodePrefix(MemoryOperand, MI, STI, OS);
697
698 uint64_t Form = TSFlags & X86II::FormMask;
699 switch (Form) {
700 default:
701 break;
702 case X86II::RawFrmDstSrc: {
703 unsigned siReg = MI.getOperand(1).getReg();
704 assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) ||
705 (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) ||
706 (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) &&
707 "SI and DI register sizes do not match");
708 // Emit segment override opcode prefix as needed (not for %ds).
709 if (MI.getOperand(2).getReg() != X86::DS)
710 emitSegmentOverridePrefix(2, MI, OS);
711 // Emit AdSize prefix as needed.
712 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
713 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
714 emitByte(0x67, OS);
715 CurOp += 3; // Consume operands.
716 break;
717 }
718 case X86II::RawFrmSrc: {
719 unsigned siReg = MI.getOperand(0).getReg();
720 // Emit segment override opcode prefix as needed (not for %ds).
721 if (MI.getOperand(1).getReg() != X86::DS)
722 emitSegmentOverridePrefix(1, MI, OS);
723 // Emit AdSize prefix as needed.
724 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::ESI) ||
725 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::SI))
726 emitByte(0x67, OS);
727 CurOp += 2; // Consume operands.
728 break;
729 }
730 case X86II::RawFrmDst: {
731 unsigned siReg = MI.getOperand(0).getReg();
732 // Emit AdSize prefix as needed.
733 if ((!STI.hasFeature(X86::Mode32Bit) && siReg == X86::EDI) ||
734 (STI.hasFeature(X86::Mode32Bit) && siReg == X86::DI))
735 emitByte(0x67, OS);
736 ++CurOp; // Consume operand.
737 break;
738 }
739 case X86II::RawFrmMemOffs: {
740 // Emit segment override opcode prefix as needed.
741 emitSegmentOverridePrefix(1, MI, OS);
742 break;
743 }
744 }
745
746 return HasREX;
747 }
748
749 /// AVX instructions are encoded using a opcode prefix called VEX.
emitVEXOpcodePrefix(int MemOperand,const MCInst & MI,raw_ostream & OS) const750 void X86MCCodeEmitter::emitVEXOpcodePrefix(int MemOperand, const MCInst &MI,
751 raw_ostream &OS) const {
752 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
753 uint64_t TSFlags = Desc.TSFlags;
754
755 assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
756
757 uint64_t Encoding = TSFlags & X86II::EncodingMask;
758 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
759 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
760 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
761
762 // VEX_R: opcode externsion equivalent to REX.R in
763 // 1's complement (inverted) form
764 //
765 // 1: Same as REX_R=0 (must be 1 in 32-bit mode)
766 // 0: Same as REX_R=1 (64 bit mode only)
767 //
768 uint8_t VEX_R = 0x1;
769 uint8_t EVEX_R2 = 0x1;
770
771 // VEX_X: equivalent to REX.X, only used when a
772 // register is used for index in SIB Byte.
773 //
774 // 1: Same as REX.X=0 (must be 1 in 32-bit mode)
775 // 0: Same as REX.X=1 (64-bit mode only)
776 uint8_t VEX_X = 0x1;
777
778 // VEX_B:
779 //
780 // 1: Same as REX_B=0 (ignored in 32-bit mode)
781 // 0: Same as REX_B=1 (64 bit mode only)
782 //
783 uint8_t VEX_B = 0x1;
784
785 // VEX_W: opcode specific (use like REX.W, or used for
786 // opcode extension, or ignored, depending on the opcode byte)
787 uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 1 : 0;
788
789 // VEX_5M (VEX m-mmmmm field):
790 //
791 // 0b00000: Reserved for future use
792 // 0b00001: implied 0F leading opcode
793 // 0b00010: implied 0F 38 leading opcode bytes
794 // 0b00011: implied 0F 3A leading opcode bytes
795 // 0b00100-0b11111: Reserved for future use
796 // 0b01000: XOP map select - 08h instructions with imm byte
797 // 0b01001: XOP map select - 09h instructions with no imm byte
798 // 0b01010: XOP map select - 0Ah instructions with imm dword
799 uint8_t VEX_5M;
800 switch (TSFlags & X86II::OpMapMask) {
801 default:
802 llvm_unreachable("Invalid prefix!");
803 case X86II::TB:
804 VEX_5M = 0x1;
805 break; // 0F
806 case X86II::T8:
807 VEX_5M = 0x2;
808 break; // 0F 38
809 case X86II::TA:
810 VEX_5M = 0x3;
811 break; // 0F 3A
812 case X86II::XOP8:
813 VEX_5M = 0x8;
814 break;
815 case X86II::XOP9:
816 VEX_5M = 0x9;
817 break;
818 case X86II::XOPA:
819 VEX_5M = 0xA;
820 break;
821 }
822
823 // VEX_4V (VEX vvvv field): a register specifier
824 // (in 1's complement form) or 1111 if unused.
825 uint8_t VEX_4V = 0xf;
826 uint8_t EVEX_V2 = 0x1;
827
828 // EVEX_L2/VEX_L (Vector Length):
829 //
830 // L2 L
831 // 0 0: scalar or 128-bit vector
832 // 0 1: 256-bit vector
833 // 1 0: 512-bit vector
834 //
835 uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 1 : 0;
836 uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 1 : 0;
837
838 // VEX_PP: opcode extension providing equivalent
839 // functionality of a SIMD prefix
840 //
841 // 0b00: None
842 // 0b01: 66
843 // 0b10: F3
844 // 0b11: F2
845 //
846 uint8_t VEX_PP = 0;
847 switch (TSFlags & X86II::OpPrefixMask) {
848 case X86II::PD:
849 VEX_PP = 0x1;
850 break; // 66
851 case X86II::XS:
852 VEX_PP = 0x2;
853 break; // F3
854 case X86II::XD:
855 VEX_PP = 0x3;
856 break; // F2
857 }
858
859 // EVEX_U
860 uint8_t EVEX_U = 1; // Always '1' so far
861
862 // EVEX_z
863 uint8_t EVEX_z = (HasEVEX_K && (TSFlags & X86II::EVEX_Z)) ? 1 : 0;
864
865 // EVEX_b
866 uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 1 : 0;
867
868 // EVEX_rc
869 uint8_t EVEX_rc = 0;
870
871 // EVEX_aaa
872 uint8_t EVEX_aaa = 0;
873
874 bool EncodeRC = false;
875
876 // Classify VEX_B, VEX_4V, VEX_R, VEX_X
877 unsigned NumOps = Desc.getNumOperands();
878 unsigned CurOp = X86II::getOperandBias(Desc);
879
880 switch (TSFlags & X86II::FormMask) {
881 default:
882 llvm_unreachable("Unexpected form in emitVEXOpcodePrefix!");
883 case X86II::MRM_C0:
884 case X86II::RawFrm:
885 case X86II::PrefixByte:
886 break;
887 case X86II::MRMDestMemFSIB:
888 case X86II::MRMDestMem: {
889 // MRMDestMem instructions forms:
890 // MemAddr, src1(ModR/M)
891 // MemAddr, src1(VEX_4V), src2(ModR/M)
892 // MemAddr, src1(ModR/M), imm8
893 //
894 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
895 VEX_B = ~(BaseRegEnc >> 3) & 1;
896 unsigned IndexRegEnc =
897 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
898 VEX_X = ~(IndexRegEnc >> 3) & 1;
899 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
900 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
901
902 CurOp += X86::AddrNumOperands;
903
904 if (HasEVEX_K)
905 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
906
907 if (HasVEX_4V) {
908 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
909 VEX_4V = ~VRegEnc & 0xf;
910 EVEX_V2 = ~(VRegEnc >> 4) & 1;
911 }
912
913 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
914 VEX_R = ~(RegEnc >> 3) & 1;
915 EVEX_R2 = ~(RegEnc >> 4) & 1;
916 break;
917 }
918 case X86II::MRMSrcMemFSIB:
919 case X86II::MRMSrcMem: {
920 // MRMSrcMem instructions forms:
921 // src1(ModR/M), MemAddr
922 // src1(ModR/M), src2(VEX_4V), MemAddr
923 // src1(ModR/M), MemAddr, imm8
924 // src1(ModR/M), MemAddr, src2(Imm[7:4])
925 //
926 // FMA4:
927 // dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
928 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
929 VEX_R = ~(RegEnc >> 3) & 1;
930 EVEX_R2 = ~(RegEnc >> 4) & 1;
931
932 if (HasEVEX_K)
933 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
934
935 if (HasVEX_4V) {
936 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
937 VEX_4V = ~VRegEnc & 0xf;
938 EVEX_V2 = ~(VRegEnc >> 4) & 1;
939 }
940
941 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
942 VEX_B = ~(BaseRegEnc >> 3) & 1;
943 unsigned IndexRegEnc =
944 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
945 VEX_X = ~(IndexRegEnc >> 3) & 1;
946 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
947 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
948
949 break;
950 }
951 case X86II::MRMSrcMem4VOp3: {
952 // Instruction format for 4VOp3:
953 // src1(ModR/M), MemAddr, src3(VEX_4V)
954 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
955 VEX_R = ~(RegEnc >> 3) & 1;
956
957 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
958 VEX_B = ~(BaseRegEnc >> 3) & 1;
959 unsigned IndexRegEnc =
960 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
961 VEX_X = ~(IndexRegEnc >> 3) & 1;
962
963 VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
964 break;
965 }
966 case X86II::MRMSrcMemOp4: {
967 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
968 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
969 VEX_R = ~(RegEnc >> 3) & 1;
970
971 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
972 VEX_4V = ~VRegEnc & 0xf;
973
974 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
975 VEX_B = ~(BaseRegEnc >> 3) & 1;
976 unsigned IndexRegEnc =
977 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
978 VEX_X = ~(IndexRegEnc >> 3) & 1;
979 break;
980 }
981 case X86II::MRM0m:
982 case X86II::MRM1m:
983 case X86II::MRM2m:
984 case X86II::MRM3m:
985 case X86II::MRM4m:
986 case X86II::MRM5m:
987 case X86II::MRM6m:
988 case X86II::MRM7m: {
989 // MRM[0-9]m instructions forms:
990 // MemAddr
991 // src1(VEX_4V), MemAddr
992 if (HasVEX_4V) {
993 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
994 VEX_4V = ~VRegEnc & 0xf;
995 EVEX_V2 = ~(VRegEnc >> 4) & 1;
996 }
997
998 if (HasEVEX_K)
999 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1000
1001 unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
1002 VEX_B = ~(BaseRegEnc >> 3) & 1;
1003 unsigned IndexRegEnc =
1004 getX86RegEncoding(MI, MemOperand + X86::AddrIndexReg);
1005 VEX_X = ~(IndexRegEnc >> 3) & 1;
1006 if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
1007 EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
1008
1009 break;
1010 }
1011 case X86II::MRMSrcReg: {
1012 // MRMSrcReg instructions forms:
1013 // dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
1014 // dst(ModR/M), src1(ModR/M)
1015 // dst(ModR/M), src1(ModR/M), imm8
1016 //
1017 // FMA4:
1018 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1019 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1020 VEX_R = ~(RegEnc >> 3) & 1;
1021 EVEX_R2 = ~(RegEnc >> 4) & 1;
1022
1023 if (HasEVEX_K)
1024 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1025
1026 if (HasVEX_4V) {
1027 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1028 VEX_4V = ~VRegEnc & 0xf;
1029 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1030 }
1031
1032 RegEnc = getX86RegEncoding(MI, CurOp++);
1033 VEX_B = ~(RegEnc >> 3) & 1;
1034 VEX_X = ~(RegEnc >> 4) & 1;
1035
1036 if (EVEX_b) {
1037 if (HasEVEX_RC) {
1038 unsigned RcOperand = NumOps - 1;
1039 assert(RcOperand >= CurOp);
1040 EVEX_rc = MI.getOperand(RcOperand).getImm();
1041 assert(EVEX_rc <= 3 && "Invalid rounding control!");
1042 }
1043 EncodeRC = true;
1044 }
1045 break;
1046 }
1047 case X86II::MRMSrcReg4VOp3: {
1048 // Instruction format for 4VOp3:
1049 // src1(ModR/M), src2(ModR/M), src3(VEX_4V)
1050 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1051 VEX_R = ~(RegEnc >> 3) & 1;
1052
1053 RegEnc = getX86RegEncoding(MI, CurOp++);
1054 VEX_B = ~(RegEnc >> 3) & 1;
1055
1056 VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
1057 break;
1058 }
1059 case X86II::MRMSrcRegOp4: {
1060 // dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
1061 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1062 VEX_R = ~(RegEnc >> 3) & 1;
1063
1064 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1065 VEX_4V = ~VRegEnc & 0xf;
1066
1067 // Skip second register source (encoded in Imm[7:4])
1068 ++CurOp;
1069
1070 RegEnc = getX86RegEncoding(MI, CurOp++);
1071 VEX_B = ~(RegEnc >> 3) & 1;
1072 VEX_X = ~(RegEnc >> 4) & 1;
1073 break;
1074 }
1075 case X86II::MRMDestReg: {
1076 // MRMDestReg instructions forms:
1077 // dst(ModR/M), src(ModR/M)
1078 // dst(ModR/M), src(ModR/M), imm8
1079 // dst(ModR/M), src1(VEX_4V), src2(ModR/M)
1080 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1081 VEX_B = ~(RegEnc >> 3) & 1;
1082 VEX_X = ~(RegEnc >> 4) & 1;
1083
1084 if (HasEVEX_K)
1085 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1086
1087 if (HasVEX_4V) {
1088 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1089 VEX_4V = ~VRegEnc & 0xf;
1090 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1091 }
1092
1093 RegEnc = getX86RegEncoding(MI, CurOp++);
1094 VEX_R = ~(RegEnc >> 3) & 1;
1095 EVEX_R2 = ~(RegEnc >> 4) & 1;
1096 if (EVEX_b)
1097 EncodeRC = true;
1098 break;
1099 }
1100 case X86II::MRMr0: {
1101 // MRMr0 instructions forms:
1102 // 11:rrr:000
1103 // dst(ModR/M)
1104 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1105 VEX_R = ~(RegEnc >> 3) & 1;
1106 EVEX_R2 = ~(RegEnc >> 4) & 1;
1107 break;
1108 }
1109 case X86II::MRM0r:
1110 case X86II::MRM1r:
1111 case X86II::MRM2r:
1112 case X86II::MRM3r:
1113 case X86II::MRM4r:
1114 case X86II::MRM5r:
1115 case X86II::MRM6r:
1116 case X86II::MRM7r: {
1117 // MRM0r-MRM7r instructions forms:
1118 // dst(VEX_4V), src(ModR/M), imm8
1119 if (HasVEX_4V) {
1120 unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
1121 VEX_4V = ~VRegEnc & 0xf;
1122 EVEX_V2 = ~(VRegEnc >> 4) & 1;
1123 }
1124 if (HasEVEX_K)
1125 EVEX_aaa = getX86RegEncoding(MI, CurOp++);
1126
1127 unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
1128 VEX_B = ~(RegEnc >> 3) & 1;
1129 VEX_X = ~(RegEnc >> 4) & 1;
1130 break;
1131 }
1132 }
1133
1134 if (Encoding == X86II::VEX || Encoding == X86II::XOP) {
1135 // VEX opcode prefix can have 2 or 3 bytes
1136 //
1137 // 3 bytes:
1138 // +-----+ +--------------+ +-------------------+
1139 // | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
1140 // +-----+ +--------------+ +-------------------+
1141 // 2 bytes:
1142 // +-----+ +-------------------+
1143 // | C5h | | R | vvvv | L | pp |
1144 // +-----+ +-------------------+
1145 //
1146 // XOP uses a similar prefix:
1147 // +-----+ +--------------+ +-------------------+
1148 // | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
1149 // +-----+ +--------------+ +-------------------+
1150 uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
1151
1152 // Can we use the 2 byte VEX prefix?
1153 if (!(MI.getFlags() & X86::IP_USE_VEX3) && Encoding == X86II::VEX &&
1154 VEX_B && VEX_X && !VEX_W && (VEX_5M == 1)) {
1155 emitByte(0xC5, OS);
1156 emitByte(LastByte | (VEX_R << 7), OS);
1157 return;
1158 }
1159
1160 // 3 byte VEX prefix
1161 emitByte(Encoding == X86II::XOP ? 0x8F : 0xC4, OS);
1162 emitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, OS);
1163 emitByte(LastByte | (VEX_W << 7), OS);
1164 } else {
1165 assert(Encoding == X86II::EVEX && "unknown encoding!");
1166 // EVEX opcode prefix can have 4 bytes
1167 //
1168 // +-----+ +--------------+ +-------------------+ +------------------------+
1169 // | 62h | | RXBR' | 00mm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
1170 // +-----+ +--------------+ +-------------------+ +------------------------+
1171 assert((VEX_5M & 0x3) == VEX_5M &&
1172 "More than 2 significant bits in VEX.m-mmmm fields for EVEX!");
1173
1174 emitByte(0x62, OS);
1175 emitByte((VEX_R << 7) | (VEX_X << 6) | (VEX_B << 5) | (EVEX_R2 << 4) |
1176 VEX_5M,
1177 OS);
1178 emitByte((VEX_W << 7) | (VEX_4V << 3) | (EVEX_U << 2) | VEX_PP, OS);
1179 if (EncodeRC)
1180 emitByte((EVEX_z << 7) | (EVEX_rc << 5) | (EVEX_b << 4) | (EVEX_V2 << 3) |
1181 EVEX_aaa,
1182 OS);
1183 else
1184 emitByte((EVEX_z << 7) | (EVEX_L2 << 6) | (VEX_L << 5) | (EVEX_b << 4) |
1185 (EVEX_V2 << 3) | EVEX_aaa,
1186 OS);
1187 }
1188 }
1189
1190 /// Emit REX prefix which specifies
1191 /// 1) 64-bit instructions,
1192 /// 2) non-default operand size, and
1193 /// 3) use of X86-64 extended registers.
1194 ///
1195 /// \returns true if REX prefix is used, otherwise returns false.
emitREXPrefix(int MemOperand,const MCInst & MI,raw_ostream & OS) const1196 bool X86MCCodeEmitter::emitREXPrefix(int MemOperand, const MCInst &MI,
1197 raw_ostream &OS) const {
1198 uint8_t REX = [&, MemOperand]() {
1199 uint8_t REX = 0;
1200 bool UsesHighByteReg = false;
1201
1202 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1203 uint64_t TSFlags = Desc.TSFlags;
1204
1205 if (TSFlags & X86II::REX_W)
1206 REX |= 1 << 3; // set REX.W
1207
1208 if (MI.getNumOperands() == 0)
1209 return REX;
1210
1211 unsigned NumOps = MI.getNumOperands();
1212 unsigned CurOp = X86II::getOperandBias(Desc);
1213
1214 // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1215 for (unsigned i = CurOp; i != NumOps; ++i) {
1216 const MCOperand &MO = MI.getOperand(i);
1217 if (!MO.isReg())
1218 continue;
1219 unsigned Reg = MO.getReg();
1220 if (Reg == X86::AH || Reg == X86::BH || Reg == X86::CH || Reg == X86::DH)
1221 UsesHighByteReg = true;
1222 if (X86II::isX86_64NonExtLowByteReg(Reg))
1223 // FIXME: The caller of determineREXPrefix slaps this prefix onto
1224 // anything that returns non-zero.
1225 REX |= 0x40; // REX fixed encoding prefix
1226 }
1227
1228 switch (TSFlags & X86II::FormMask) {
1229 case X86II::AddRegFrm:
1230 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1231 break;
1232 case X86II::MRMSrcReg:
1233 case X86II::MRMSrcRegCC:
1234 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1235 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1236 break;
1237 case X86II::MRMSrcMem:
1238 case X86II::MRMSrcMemCC:
1239 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1240 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1241 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1242 CurOp += X86::AddrNumOperands;
1243 break;
1244 case X86II::MRMDestReg:
1245 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1246 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1247 break;
1248 case X86II::MRMDestMem:
1249 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1250 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1251 CurOp += X86::AddrNumOperands;
1252 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1253 break;
1254 case X86II::MRMXmCC:
1255 case X86II::MRMXm:
1256 case X86II::MRM0m:
1257 case X86II::MRM1m:
1258 case X86II::MRM2m:
1259 case X86II::MRM3m:
1260 case X86II::MRM4m:
1261 case X86II::MRM5m:
1262 case X86II::MRM6m:
1263 case X86II::MRM7m:
1264 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrBaseReg) << 0; // REX.B
1265 REX |= isREXExtendedReg(MI, MemOperand + X86::AddrIndexReg) << 1; // REX.X
1266 break;
1267 case X86II::MRMXrCC:
1268 case X86II::MRMXr:
1269 case X86II::MRM0r:
1270 case X86II::MRM1r:
1271 case X86II::MRM2r:
1272 case X86II::MRM3r:
1273 case X86II::MRM4r:
1274 case X86II::MRM5r:
1275 case X86II::MRM6r:
1276 case X86II::MRM7r:
1277 REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1278 break;
1279 case X86II::MRMr0:
1280 REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1281 break;
1282 case X86II::MRMDestMemFSIB:
1283 llvm_unreachable("FSIB format never need REX prefix!");
1284 }
1285 if (REX && UsesHighByteReg)
1286 report_fatal_error(
1287 "Cannot encode high byte register in REX-prefixed instruction");
1288 return REX;
1289 }();
1290
1291 if (!REX)
1292 return false;
1293
1294 emitByte(0x40 | REX, OS);
1295 return true;
1296 }
1297
1298 /// Emit segment override opcode prefix as needed.
emitSegmentOverridePrefix(unsigned SegOperand,const MCInst & MI,raw_ostream & OS) const1299 void X86MCCodeEmitter::emitSegmentOverridePrefix(unsigned SegOperand,
1300 const MCInst &MI,
1301 raw_ostream &OS) const {
1302 // Check for explicit segment override on memory operand.
1303 if (unsigned Reg = MI.getOperand(SegOperand).getReg())
1304 emitByte(X86::getSegmentOverridePrefixForReg(Reg), OS);
1305 }
1306
1307 /// Emit all instruction prefixes prior to the opcode.
1308 ///
1309 /// \param MemOperand the operand # of the start of a memory operand if present.
1310 /// If not present, it is -1.
1311 ///
1312 /// \returns true if REX prefix is used, otherwise returns false.
emitOpcodePrefix(int MemOperand,const MCInst & MI,const MCSubtargetInfo & STI,raw_ostream & OS) const1313 bool X86MCCodeEmitter::emitOpcodePrefix(int MemOperand, const MCInst &MI,
1314 const MCSubtargetInfo &STI,
1315 raw_ostream &OS) const {
1316 const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
1317 uint64_t TSFlags = Desc.TSFlags;
1318
1319 // Emit the operand size opcode prefix as needed.
1320 if ((TSFlags & X86II::OpSizeMask) ==
1321 (STI.hasFeature(X86::Mode16Bit) ? X86II::OpSize32 : X86II::OpSize16))
1322 emitByte(0x66, OS);
1323
1324 // Emit the LOCK opcode prefix.
1325 if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK)
1326 emitByte(0xF0, OS);
1327
1328 // Emit the NOTRACK opcode prefix.
1329 if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK)
1330 emitByte(0x3E, OS);
1331
1332 switch (TSFlags & X86II::OpPrefixMask) {
1333 case X86II::PD: // 66
1334 emitByte(0x66, OS);
1335 break;
1336 case X86II::XS: // F3
1337 emitByte(0xF3, OS);
1338 break;
1339 case X86II::XD: // F2
1340 emitByte(0xF2, OS);
1341 break;
1342 }
1343
1344 // Handle REX prefix.
1345 assert((STI.hasFeature(X86::Mode64Bit) || !(TSFlags & X86II::REX_W)) &&
1346 "REX.W requires 64bit mode.");
1347 bool HasREX = STI.hasFeature(X86::Mode64Bit)
1348 ? emitREXPrefix(MemOperand, MI, OS)
1349 : false;
1350
1351 // 0x0F escape code must be emitted just before the opcode.
1352 switch (TSFlags & X86II::OpMapMask) {
1353 case X86II::TB: // Two-byte opcode map
1354 case X86II::T8: // 0F 38
1355 case X86II::TA: // 0F 3A
1356 case X86II::ThreeDNow: // 0F 0F, second 0F emitted by caller.
1357 emitByte(0x0F, OS);
1358 break;
1359 }
1360
1361 switch (TSFlags & X86II::OpMapMask) {
1362 case X86II::T8: // 0F 38
1363 emitByte(0x38, OS);
1364 break;
1365 case X86II::TA: // 0F 3A
1366 emitByte(0x3A, OS);
1367 break;
1368 }
1369
1370 return HasREX;
1371 }
1372
emitPrefix(const MCInst & MI,raw_ostream & OS,const MCSubtargetInfo & STI) const1373 void X86MCCodeEmitter::emitPrefix(const MCInst &MI, raw_ostream &OS,
1374 const MCSubtargetInfo &STI) const {
1375 unsigned Opcode = MI.getOpcode();
1376 const MCInstrDesc &Desc = MCII.get(Opcode);
1377 uint64_t TSFlags = Desc.TSFlags;
1378
1379 // Pseudo instructions don't get encoded.
1380 if (X86II::isPseudo(TSFlags))
1381 return;
1382
1383 unsigned CurOp = X86II::getOperandBias(Desc);
1384
1385 emitPrefixImpl(CurOp, MI, STI, OS);
1386 }
1387
encodeInstruction(const MCInst & MI,raw_ostream & OS,SmallVectorImpl<MCFixup> & Fixups,const MCSubtargetInfo & STI) const1388 void X86MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
1389 SmallVectorImpl<MCFixup> &Fixups,
1390 const MCSubtargetInfo &STI) const {
1391 unsigned Opcode = MI.getOpcode();
1392 const MCInstrDesc &Desc = MCII.get(Opcode);
1393 uint64_t TSFlags = Desc.TSFlags;
1394
1395 // Pseudo instructions don't get encoded.
1396 if (X86II::isPseudo(TSFlags))
1397 return;
1398
1399 unsigned NumOps = Desc.getNumOperands();
1400 unsigned CurOp = X86II::getOperandBias(Desc);
1401
1402 uint64_t StartByte = OS.tell();
1403
1404 bool HasREX = emitPrefixImpl(CurOp, MI, STI, OS);
1405
1406 // It uses the VEX.VVVV field?
1407 bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1408 bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1409
1410 // It uses the EVEX.aaa field?
1411 bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1412 bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1413
1414 // Used if a register is encoded in 7:4 of immediate.
1415 unsigned I8RegNum = 0;
1416
1417 uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1418
1419 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1420 BaseOpcode = 0x0F; // Weird 3DNow! encoding.
1421
1422 unsigned OpcodeOffset = 0;
1423
1424 uint64_t Form = TSFlags & X86II::FormMask;
1425 switch (Form) {
1426 default:
1427 errs() << "FORM: " << Form << "\n";
1428 llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1429 case X86II::Pseudo:
1430 llvm_unreachable("Pseudo instruction shouldn't be emitted");
1431 case X86II::RawFrmDstSrc:
1432 case X86II::RawFrmSrc:
1433 case X86II::RawFrmDst:
1434 case X86II::PrefixByte:
1435 emitByte(BaseOpcode, OS);
1436 break;
1437 case X86II::AddCCFrm: {
1438 // This will be added to the opcode in the fallthrough.
1439 OpcodeOffset = MI.getOperand(NumOps - 1).getImm();
1440 assert(OpcodeOffset < 16 && "Unexpected opcode offset!");
1441 --NumOps; // Drop the operand from the end.
1442 LLVM_FALLTHROUGH;
1443 case X86II::RawFrm:
1444 emitByte(BaseOpcode + OpcodeOffset, OS);
1445
1446 if (!STI.hasFeature(X86::Mode64Bit) || !isPCRel32Branch(MI, MCII))
1447 break;
1448
1449 const MCOperand &Op = MI.getOperand(CurOp++);
1450 emitImmediate(Op, MI.getLoc(), X86II::getSizeOfImm(TSFlags),
1451 MCFixupKind(X86::reloc_branch_4byte_pcrel), StartByte, OS,
1452 Fixups);
1453 break;
1454 }
1455 case X86II::RawFrmMemOffs:
1456 emitByte(BaseOpcode, OS);
1457 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1458 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1459 StartByte, OS, Fixups);
1460 ++CurOp; // skip segment operand
1461 break;
1462 case X86II::RawFrmImm8:
1463 emitByte(BaseOpcode, OS);
1464 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1465 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1466 StartByte, OS, Fixups);
1467 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, StartByte,
1468 OS, Fixups);
1469 break;
1470 case X86II::RawFrmImm16:
1471 emitByte(BaseOpcode, OS);
1472 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1473 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1474 StartByte, OS, Fixups);
1475 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, StartByte,
1476 OS, Fixups);
1477 break;
1478
1479 case X86II::AddRegFrm:
1480 emitByte(BaseOpcode + getX86RegNum(MI.getOperand(CurOp++)), OS);
1481 break;
1482
1483 case X86II::MRMDestReg: {
1484 emitByte(BaseOpcode, OS);
1485 unsigned SrcRegNum = CurOp + 1;
1486
1487 if (HasEVEX_K) // Skip writemask
1488 ++SrcRegNum;
1489
1490 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1491 ++SrcRegNum;
1492
1493 emitRegModRMByte(MI.getOperand(CurOp),
1494 getX86RegNum(MI.getOperand(SrcRegNum)), OS);
1495 CurOp = SrcRegNum + 1;
1496 break;
1497 }
1498 case X86II::MRMDestMemFSIB:
1499 case X86II::MRMDestMem: {
1500 emitByte(BaseOpcode, OS);
1501 unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1502
1503 if (HasEVEX_K) // Skip writemask
1504 ++SrcRegNum;
1505
1506 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1507 ++SrcRegNum;
1508
1509 bool ForceSIB = (Form == X86II::MRMDestMemFSIB);
1510 emitMemModRMByte(MI, CurOp, getX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1511 HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1512 CurOp = SrcRegNum + 1;
1513 break;
1514 }
1515 case X86II::MRMSrcReg: {
1516 emitByte(BaseOpcode, OS);
1517 unsigned SrcRegNum = CurOp + 1;
1518
1519 if (HasEVEX_K) // Skip writemask
1520 ++SrcRegNum;
1521
1522 if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1523 ++SrcRegNum;
1524
1525 emitRegModRMByte(MI.getOperand(SrcRegNum),
1526 getX86RegNum(MI.getOperand(CurOp)), OS);
1527 CurOp = SrcRegNum + 1;
1528 if (HasVEX_I8Reg)
1529 I8RegNum = getX86RegEncoding(MI, CurOp++);
1530 // do not count the rounding control operand
1531 if (HasEVEX_RC)
1532 --NumOps;
1533 break;
1534 }
1535 case X86II::MRMSrcReg4VOp3: {
1536 emitByte(BaseOpcode, OS);
1537 unsigned SrcRegNum = CurOp + 1;
1538
1539 emitRegModRMByte(MI.getOperand(SrcRegNum),
1540 getX86RegNum(MI.getOperand(CurOp)), OS);
1541 CurOp = SrcRegNum + 1;
1542 ++CurOp; // Encoded in VEX.VVVV
1543 break;
1544 }
1545 case X86II::MRMSrcRegOp4: {
1546 emitByte(BaseOpcode, OS);
1547 unsigned SrcRegNum = CurOp + 1;
1548
1549 // Skip 1st src (which is encoded in VEX_VVVV)
1550 ++SrcRegNum;
1551
1552 // Capture 2nd src (which is encoded in Imm[7:4])
1553 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1554 I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1555
1556 emitRegModRMByte(MI.getOperand(SrcRegNum),
1557 getX86RegNum(MI.getOperand(CurOp)), OS);
1558 CurOp = SrcRegNum + 1;
1559 break;
1560 }
1561 case X86II::MRMSrcRegCC: {
1562 unsigned FirstOp = CurOp++;
1563 unsigned SecondOp = CurOp++;
1564
1565 unsigned CC = MI.getOperand(CurOp++).getImm();
1566 emitByte(BaseOpcode + CC, OS);
1567
1568 emitRegModRMByte(MI.getOperand(SecondOp),
1569 getX86RegNum(MI.getOperand(FirstOp)), OS);
1570 break;
1571 }
1572 case X86II::MRMSrcMemFSIB:
1573 case X86II::MRMSrcMem: {
1574 unsigned FirstMemOp = CurOp + 1;
1575
1576 if (HasEVEX_K) // Skip writemask
1577 ++FirstMemOp;
1578
1579 if (HasVEX_4V)
1580 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1581
1582 emitByte(BaseOpcode, OS);
1583
1584 bool ForceSIB = (Form == X86II::MRMSrcMemFSIB);
1585 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1586 TSFlags, HasREX, StartByte, OS, Fixups, STI, ForceSIB);
1587 CurOp = FirstMemOp + X86::AddrNumOperands;
1588 if (HasVEX_I8Reg)
1589 I8RegNum = getX86RegEncoding(MI, CurOp++);
1590 break;
1591 }
1592 case X86II::MRMSrcMem4VOp3: {
1593 unsigned FirstMemOp = CurOp + 1;
1594
1595 emitByte(BaseOpcode, OS);
1596
1597 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1598 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1599 CurOp = FirstMemOp + X86::AddrNumOperands;
1600 ++CurOp; // Encoded in VEX.VVVV.
1601 break;
1602 }
1603 case X86II::MRMSrcMemOp4: {
1604 unsigned FirstMemOp = CurOp + 1;
1605
1606 ++FirstMemOp; // Skip the register source (which is encoded in VEX_VVVV).
1607
1608 // Capture second register source (encoded in Imm[7:4])
1609 assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1610 I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1611
1612 emitByte(BaseOpcode, OS);
1613
1614 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(CurOp)),
1615 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1616 CurOp = FirstMemOp + X86::AddrNumOperands;
1617 break;
1618 }
1619 case X86II::MRMSrcMemCC: {
1620 unsigned RegOp = CurOp++;
1621 unsigned FirstMemOp = CurOp;
1622 CurOp = FirstMemOp + X86::AddrNumOperands;
1623
1624 unsigned CC = MI.getOperand(CurOp++).getImm();
1625 emitByte(BaseOpcode + CC, OS);
1626
1627 emitMemModRMByte(MI, FirstMemOp, getX86RegNum(MI.getOperand(RegOp)),
1628 TSFlags, HasREX, StartByte, OS, Fixups, STI);
1629 break;
1630 }
1631
1632 case X86II::MRMXrCC: {
1633 unsigned RegOp = CurOp++;
1634
1635 unsigned CC = MI.getOperand(CurOp++).getImm();
1636 emitByte(BaseOpcode + CC, OS);
1637 emitRegModRMByte(MI.getOperand(RegOp), 0, OS);
1638 break;
1639 }
1640
1641 case X86II::MRMXr:
1642 case X86II::MRM0r:
1643 case X86II::MRM1r:
1644 case X86II::MRM2r:
1645 case X86II::MRM3r:
1646 case X86II::MRM4r:
1647 case X86II::MRM5r:
1648 case X86II::MRM6r:
1649 case X86II::MRM7r:
1650 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1651 ++CurOp;
1652 if (HasEVEX_K) // Skip writemask
1653 ++CurOp;
1654 emitByte(BaseOpcode, OS);
1655 emitRegModRMByte(MI.getOperand(CurOp++),
1656 (Form == X86II::MRMXr) ? 0 : Form - X86II::MRM0r, OS);
1657 break;
1658 case X86II::MRMr0:
1659 emitByte(BaseOpcode, OS);
1660 emitByte(modRMByte(3, getX86RegNum(MI.getOperand(CurOp++)),0), OS);
1661 break;
1662
1663 case X86II::MRMXmCC: {
1664 unsigned FirstMemOp = CurOp;
1665 CurOp = FirstMemOp + X86::AddrNumOperands;
1666
1667 unsigned CC = MI.getOperand(CurOp++).getImm();
1668 emitByte(BaseOpcode + CC, OS);
1669
1670 emitMemModRMByte(MI, FirstMemOp, 0, TSFlags, HasREX, StartByte, OS, Fixups,
1671 STI);
1672 break;
1673 }
1674
1675 case X86II::MRMXm:
1676 case X86II::MRM0m:
1677 case X86II::MRM1m:
1678 case X86II::MRM2m:
1679 case X86II::MRM3m:
1680 case X86II::MRM4m:
1681 case X86II::MRM5m:
1682 case X86II::MRM6m:
1683 case X86II::MRM7m:
1684 if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1685 ++CurOp;
1686 if (HasEVEX_K) // Skip writemask
1687 ++CurOp;
1688 emitByte(BaseOpcode, OS);
1689 emitMemModRMByte(MI, CurOp,
1690 (Form == X86II::MRMXm) ? 0 : Form - X86II::MRM0m, TSFlags,
1691 HasREX, StartByte, OS, Fixups, STI);
1692 CurOp += X86::AddrNumOperands;
1693 break;
1694
1695 case X86II::MRM0X:
1696 case X86II::MRM1X:
1697 case X86II::MRM2X:
1698 case X86II::MRM3X:
1699 case X86II::MRM4X:
1700 case X86II::MRM5X:
1701 case X86II::MRM6X:
1702 case X86II::MRM7X:
1703 emitByte(BaseOpcode, OS);
1704 emitByte(0xC0 + ((Form - X86II::MRM0X) << 3), OS);
1705 break;
1706
1707 case X86II::MRM_C0:
1708 case X86II::MRM_C1:
1709 case X86II::MRM_C2:
1710 case X86II::MRM_C3:
1711 case X86II::MRM_C4:
1712 case X86II::MRM_C5:
1713 case X86II::MRM_C6:
1714 case X86II::MRM_C7:
1715 case X86II::MRM_C8:
1716 case X86II::MRM_C9:
1717 case X86II::MRM_CA:
1718 case X86II::MRM_CB:
1719 case X86II::MRM_CC:
1720 case X86II::MRM_CD:
1721 case X86II::MRM_CE:
1722 case X86II::MRM_CF:
1723 case X86II::MRM_D0:
1724 case X86II::MRM_D1:
1725 case X86II::MRM_D2:
1726 case X86II::MRM_D3:
1727 case X86II::MRM_D4:
1728 case X86II::MRM_D5:
1729 case X86II::MRM_D6:
1730 case X86II::MRM_D7:
1731 case X86II::MRM_D8:
1732 case X86II::MRM_D9:
1733 case X86II::MRM_DA:
1734 case X86II::MRM_DB:
1735 case X86II::MRM_DC:
1736 case X86II::MRM_DD:
1737 case X86II::MRM_DE:
1738 case X86II::MRM_DF:
1739 case X86II::MRM_E0:
1740 case X86II::MRM_E1:
1741 case X86II::MRM_E2:
1742 case X86II::MRM_E3:
1743 case X86II::MRM_E4:
1744 case X86II::MRM_E5:
1745 case X86II::MRM_E6:
1746 case X86II::MRM_E7:
1747 case X86II::MRM_E8:
1748 case X86II::MRM_E9:
1749 case X86II::MRM_EA:
1750 case X86II::MRM_EB:
1751 case X86II::MRM_EC:
1752 case X86II::MRM_ED:
1753 case X86II::MRM_EE:
1754 case X86II::MRM_EF:
1755 case X86II::MRM_F0:
1756 case X86II::MRM_F1:
1757 case X86II::MRM_F2:
1758 case X86II::MRM_F3:
1759 case X86II::MRM_F4:
1760 case X86II::MRM_F5:
1761 case X86II::MRM_F6:
1762 case X86II::MRM_F7:
1763 case X86II::MRM_F8:
1764 case X86II::MRM_F9:
1765 case X86II::MRM_FA:
1766 case X86II::MRM_FB:
1767 case X86II::MRM_FC:
1768 case X86II::MRM_FD:
1769 case X86II::MRM_FE:
1770 case X86II::MRM_FF:
1771 emitByte(BaseOpcode, OS);
1772 emitByte(0xC0 + Form - X86II::MRM_C0, OS);
1773 break;
1774 }
1775
1776 if (HasVEX_I8Reg) {
1777 // The last source register of a 4 operand instruction in AVX is encoded
1778 // in bits[7:4] of a immediate byte.
1779 assert(I8RegNum < 16 && "Register encoding out of range");
1780 I8RegNum <<= 4;
1781 if (CurOp != NumOps) {
1782 unsigned Val = MI.getOperand(CurOp++).getImm();
1783 assert(Val < 16 && "Immediate operand value out of range");
1784 I8RegNum |= Val;
1785 }
1786 emitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1787 StartByte, OS, Fixups);
1788 } else {
1789 // If there is a remaining operand, it must be a trailing immediate. Emit it
1790 // according to the right size for the instruction. Some instructions
1791 // (SSE4a extrq and insertq) have two trailing immediates.
1792 while (CurOp != NumOps && NumOps - CurOp <= 2) {
1793 emitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1794 X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1795 StartByte, OS, Fixups);
1796 }
1797 }
1798
1799 if ((TSFlags & X86II::OpMapMask) == X86II::ThreeDNow)
1800 emitByte(X86II::getBaseOpcodeFor(TSFlags), OS);
1801
1802 assert(OS.tell() - StartByte <= 15 &&
1803 "The size of instruction must be no longer than 15.");
1804 #ifndef NDEBUG
1805 // FIXME: Verify.
1806 if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1807 errs() << "Cannot encode all operands of: ";
1808 MI.dump();
1809 errs() << '\n';
1810 abort();
1811 }
1812 #endif
1813 }
1814
createX86MCCodeEmitter(const MCInstrInfo & MCII,const MCRegisterInfo & MRI,MCContext & Ctx)1815 MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1816 const MCRegisterInfo &MRI,
1817 MCContext &Ctx) {
1818 return new X86MCCodeEmitter(MCII, Ctx);
1819 }
1820