1 // AsmJit - Machine code generation for C++
2 //
3 // * Official AsmJit Home Page: https://asmjit.com
4 // * Official Github Repository: https://github.com/asmjit/asmjit
5 //
6 // Copyright (c) 2008-2020 The AsmJit Authors
7 //
8 // This software is provided 'as-is', without any express or implied
9 // warranty. In no event will the authors be held liable for any damages
10 // arising from the use of this software.
11 //
12 // Permission is granted to anyone to use this software for any purpose,
13 // including commercial applications, and to alter it and redistribute it
14 // freely, subject to the following restrictions:
15 //
16 // 1. The origin of this software must not be misrepresented; you must not
17 // claim that you wrote the original software. If you use this software
18 // in a product, an acknowledgment in the product documentation would be
19 // appreciated but is not required.
20 // 2. Altered source versions must be plainly marked as such, and must not be
21 // misrepresented as being the original software.
22 // 3. This notice may not be removed or altered from any source distribution.
23
24 #include "../core/api-build_p.h"
25 #if defined(ASMJIT_BUILD_X86) && !defined(ASMJIT_NO_COMPILER)
26
27 #include "../core/cpuinfo.h"
28 #include "../core/support.h"
29 #include "../core/type.h"
30 #include "../x86/x86assembler.h"
31 #include "../x86/x86compiler.h"
32 #include "../x86/x86instapi_p.h"
33 #include "../x86/x86instdb_p.h"
34 #include "../x86/x86internal_p.h"
35 #include "../x86/x86rapass_p.h"
36
ASMJIT_BEGIN_SUB_NAMESPACE(x86)37 ASMJIT_BEGIN_SUB_NAMESPACE(x86)
38
39 // ============================================================================
40 // [asmjit::x86::X86RAPass - Helpers]
41 // ============================================================================
42
43 static ASMJIT_INLINE uint64_t raImmMaskFromSize(uint32_t size) noexcept {
44 ASMJIT_ASSERT(size > 0 && size < 256);
45 static const uint64_t masks[] = {
46 0x00000000000000FFu, // 1
47 0x000000000000FFFFu, // 2
48 0x00000000FFFFFFFFu, // 4
49 0xFFFFFFFFFFFFFFFFu, // 8
50 0x0000000000000000u, // 16
51 0x0000000000000000u, // 32
52 0x0000000000000000u, // 64
53 0x0000000000000000u, // 128
54 0x0000000000000000u // 256
55 };
56 return masks[Support::ctz(size)];
57 }
58
raUseOutFlagsFromRWFlags(uint32_t rwFlags)59 static ASMJIT_INLINE uint32_t raUseOutFlagsFromRWFlags(uint32_t rwFlags) noexcept {
60 static const uint32_t map[] = {
61 0,
62 RATiedReg::kRead | RATiedReg::kUse, // kRead
63 RATiedReg::kWrite | RATiedReg::kOut, // kWrite
64 RATiedReg::kRW | RATiedReg::kUse, // kRW
65 0,
66 RATiedReg::kRead | RATiedReg::kUse | RATiedReg::kUseRM, // kRead | kRegMem
67 RATiedReg::kWrite | RATiedReg::kOut | RATiedReg::kOutRM, // kWrite | kRegMem
68 RATiedReg::kRW | RATiedReg::kUse | RATiedReg::kUseRM // kRW | kRegMem
69 };
70
71 return map[rwFlags & (OpRWInfo::kRW | OpRWInfo::kRegMem)];
72 }
73
raRegRwFlags(uint32_t flags)74 static ASMJIT_INLINE uint32_t raRegRwFlags(uint32_t flags) noexcept {
75 return raUseOutFlagsFromRWFlags(flags);
76 }
77
raMemBaseRwFlags(uint32_t flags)78 static ASMJIT_INLINE uint32_t raMemBaseRwFlags(uint32_t flags) noexcept {
79 constexpr uint32_t shift = Support::constCtz(OpRWInfo::kMemBaseRW);
80 return raUseOutFlagsFromRWFlags((flags >> shift) & OpRWInfo::kRW);
81 }
82
raMemIndexRwFlags(uint32_t flags)83 static ASMJIT_INLINE uint32_t raMemIndexRwFlags(uint32_t flags) noexcept {
84 constexpr uint32_t shift = Support::constCtz(OpRWInfo::kMemIndexRW);
85 return raUseOutFlagsFromRWFlags((flags >> shift) & OpRWInfo::kRW);
86 }
87
88 // ============================================================================
89 // [asmjit::x86::X86RACFGBuilder]
90 // ============================================================================
91
92 class X86RACFGBuilder : public RACFGBuilder<X86RACFGBuilder> {
93 public:
94 uint32_t _arch;
95 bool _is64Bit;
96 bool _avxEnabled;
97
X86RACFGBuilder(X86RAPass * pass)98 inline X86RACFGBuilder(X86RAPass* pass) noexcept
99 : RACFGBuilder<X86RACFGBuilder>(pass),
100 _arch(pass->cc()->arch()),
101 _is64Bit(pass->registerSize() == 8),
102 _avxEnabled(pass->_avxEnabled) {
103 }
104
cc() const105 inline Compiler* cc() const noexcept { return static_cast<Compiler*>(_cc); }
106
choose(uint32_t sseInst,uint32_t avxInst) const107 inline uint32_t choose(uint32_t sseInst, uint32_t avxInst) const noexcept {
108 return _avxEnabled ? avxInst : sseInst;
109 }
110
111 Error onInst(InstNode* inst, uint32_t& controlType, RAInstBuilder& ib) noexcept;
112
113 Error onBeforeInvoke(InvokeNode* invokeNode) noexcept;
114 Error onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept;
115
116 Error moveVecToPtr(InvokeNode* invokeNode, const FuncValue& arg, const Vec& src, BaseReg* out) noexcept;
117 Error moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept;
118 Error moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept;
119 Error moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept;
120
121 Error onBeforeRet(FuncRetNode* funcRet) noexcept;
122 Error onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept;
123 };
124
125 // ============================================================================
126 // [asmjit::x86::X86RACFGBuilder - OnInst]
127 // ============================================================================
128
onInst(InstNode * inst,uint32_t & controlType,RAInstBuilder & ib)129 Error X86RACFGBuilder::onInst(InstNode* inst, uint32_t& controlType, RAInstBuilder& ib) noexcept {
130 InstRWInfo rwInfo;
131
132 uint32_t instId = inst->id();
133 if (Inst::isDefinedId(instId)) {
134 uint32_t opCount = inst->opCount();
135 const Operand* opArray = inst->operands();
136 ASMJIT_PROPAGATE(InstInternal::queryRWInfo(_arch, inst->baseInst(), opArray, opCount, &rwInfo));
137
138 const InstDB::InstInfo& instInfo = InstDB::infoById(instId);
139 bool hasGpbHiConstraint = false;
140 uint32_t singleRegOps = 0;
141
142 if (opCount) {
143 for (uint32_t i = 0; i < opCount; i++) {
144 const Operand& op = opArray[i];
145 const OpRWInfo& opRwInfo = rwInfo.operand(i);
146
147 if (op.isReg()) {
148 // Register Operand
149 // ----------------
150 const Reg& reg = op.as<Reg>();
151
152 uint32_t flags = raRegRwFlags(opRwInfo.opFlags());
153 uint32_t allowedRegs = 0xFFFFFFFFu;
154
155 // X86-specific constraints related to LO|HI general purpose registers.
156 // This is only required when the register is part of the encoding. If
157 // the register is fixed we won't restrict anything as it doesn't restrict
158 // encoding of other registers.
159 if (reg.isGpb() && !(opRwInfo.opFlags() & OpRWInfo::kRegPhysId)) {
160 flags |= RATiedReg::kX86Gpb;
161 if (!_is64Bit) {
162 // Restrict to first four - AL|AH|BL|BH|CL|CH|DL|DH. In 32-bit mode
163 // it's not possible to access SIL|DIL, etc, so this is just enough.
164 allowedRegs = 0x0Fu;
165 }
166 else {
167 // If we encountered GPB-HI register the situation is much more
168 // complicated than in 32-bit mode. We need to patch all registers
169 // to not use ID higher than 7 and all GPB-LO registers to not use
170 // index higher than 3. Instead of doing the patching here we just
171 // set a flag and will do it later, to not complicate this loop.
172 if (reg.isGpbHi()) {
173 hasGpbHiConstraint = true;
174 allowedRegs = 0x0Fu;
175 }
176 }
177 }
178
179 uint32_t vIndex = Operand::virtIdToIndex(reg.id());
180 if (vIndex < Operand::kVirtIdCount) {
181 RAWorkReg* workReg;
182 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
183
184 // Use RW instead of Write in case that not the whole register is
185 // overwritten. This is important for liveness as we cannot kill a
186 // register that will be used. For example `mov al, 0xFF` is not a
187 // write-only operation if user allocated the whole `rax` register.
188 if ((flags & RATiedReg::kRW) == RATiedReg::kWrite) {
189 if (workReg->regByteMask() & ~(opRwInfo.writeByteMask() | opRwInfo.extendByteMask())) {
190 // Not write-only operation.
191 flags = (flags & ~RATiedReg::kOut) | (RATiedReg::kRead | RATiedReg::kUse);
192 }
193 }
194
195 // Do not use RegMem flag if changing Reg to Mem requires additional
196 // CPU feature that may not be enabled.
197 if (rwInfo.rmFeature() && (flags & (RATiedReg::kUseRM | RATiedReg::kOutRM))) {
198 flags &= ~(RATiedReg::kUseRM | RATiedReg::kOutRM);
199 }
200
201 uint32_t group = workReg->group();
202 uint32_t allocable = _pass->_availableRegs[group] & allowedRegs;
203
204 uint32_t useId = BaseReg::kIdBad;
205 uint32_t outId = BaseReg::kIdBad;
206
207 uint32_t useRewriteMask = 0;
208 uint32_t outRewriteMask = 0;
209
210 if (flags & RATiedReg::kUse) {
211 useRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId));
212 if (opRwInfo.opFlags() & OpRWInfo::kRegPhysId) {
213 useId = opRwInfo.physId();
214 flags |= RATiedReg::kUseFixed;
215 }
216 }
217 else {
218 outRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId));
219 if (opRwInfo.opFlags() & OpRWInfo::kRegPhysId) {
220 outId = opRwInfo.physId();
221 flags |= RATiedReg::kOutFixed;
222 }
223 }
224
225 ASMJIT_PROPAGATE(ib.add(workReg, flags, allocable, useId, useRewriteMask, outId, outRewriteMask, opRwInfo.rmSize()));
226 if (singleRegOps == i)
227 singleRegOps++;
228 }
229 }
230 else if (op.isMem()) {
231 // Memory Operand
232 // --------------
233 const Mem& mem = op.as<Mem>();
234 ib.addForbiddenFlags(RATiedReg::kUseRM | RATiedReg::kOutRM);
235
236 if (mem.isRegHome()) {
237 RAWorkReg* workReg;
238 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(mem.baseId()), &workReg));
239 _pass->getOrCreateStackSlot(workReg);
240 }
241 else if (mem.hasBaseReg()) {
242 uint32_t vIndex = Operand::virtIdToIndex(mem.baseId());
243 if (vIndex < Operand::kVirtIdCount) {
244 RAWorkReg* workReg;
245 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
246
247 uint32_t flags = raMemBaseRwFlags(opRwInfo.opFlags());
248 uint32_t group = workReg->group();
249 uint32_t allocable = _pass->_availableRegs[group];
250
251 uint32_t useId = BaseReg::kIdBad;
252 uint32_t outId = BaseReg::kIdBad;
253
254 uint32_t useRewriteMask = 0;
255 uint32_t outRewriteMask = 0;
256
257 if (flags & RATiedReg::kUse) {
258 useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId));
259 if (opRwInfo.opFlags() & OpRWInfo::kMemPhysId) {
260 useId = opRwInfo.physId();
261 flags |= RATiedReg::kUseFixed;
262 }
263 }
264 else {
265 outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId));
266 if (opRwInfo.opFlags() & OpRWInfo::kMemPhysId) {
267 outId = opRwInfo.physId();
268 flags |= RATiedReg::kOutFixed;
269 }
270 }
271
272 ASMJIT_PROPAGATE(ib.add(workReg, flags, allocable, useId, useRewriteMask, outId, outRewriteMask));
273 }
274 }
275
276 if (mem.hasIndexReg()) {
277 uint32_t vIndex = Operand::virtIdToIndex(mem.indexId());
278 if (vIndex < Operand::kVirtIdCount) {
279 RAWorkReg* workReg;
280 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
281
282 uint32_t flags = raMemIndexRwFlags(opRwInfo.opFlags());
283 uint32_t group = workReg->group();
284 uint32_t allocable = _pass->_availableRegs[group];
285
286 // Index registers have never fixed id on X86/x64.
287 const uint32_t useId = BaseReg::kIdBad;
288 const uint32_t outId = BaseReg::kIdBad;
289
290 uint32_t useRewriteMask = 0;
291 uint32_t outRewriteMask = 0;
292
293 if (flags & RATiedReg::kUse)
294 useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId]));
295 else
296 outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId]));
297
298 ASMJIT_PROPAGATE(ib.add(workReg, RATiedReg::kUse | RATiedReg::kRead, allocable, useId, useRewriteMask, outId, outRewriteMask));
299 }
300 }
301 }
302 }
303 }
304
305 // Handle extra operand (either REP {cx|ecx|rcx} or AVX-512 {k} selector).
306 if (inst->hasExtraReg()) {
307 uint32_t vIndex = Operand::virtIdToIndex(inst->extraReg().id());
308 if (vIndex < Operand::kVirtIdCount) {
309 RAWorkReg* workReg;
310 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
311
312 uint32_t group = workReg->group();
313 uint32_t rewriteMask = Support::bitMask(inst->getRewriteIndex(&inst->extraReg()._id));
314
315 if (group == Gp::kGroupKReg) {
316 // AVX-512 mask selector {k} register - read-only, allocable to any register except {k0}.
317 uint32_t allocableRegs= _pass->_availableRegs[group] & ~Support::bitMask(0);
318 ASMJIT_PROPAGATE(ib.add(workReg, RATiedReg::kUse | RATiedReg::kRead, allocableRegs, BaseReg::kIdBad, rewriteMask, BaseReg::kIdBad, 0));
319 singleRegOps = 0;
320 }
321 else {
322 // REP {cx|ecx|rcx} register - read & write, allocable to {cx|ecx|rcx} only.
323 ASMJIT_PROPAGATE(ib.add(workReg, RATiedReg::kUse | RATiedReg::kRW, 0, Gp::kIdCx, rewriteMask, Gp::kIdBad, 0));
324 }
325 }
326 else {
327 uint32_t group = inst->extraReg().group();
328 if (group == Gp::kGroupKReg && inst->extraReg().id() != 0)
329 singleRegOps = 0;
330 }
331 }
332
333 // Handle X86 constraints.
334 if (hasGpbHiConstraint) {
335 for (RATiedReg& tiedReg : ib) {
336 tiedReg._allocableRegs &= tiedReg.hasFlag(RATiedReg::kX86Gpb) ? 0x0Fu : 0xFFu;
337 }
338 }
339
340 if (ib.tiedRegCount() == 1) {
341 // Handle special cases of some instructions where all operands share the same
342 // register. In such case the single operand becomes read-only or write-only.
343 uint32_t singleRegCase = InstDB::kSingleRegNone;
344 if (singleRegOps == opCount) {
345 singleRegCase = instInfo.singleRegCase();
346 }
347 else if (opCount == 2 && inst->op(1).isImm()) {
348 // Handle some tricks used by X86 asm.
349 const BaseReg& reg = inst->op(0).as<BaseReg>();
350 const Imm& imm = inst->op(1).as<Imm>();
351
352 const RAWorkReg* workReg = _pass->workRegById(ib[0]->workId());
353 uint32_t workRegSize = workReg->info().size();
354
355 switch (inst->id()) {
356 case Inst::kIdOr: {
357 // Sets the value of the destination register to -1, previous content unused.
358 if (reg.size() >= 4 || reg.size() >= workRegSize) {
359 if (imm.value() == -1 || imm.valueAs<uint64_t>() == raImmMaskFromSize(reg.size()))
360 singleRegCase = InstDB::kSingleRegWO;
361 }
362 ASMJIT_FALLTHROUGH;
363 }
364
365 case Inst::kIdAdd:
366 case Inst::kIdAnd:
367 case Inst::kIdRol:
368 case Inst::kIdRor:
369 case Inst::kIdSar:
370 case Inst::kIdShl:
371 case Inst::kIdShr:
372 case Inst::kIdSub:
373 case Inst::kIdXor: {
374 // Updates [E|R]FLAGS without changing the content.
375 if (reg.size() != 4 || reg.size() >= workRegSize) {
376 if (imm.value() == 0)
377 singleRegCase = InstDB::kSingleRegRO;
378 }
379 break;
380 }
381 }
382 }
383
384 switch (singleRegCase) {
385 case InstDB::kSingleRegNone:
386 break;
387 case InstDB::kSingleRegRO:
388 ib[0]->makeReadOnly();
389 break;
390 case InstDB::kSingleRegWO:
391 ib[0]->makeWriteOnly();
392 break;
393 }
394 }
395
396 controlType = instInfo.controlType();
397 }
398
399 return kErrorOk;
400 }
401
402 // ============================================================================
403 // [asmjit::x86::X86RACFGBuilder - OnCall]
404 // ============================================================================
405
onBeforeInvoke(InvokeNode * invokeNode)406 Error X86RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept {
407 uint32_t argCount = invokeNode->argCount();
408 uint32_t retCount = invokeNode->retCount();
409 const FuncDetail& fd = invokeNode->detail();
410
411 cc()->_setCursor(invokeNode->prev());
412
413 uint32_t nativeRegType = cc()->_gpRegInfo.type();
414
415 for (uint32_t loIndex = 0; loIndex < argCount; loIndex++) {
416 for (uint32_t hiIndex = 0; hiIndex <= kFuncArgHi; hiIndex += kFuncArgHi) {
417 uint32_t argIndex = loIndex + hiIndex;
418 if (!fd.hasArg(argIndex))
419 continue;
420
421 const FuncValue& arg = fd.arg(argIndex);
422 const Operand& op = invokeNode->arg(argIndex);
423
424 if (op.isNone())
425 continue;
426
427 if (op.isReg()) {
428 const Reg& reg = op.as<Reg>();
429 RAWorkReg* workReg;
430 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
431
432 if (arg.isReg()) {
433 uint32_t regGroup = workReg->group();
434 uint32_t argGroup = Reg::groupOf(arg.regType());
435
436 if (arg.isIndirect()) {
437 if (reg.isGp()) {
438 if (reg.type() != nativeRegType)
439 return DebugUtils::errored(kErrorInvalidAssignment);
440 // It's considered allocated if this is an indirect argument and the user used GP.
441 continue;
442 }
443
444 BaseReg indirectReg;
445 moveVecToPtr(invokeNode, arg, reg.as<Vec>(), &indirectReg);
446 invokeNode->_args[argIndex] = indirectReg;
447 }
448 else {
449 if (regGroup != argGroup) {
450 // TODO: Conversion is not supported.
451 return DebugUtils::errored(kErrorInvalidAssignment);
452 }
453 }
454 }
455 else {
456 if (arg.isIndirect()) {
457 if (reg.isGp()) {
458 if (reg.type() != nativeRegType)
459 return DebugUtils::errored(kErrorInvalidAssignment);
460
461 ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, reg));
462 continue;
463 }
464
465 BaseReg indirectReg;
466 moveVecToPtr(invokeNode, arg, reg.as<Vec>(), &indirectReg);
467 ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, indirectReg));
468 }
469 else {
470 ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, reg));
471 }
472 }
473 }
474 else if (op.isImm()) {
475 if (arg.isReg()) {
476 BaseReg reg;
477 ASMJIT_PROPAGATE(moveImmToRegArg(invokeNode, arg, op.as<Imm>(), ®));
478 invokeNode->_args[argIndex] = reg;
479 }
480 else {
481 ASMJIT_PROPAGATE(moveImmToStackArg(invokeNode, arg, op.as<Imm>()));
482 }
483 }
484 }
485 }
486
487 cc()->_setCursor(invokeNode);
488 if (fd.hasFlag(CallConv::kFlagCalleePopsStack))
489 ASMJIT_PROPAGATE(cc()->sub(cc()->zsp(), fd.argStackSize()));
490
491 for (uint32_t retIndex = 0; retIndex < retCount; retIndex++) {
492 const FuncValue& ret = fd.ret(retIndex);
493 const Operand& op = invokeNode->ret(retIndex);
494
495 if (op.isReg()) {
496 const Reg& reg = op.as<Reg>();
497 RAWorkReg* workReg;
498 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
499
500 if (ret.isReg()) {
501 if (ret.regType() == Reg::kTypeSt) {
502 if (workReg->group() != Reg::kGroupVec)
503 return DebugUtils::errored(kErrorInvalidAssignment);
504
505 Reg dst = Reg(workReg->signature(), workReg->virtId());
506 Mem mem;
507
508 uint32_t typeId = Type::baseOf(workReg->typeId());
509 if (ret.hasTypeId())
510 typeId = ret.typeId();
511
512 switch (typeId) {
513 case Type::kIdF32:
514 ASMJIT_PROPAGATE(_pass->useTemporaryMem(mem, 4, 4));
515 mem.setSize(4);
516 ASMJIT_PROPAGATE(cc()->fstp(mem));
517 ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovss, Inst::kIdVmovss), dst.as<Xmm>(), mem));
518 break;
519
520 case Type::kIdF64:
521 ASMJIT_PROPAGATE(_pass->useTemporaryMem(mem, 8, 4));
522 mem.setSize(8);
523 ASMJIT_PROPAGATE(cc()->fstp(mem));
524 ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovsd, Inst::kIdVmovsd), dst.as<Xmm>(), mem));
525 break;
526
527 default:
528 return DebugUtils::errored(kErrorInvalidAssignment);
529 }
530 }
531 else {
532 uint32_t regGroup = workReg->group();
533 uint32_t retGroup = Reg::groupOf(ret.regType());
534
535 if (regGroup != retGroup) {
536 // TODO: Conversion is not supported.
537 return DebugUtils::errored(kErrorInvalidAssignment);
538 }
539 }
540 }
541 }
542 }
543
544 // This block has function invokeNode(s).
545 _curBlock->addFlags(RABlock::kFlagHasFuncCalls);
546 _pass->func()->frame().addAttributes(FuncFrame::kAttrHasFuncCalls);
547 _pass->func()->frame().updateCallStackSize(fd.argStackSize());
548
549 return kErrorOk;
550 }
551
onInvoke(InvokeNode * invokeNode,RAInstBuilder & ib)552 Error X86RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept {
553 uint32_t argCount = invokeNode->argCount();
554 uint32_t retCount = invokeNode->retCount();
555 const FuncDetail& fd = invokeNode->detail();
556
557 for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
558 for (uint32_t argHi = 0; argHi <= kFuncArgHi; argHi += kFuncArgHi) {
559 if (!fd.hasArg(argIndex + argHi))
560 continue;
561
562 const FuncValue& arg = fd.arg(argIndex + argHi);
563 const Operand& op = invokeNode->arg(argIndex + argHi);
564
565 if (op.isNone())
566 continue;
567
568 if (op.isReg()) {
569 const Reg& reg = op.as<Reg>();
570 RAWorkReg* workReg;
571 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
572
573 if (arg.isIndirect()) {
574 uint32_t regGroup = workReg->group();
575 if (regGroup != BaseReg::kGroupGp)
576 return DebugUtils::errored(kErrorInvalidState);
577 ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId()));
578 }
579 else if (arg.isReg()) {
580 uint32_t regGroup = workReg->group();
581 uint32_t argGroup = Reg::groupOf(arg.regType());
582
583 if (regGroup == argGroup) {
584 ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId()));
585 }
586 }
587 }
588 }
589 }
590
591 for (uint32_t retIndex = 0; retIndex < retCount; retIndex++) {
592 const FuncValue& ret = fd.ret(retIndex);
593 const Operand& op = invokeNode->ret(retIndex);
594
595 // Not handled here...
596 if (ret.regType() == Reg::kTypeSt)
597 continue;
598
599 if (op.isReg()) {
600 const Reg& reg = op.as<Reg>();
601 RAWorkReg* workReg;
602 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(reg.id()), &workReg));
603
604 if (ret.isReg()) {
605 uint32_t regGroup = workReg->group();
606 uint32_t retGroup = Reg::groupOf(ret.regType());
607
608 if (regGroup == retGroup) {
609 ASMJIT_PROPAGATE(ib.addCallRet(workReg, ret.regId()));
610 }
611 }
612 else {
613 return DebugUtils::errored(kErrorInvalidAssignment);
614 }
615 }
616 }
617
618 // Setup clobbered registers.
619 ib._clobbered[0] = Support::lsbMask<uint32_t>(_pass->_physRegCount[0]) & ~fd.preservedRegs(0);
620 ib._clobbered[1] = Support::lsbMask<uint32_t>(_pass->_physRegCount[1]) & ~fd.preservedRegs(1);
621 ib._clobbered[2] = Support::lsbMask<uint32_t>(_pass->_physRegCount[2]) & ~fd.preservedRegs(2);
622 ib._clobbered[3] = Support::lsbMask<uint32_t>(_pass->_physRegCount[3]) & ~fd.preservedRegs(3);
623
624 return kErrorOk;
625 }
626
627 // ============================================================================
628 // [asmjit::x86::X86RACFGBuilder - MoveVecToPtr]
629 // ============================================================================
630
x86VecRegSignatureBySize(uint32_t size)631 static uint32_t x86VecRegSignatureBySize(uint32_t size) noexcept {
632 if (size >= 64)
633 return Zmm::kSignature;
634 else if (size >= 32)
635 return Ymm::kSignature;
636 else
637 return Xmm::kSignature;
638 }
639
moveVecToPtr(InvokeNode * invokeNode,const FuncValue & arg,const Vec & src,BaseReg * out)640 Error X86RACFGBuilder::moveVecToPtr(InvokeNode* invokeNode, const FuncValue& arg, const Vec& src, BaseReg* out) noexcept {
641 DebugUtils::unused(invokeNode);
642 ASMJIT_ASSERT(arg.isReg());
643
644 uint32_t argSize = Type::sizeOf(arg.typeId());
645 if (argSize == 0)
646 return DebugUtils::errored(kErrorInvalidState);
647
648 if (argSize < 16)
649 argSize = 16;
650
651 uint32_t argStackOffset = Support::alignUp(invokeNode->detail()._argStackSize, argSize);
652 _funcNode->frame().updateCallStackAlignment(argSize);
653 invokeNode->detail()._argStackSize = argStackOffset + argSize;
654
655 Vec vecReg(x86VecRegSignatureBySize(argSize), src.id());
656 Mem vecPtr = ptr(_pass->_sp.as<Gp>(), int32_t(argStackOffset));
657
658 uint32_t vMovInstId = choose(Inst::kIdMovaps, Inst::kIdVmovaps);
659 if (argSize > 16)
660 vMovInstId = Inst::kIdVmovaps;
661
662 ASMJIT_PROPAGATE(cc()->_newReg(out, cc()->_gpRegInfo.type(), nullptr));
663
664 VirtReg* vReg = cc()->virtRegById(out->id());
665 vReg->setWeight(RAPass::kCallArgWeight);
666
667 ASMJIT_PROPAGATE(cc()->lea(out->as<Gp>(), vecPtr));
668 ASMJIT_PROPAGATE(cc()->emit(vMovInstId, ptr(out->as<Gp>()), vecReg));
669
670 if (arg.isStack()) {
671 Mem stackPtr = ptr(_pass->_sp.as<Gp>(), arg.stackOffset());
672 ASMJIT_PROPAGATE(cc()->mov(stackPtr, out->as<Gp>()));
673 }
674
675 return kErrorOk;
676 }
677
678 // ============================================================================
679 // [asmjit::x86::X86RACFGBuilder - MoveImmToRegArg]
680 // ============================================================================
681
moveImmToRegArg(InvokeNode * invokeNode,const FuncValue & arg,const Imm & imm_,BaseReg * out)682 Error X86RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept {
683 DebugUtils::unused(invokeNode);
684 ASMJIT_ASSERT(arg.isReg());
685
686 Imm imm(imm_);
687 uint32_t rTypeId = Type::kIdU32;
688
689 switch (arg.typeId()) {
690 case Type::kIdI8: imm.signExtend8Bits(); goto MovU32;
691 case Type::kIdU8: imm.zeroExtend8Bits(); goto MovU32;
692 case Type::kIdI16: imm.signExtend16Bits(); goto MovU32;
693 case Type::kIdU16: imm.zeroExtend16Bits(); goto MovU32;
694
695 case Type::kIdI32:
696 case Type::kIdU32:
697 MovU32:
698 imm.zeroExtend32Bits();
699 break;
700
701 case Type::kIdI64:
702 case Type::kIdU64:
703 // Moving to GPD automatically zero extends in 64-bit mode.
704 if (imm.isUInt32()) {
705 imm.zeroExtend32Bits();
706 break;
707 }
708
709 rTypeId = Type::kIdU64;
710 break;
711
712 default:
713 return DebugUtils::errored(kErrorInvalidAssignment);
714 }
715
716 ASMJIT_PROPAGATE(cc()->_newReg(out, rTypeId, nullptr));
717 cc()->virtRegById(out->id())->setWeight(RAPass::kCallArgWeight);
718
719 return cc()->mov(out->as<x86::Gp>(), imm);
720 }
721
722 // ============================================================================
723 // [asmjit::x86::X86RACFGBuilder - MoveImmToStackArg]
724 // ============================================================================
725
moveImmToStackArg(InvokeNode * invokeNode,const FuncValue & arg,const Imm & imm_)726 Error X86RACFGBuilder::moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept {
727 DebugUtils::unused(invokeNode);
728 ASMJIT_ASSERT(arg.isStack());
729
730 Mem stackPtr = ptr(_pass->_sp.as<Gp>(), arg.stackOffset());
731 Imm imm[2];
732
733 stackPtr.setSize(4);
734 imm[0] = imm_;
735 uint32_t nMovs = 0;
736
737 // One stack entry has the same size as the native register size. That means
738 // that if we want to move a 32-bit integer on the stack in 64-bit mode, we
739 // need to extend it to a 64-bit integer first. In 32-bit mode, pushing a
740 // 64-bit on stack is done in two steps by pushing low and high parts
741 // separately.
742 switch (arg.typeId()) {
743 case Type::kIdI8: imm[0].signExtend8Bits(); goto MovU32;
744 case Type::kIdU8: imm[0].zeroExtend8Bits(); goto MovU32;
745 case Type::kIdI16: imm[0].signExtend16Bits(); goto MovU32;
746 case Type::kIdU16: imm[0].zeroExtend16Bits(); goto MovU32;
747
748 case Type::kIdI32:
749 case Type::kIdU32:
750 case Type::kIdF32:
751 MovU32:
752 imm[0].zeroExtend32Bits();
753 nMovs = 1;
754 break;
755
756 case Type::kIdI64:
757 case Type::kIdU64:
758 case Type::kIdF64:
759 case Type::kIdMmx32:
760 case Type::kIdMmx64:
761 if (_is64Bit && imm[0].isInt32()) {
762 stackPtr.setSize(8);
763 nMovs = 1;
764 break;
765 }
766
767 imm[1].setValue(imm[0].uint32Hi());
768 imm[0].zeroExtend32Bits();
769 nMovs = 2;
770 break;
771
772 default:
773 return DebugUtils::errored(kErrorInvalidAssignment);
774 }
775
776 for (uint32_t i = 0; i < nMovs; i++) {
777 ASMJIT_PROPAGATE(cc()->mov(stackPtr, imm[i]));
778 stackPtr.addOffsetLo32(int32_t(stackPtr.size()));
779 }
780
781 return kErrorOk;
782 }
783
784 // ============================================================================
785 // [asmjit::x86::X86RACFGBuilder - MoveRegToStackArg]
786 // ============================================================================
787
moveRegToStackArg(InvokeNode * invokeNode,const FuncValue & arg,const BaseReg & reg)788 Error X86RACFGBuilder::moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept {
789 DebugUtils::unused(invokeNode);
790 ASMJIT_ASSERT(arg.isStack());
791
792 Mem stackPtr = ptr(_pass->_sp.as<Gp>(), arg.stackOffset());
793 Reg r0, r1;
794
795 VirtReg* vr = cc()->virtRegById(reg.id());
796 uint32_t registerSize = cc()->registerSize();
797 uint32_t instId = 0;
798
799 uint32_t dstTypeId = arg.typeId();
800 uint32_t srcTypeId = vr->typeId();
801
802 switch (dstTypeId) {
803 case Type::kIdI64:
804 case Type::kIdU64:
805 // Extend BYTE->QWORD (GP).
806 if (Type::isGp8(srcTypeId)) {
807 r1.setRegT<Reg::kTypeGpbLo>(reg.id());
808
809 instId = (dstTypeId == Type::kIdI64 && srcTypeId == Type::kIdI8) ? Inst::kIdMovsx : Inst::kIdMovzx;
810 goto ExtendMovGpXQ;
811 }
812
813 // Extend WORD->QWORD (GP).
814 if (Type::isGp16(srcTypeId)) {
815 r1.setRegT<Reg::kTypeGpw>(reg.id());
816
817 instId = (dstTypeId == Type::kIdI64 && srcTypeId == Type::kIdI16) ? Inst::kIdMovsx : Inst::kIdMovzx;
818 goto ExtendMovGpXQ;
819 }
820
821 // Extend DWORD->QWORD (GP).
822 if (Type::isGp32(srcTypeId)) {
823 r1.setRegT<Reg::kTypeGpd>(reg.id());
824
825 instId = Inst::kIdMovsxd;
826 if (dstTypeId == Type::kIdI64 && srcTypeId == Type::kIdI32)
827 goto ExtendMovGpXQ;
828 else
829 goto ZeroExtendGpDQ;
830 }
831
832 // Move QWORD (GP).
833 if (Type::isGp64(srcTypeId)) goto MovGpQ;
834 if (Type::isMmx(srcTypeId)) goto MovMmQ;
835 if (Type::isVec(srcTypeId)) goto MovXmmQ;
836 break;
837
838 case Type::kIdI32:
839 case Type::kIdU32:
840 case Type::kIdI16:
841 case Type::kIdU16:
842 // DWORD <- WORD (Zero|Sign Extend).
843 if (Type::isGp16(srcTypeId)) {
844 bool isDstSigned = dstTypeId == Type::kIdI16 || dstTypeId == Type::kIdI32;
845 bool isSrcSigned = srcTypeId == Type::kIdI8 || srcTypeId == Type::kIdI16;
846
847 r1.setRegT<Reg::kTypeGpw>(reg.id());
848 instId = isDstSigned && isSrcSigned ? Inst::kIdMovsx : Inst::kIdMovzx;
849 goto ExtendMovGpD;
850 }
851
852 // DWORD <- BYTE (Zero|Sign Extend).
853 if (Type::isGp8(srcTypeId)) {
854 bool isDstSigned = dstTypeId == Type::kIdI16 || dstTypeId == Type::kIdI32;
855 bool isSrcSigned = srcTypeId == Type::kIdI8 || srcTypeId == Type::kIdI16;
856
857 r1.setRegT<Reg::kTypeGpbLo>(reg.id());
858 instId = isDstSigned && isSrcSigned ? Inst::kIdMovsx : Inst::kIdMovzx;
859 goto ExtendMovGpD;
860 }
861 ASMJIT_FALLTHROUGH;
862
863 case Type::kIdI8:
864 case Type::kIdU8:
865 if (Type::isInt(srcTypeId)) goto MovGpD;
866 if (Type::isMmx(srcTypeId)) goto MovMmD;
867 if (Type::isVec(srcTypeId)) goto MovXmmD;
868 break;
869
870 case Type::kIdMmx32:
871 case Type::kIdMmx64:
872 // Extend BYTE->QWORD (GP).
873 if (Type::isGp8(srcTypeId)) {
874 r1.setRegT<Reg::kTypeGpbLo>(reg.id());
875
876 instId = Inst::kIdMovzx;
877 goto ExtendMovGpXQ;
878 }
879
880 // Extend WORD->QWORD (GP).
881 if (Type::isGp16(srcTypeId)) {
882 r1.setRegT<Reg::kTypeGpw>(reg.id());
883
884 instId = Inst::kIdMovzx;
885 goto ExtendMovGpXQ;
886 }
887
888 if (Type::isGp32(srcTypeId)) goto ExtendMovGpDQ;
889 if (Type::isGp64(srcTypeId)) goto MovGpQ;
890 if (Type::isMmx(srcTypeId)) goto MovMmQ;
891 if (Type::isVec(srcTypeId)) goto MovXmmQ;
892 break;
893
894 case Type::kIdF32:
895 case Type::kIdF32x1:
896 if (Type::isVec(srcTypeId)) goto MovXmmD;
897 break;
898
899 case Type::kIdF64:
900 case Type::kIdF64x1:
901 if (Type::isVec(srcTypeId)) goto MovXmmQ;
902 break;
903
904 default:
905 if (Type::isVec(dstTypeId) && reg.as<Reg>().isVec()) {
906 stackPtr.setSize(Type::sizeOf(dstTypeId));
907 uint32_t vMovInstId = choose(Inst::kIdMovaps, Inst::kIdVmovaps);
908
909 if (Type::isVec128(dstTypeId))
910 r0.setRegT<Reg::kTypeXmm>(reg.id());
911 else if (Type::isVec256(dstTypeId))
912 r0.setRegT<Reg::kTypeYmm>(reg.id());
913 else if (Type::isVec512(dstTypeId))
914 r0.setRegT<Reg::kTypeZmm>(reg.id());
915 else
916 break;
917
918 return cc()->emit(vMovInstId, stackPtr, r0);
919 }
920 break;
921 }
922 return DebugUtils::errored(kErrorInvalidAssignment);
923
924 // Extend+Move Gp.
925 ExtendMovGpD:
926 stackPtr.setSize(4);
927 r0.setRegT<Reg::kTypeGpd>(reg.id());
928
929 ASMJIT_PROPAGATE(cc()->emit(instId, r0, r1));
930 ASMJIT_PROPAGATE(cc()->emit(Inst::kIdMov, stackPtr, r0));
931 return kErrorOk;
932
933 ExtendMovGpXQ:
934 if (registerSize == 8) {
935 stackPtr.setSize(8);
936 r0.setRegT<Reg::kTypeGpq>(reg.id());
937
938 ASMJIT_PROPAGATE(cc()->emit(instId, r0, r1));
939 ASMJIT_PROPAGATE(cc()->emit(Inst::kIdMov, stackPtr, r0));
940 }
941 else {
942 stackPtr.setSize(4);
943 r0.setRegT<Reg::kTypeGpd>(reg.id());
944
945 ASMJIT_PROPAGATE(cc()->emit(instId, r0, r1));
946
947 ExtendMovGpDQ:
948 ASMJIT_PROPAGATE(cc()->emit(Inst::kIdMov, stackPtr, r0));
949 stackPtr.addOffsetLo32(4);
950 ASMJIT_PROPAGATE(cc()->emit(Inst::kIdAnd, stackPtr, 0));
951 }
952 return kErrorOk;
953
954 ZeroExtendGpDQ:
955 stackPtr.setSize(4);
956 r0.setRegT<Reg::kTypeGpd>(reg.id());
957 goto ExtendMovGpDQ;
958
959 MovGpD:
960 stackPtr.setSize(4);
961 r0.setRegT<Reg::kTypeGpd>(reg.id());
962 return cc()->emit(Inst::kIdMov, stackPtr, r0);
963
964 MovGpQ:
965 stackPtr.setSize(8);
966 r0.setRegT<Reg::kTypeGpq>(reg.id());
967 return cc()->emit(Inst::kIdMov, stackPtr, r0);
968
969 MovMmD:
970 stackPtr.setSize(4);
971 r0.setRegT<Reg::kTypeMm>(reg.id());
972 return cc()->emit(choose(Inst::kIdMovd, Inst::kIdVmovd), stackPtr, r0);
973
974 MovMmQ:
975 stackPtr.setSize(8);
976 r0.setRegT<Reg::kTypeMm>(reg.id());
977 return cc()->emit(choose(Inst::kIdMovq, Inst::kIdVmovq), stackPtr, r0);
978
979 MovXmmD:
980 stackPtr.setSize(4);
981 r0.setRegT<Reg::kTypeXmm>(reg.id());
982 return cc()->emit(choose(Inst::kIdMovss, Inst::kIdVmovss), stackPtr, r0);
983
984 MovXmmQ:
985 stackPtr.setSize(8);
986 r0.setRegT<Reg::kTypeXmm>(reg.id());
987 return cc()->emit(choose(Inst::kIdMovlps, Inst::kIdVmovlps), stackPtr, r0);
988 }
989
990 // ============================================================================
991 // [asmjit::x86::X86RACFGBuilder - OnReg]
992 // ============================================================================
993
onBeforeRet(FuncRetNode * funcRet)994 Error X86RACFGBuilder::onBeforeRet(FuncRetNode* funcRet) noexcept {
995 const FuncDetail& funcDetail = _pass->func()->detail();
996 const Operand* opArray = funcRet->operands();
997 uint32_t opCount = funcRet->opCount();
998
999 cc()->_setCursor(funcRet->prev());
1000
1001 for (uint32_t i = 0; i < opCount; i++) {
1002 const Operand& op = opArray[i];
1003 const FuncValue& ret = funcDetail.ret(i);
1004
1005 if (!op.isReg())
1006 continue;
1007
1008 if (ret.regType() == Reg::kTypeSt) {
1009 const Reg& reg = op.as<Reg>();
1010 uint32_t vIndex = Operand::virtIdToIndex(reg.id());
1011
1012 if (vIndex < Operand::kVirtIdCount) {
1013 RAWorkReg* workReg;
1014 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
1015
1016 if (workReg->group() != Reg::kGroupVec)
1017 return DebugUtils::errored(kErrorInvalidAssignment);
1018
1019 Reg src = Reg(workReg->signature(), workReg->virtId());
1020 Mem mem;
1021
1022 uint32_t typeId = Type::baseOf(workReg->typeId());
1023 if (ret.hasTypeId())
1024 typeId = ret.typeId();
1025
1026 switch (typeId) {
1027 case Type::kIdF32:
1028 ASMJIT_PROPAGATE(_pass->useTemporaryMem(mem, 4, 4));
1029 mem.setSize(4);
1030 ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovss, Inst::kIdVmovss), mem, src.as<Xmm>()));
1031 ASMJIT_PROPAGATE(cc()->fld(mem));
1032 break;
1033
1034 case Type::kIdF64:
1035 ASMJIT_PROPAGATE(_pass->useTemporaryMem(mem, 8, 4));
1036 mem.setSize(8);
1037 ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovsd, Inst::kIdVmovsd), mem, src.as<Xmm>()));
1038 ASMJIT_PROPAGATE(cc()->fld(mem));
1039 break;
1040
1041 default:
1042 return DebugUtils::errored(kErrorInvalidAssignment);
1043 }
1044 }
1045 }
1046 }
1047
1048 return kErrorOk;
1049 }
1050
onRet(FuncRetNode * funcRet,RAInstBuilder & ib)1051 Error X86RACFGBuilder::onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept {
1052 const FuncDetail& funcDetail = _pass->func()->detail();
1053 const Operand* opArray = funcRet->operands();
1054 uint32_t opCount = funcRet->opCount();
1055
1056 for (uint32_t i = 0; i < opCount; i++) {
1057 const Operand& op = opArray[i];
1058 if (op.isNone()) continue;
1059
1060 const FuncValue& ret = funcDetail.ret(i);
1061 if (ASMJIT_UNLIKELY(!ret.isReg()))
1062 return DebugUtils::errored(kErrorInvalidAssignment);
1063
1064 // Not handled here...
1065 if (ret.regType() == Reg::kTypeSt)
1066 continue;
1067
1068 if (op.isReg()) {
1069 // Register return value.
1070 const Reg& reg = op.as<Reg>();
1071 uint32_t vIndex = Operand::virtIdToIndex(reg.id());
1072
1073 if (vIndex < Operand::kVirtIdCount) {
1074 RAWorkReg* workReg;
1075 ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(vIndex, &workReg));
1076
1077 uint32_t group = workReg->group();
1078 uint32_t allocable = _pass->_availableRegs[group];
1079 ASMJIT_PROPAGATE(ib.add(workReg, RATiedReg::kUse | RATiedReg::kRead, allocable, ret.regId(), 0, BaseReg::kIdBad, 0));
1080 }
1081 }
1082 else {
1083 return DebugUtils::errored(kErrorInvalidAssignment);
1084 }
1085 }
1086
1087 return kErrorOk;
1088 }
1089
1090 // ============================================================================
1091 // [asmjit::x86::X86RAPass - Construction / Destruction]
1092 // ============================================================================
1093
X86RAPass()1094 X86RAPass::X86RAPass() noexcept
1095 : RAPass(),
1096 _avxEnabled(false) {}
~X86RAPass()1097 X86RAPass::~X86RAPass() noexcept {}
1098
1099 // ============================================================================
1100 // [asmjit::x86::X86RAPass - OnInit / OnDone]
1101 // ============================================================================
1102
onInit()1103 void X86RAPass::onInit() noexcept {
1104 uint32_t arch = cc()->arch();
1105 uint32_t baseRegCount = Environment::is32Bit(arch) ? 8u : 16u;
1106
1107 _archRegsInfo = &opData.archRegs;
1108 _archTraits[Reg::kGroupGp] |= RAArchTraits::kHasSwap;
1109
1110 _physRegCount.set(Reg::kGroupGp , baseRegCount);
1111 _physRegCount.set(Reg::kGroupVec , baseRegCount);
1112 _physRegCount.set(Reg::kGroupMm , 8);
1113 _physRegCount.set(Reg::kGroupKReg, 8);
1114 _buildPhysIndex();
1115
1116 _availableRegCount = _physRegCount;
1117 _availableRegs[Reg::kGroupGp ] = Support::lsbMask<uint32_t>(_physRegCount.get(Reg::kGroupGp ));
1118 _availableRegs[Reg::kGroupVec ] = Support::lsbMask<uint32_t>(_physRegCount.get(Reg::kGroupVec ));
1119 _availableRegs[Reg::kGroupMm ] = Support::lsbMask<uint32_t>(_physRegCount.get(Reg::kGroupMm ));
1120 _availableRegs[Reg::kGroupKReg] = Support::lsbMask<uint32_t>(_physRegCount.get(Reg::kGroupKReg));
1121
1122 _scratchRegIndexes[0] = uint8_t(Gp::kIdCx);
1123 _scratchRegIndexes[1] = uint8_t(baseRegCount - 1);
1124
1125 // The architecture specific setup makes implicitly all registers available. So
1126 // make unavailable all registers that are special and cannot be used in general.
1127 bool hasFP = _func->frame().hasPreservedFP();
1128
1129 makeUnavailable(Reg::kGroupGp, Gp::kIdSp); // ESP|RSP used as a stack-pointer (SP).
1130 if (hasFP) makeUnavailable(Reg::kGroupGp, Gp::kIdBp); // EBP|RBP used as a frame-pointer (FP).
1131
1132 _sp = cc()->zsp();
1133 _fp = cc()->zbp();
1134 _avxEnabled = _func->frame().isAvxEnabled();
1135 }
1136
onDone()1137 void X86RAPass::onDone() noexcept {}
1138
1139 // ============================================================================
1140 // [asmjit::x86::X86RAPass - BuildCFG]
1141 // ============================================================================
1142
buildCFG()1143 Error X86RAPass::buildCFG() noexcept {
1144 return X86RACFGBuilder(this).run();
1145 }
1146
1147 // ============================================================================
1148 // [asmjit::x86::X86RAPass - OnEmit]
1149 // ============================================================================
1150
onEmitMove(uint32_t workId,uint32_t dstPhysId,uint32_t srcPhysId)1151 Error X86RAPass::onEmitMove(uint32_t workId, uint32_t dstPhysId, uint32_t srcPhysId) noexcept {
1152 RAWorkReg* wReg = workRegById(workId);
1153 BaseReg dst(wReg->info().signature(), dstPhysId);
1154 BaseReg src(wReg->info().signature(), srcPhysId);
1155
1156 const char* comment = nullptr;
1157
1158 #ifndef ASMJIT_NO_LOGGING
1159 if (_loggerFlags & FormatOptions::kFlagAnnotations) {
1160 _tmpString.assignFormat("<MOVE> %s", workRegById(workId)->name());
1161 comment = _tmpString.data();
1162 }
1163 #endif
1164
1165 return X86Internal::emitRegMove(cc()->as<Emitter>(), dst, src, wReg->typeId(), _avxEnabled, comment);
1166 }
1167
onEmitSwap(uint32_t aWorkId,uint32_t aPhysId,uint32_t bWorkId,uint32_t bPhysId)1168 Error X86RAPass::onEmitSwap(uint32_t aWorkId, uint32_t aPhysId, uint32_t bWorkId, uint32_t bPhysId) noexcept {
1169 RAWorkReg* waReg = workRegById(aWorkId);
1170 RAWorkReg* wbReg = workRegById(bWorkId);
1171
1172 bool is64Bit = Support::max(waReg->typeId(), wbReg->typeId()) >= Type::kIdI64;
1173 uint32_t sign = is64Bit ? uint32_t(RegTraits<Reg::kTypeGpq>::kSignature)
1174 : uint32_t(RegTraits<Reg::kTypeGpd>::kSignature);
1175
1176 #ifndef ASMJIT_NO_LOGGING
1177 if (_loggerFlags & FormatOptions::kFlagAnnotations) {
1178 _tmpString.assignFormat("<SWAP> %s, %s", waReg->name(), wbReg->name());
1179 cc()->setInlineComment(_tmpString.data());
1180 }
1181 #endif
1182
1183 return cc()->emit(Inst::kIdXchg, Reg(sign, aPhysId), Reg(sign, bPhysId));
1184 }
1185
onEmitLoad(uint32_t workId,uint32_t dstPhysId)1186 Error X86RAPass::onEmitLoad(uint32_t workId, uint32_t dstPhysId) noexcept {
1187 RAWorkReg* wReg = workRegById(workId);
1188 BaseReg dstReg(wReg->info().signature(), dstPhysId);
1189 BaseMem srcMem(workRegAsMem(wReg));
1190
1191 const char* comment = nullptr;
1192
1193 #ifndef ASMJIT_NO_LOGGING
1194 if (_loggerFlags & FormatOptions::kFlagAnnotations) {
1195 _tmpString.assignFormat("<LOAD> %s", workRegById(workId)->name());
1196 comment = _tmpString.data();
1197 }
1198 #endif
1199
1200 return X86Internal::emitRegMove(cc()->as<Emitter>(), dstReg, srcMem, wReg->typeId(), _avxEnabled, comment);
1201 }
1202
onEmitSave(uint32_t workId,uint32_t srcPhysId)1203 Error X86RAPass::onEmitSave(uint32_t workId, uint32_t srcPhysId) noexcept {
1204 RAWorkReg* wReg = workRegById(workId);
1205 BaseMem dstMem(workRegAsMem(wReg));
1206 BaseReg srcReg(wReg->info().signature(), srcPhysId);
1207
1208 const char* comment = nullptr;
1209
1210 #ifndef ASMJIT_NO_LOGGING
1211 if (_loggerFlags & FormatOptions::kFlagAnnotations) {
1212 _tmpString.assignFormat("<SAVE> %s", workRegById(workId)->name());
1213 comment = _tmpString.data();
1214 }
1215 #endif
1216
1217 return X86Internal::emitRegMove(cc()->as<Emitter>(), dstMem, srcReg, wReg->typeId(), _avxEnabled, comment);
1218 }
1219
onEmitJump(const Label & label)1220 Error X86RAPass::onEmitJump(const Label& label) noexcept {
1221 return cc()->jmp(label);
1222 }
1223
onEmitPreCall(InvokeNode * invokeNode)1224 Error X86RAPass::onEmitPreCall(InvokeNode* invokeNode) noexcept {
1225 if (invokeNode->detail().hasVarArgs() && cc()->is64Bit()) {
1226 uint32_t argCount = invokeNode->argCount();
1227 const FuncDetail& fd = invokeNode->detail();
1228
1229 switch (invokeNode->detail().callConv().id()) {
1230 case CallConv::kIdX64SystemV: {
1231 // AL register contains the number of arguments passed in XMM register(s).
1232 uint32_t n = 0;
1233 for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
1234 for (uint32_t argHi = 0; argHi <= kFuncArgHi; argHi += kFuncArgHi) {
1235 if (!fd.hasArg(argIndex + argHi))
1236 continue;
1237
1238 const FuncValue& arg = fd.arg(argIndex + argHi);
1239 if (arg.isReg() && Reg::groupOf(arg.regType()) == Reg::kGroupVec)
1240 n++;
1241 }
1242 }
1243
1244 if (!n)
1245 ASMJIT_PROPAGATE(cc()->xor_(eax, eax));
1246 else
1247 ASMJIT_PROPAGATE(cc()->mov(eax, n));
1248 break;
1249 }
1250
1251 case CallConv::kIdX64Windows: {
1252 // Each double-precision argument passed in XMM must be also passed in GP.
1253 for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) {
1254 for (uint32_t argHi = 0; argHi <= kFuncArgHi; argHi += kFuncArgHi) {
1255 if (!fd.hasArg(argIndex + argHi))
1256 continue;
1257
1258 const FuncValue& arg = fd.arg(argIndex + argHi);
1259 if (arg.isReg() && Reg::groupOf(arg.regType()) == Reg::kGroupVec) {
1260 Gp dst = gpq(fd.callConv().passedOrder(Reg::kGroupGp)[argIndex]);
1261 Xmm src = xmm(arg.regId());
1262 ASMJIT_PROPAGATE(cc()->emit(choose(Inst::kIdMovq, Inst::kIdVmovq), dst, src));
1263 }
1264 }
1265 }
1266 break;
1267 }
1268
1269 default:
1270 return DebugUtils::errored(kErrorInvalidState);
1271 }
1272 }
1273
1274 return kErrorOk;
1275 }
1276
1277 ASMJIT_END_SUB_NAMESPACE
1278
1279 #endif // ASMJIT_BUILD_X86 && !ASMJIT_NO_COMPILER
1280