1 /* This file is part of the dynarmic project.
2  * Copyright (c) 2016 MerryMage
3  * SPDX-License-Identifier: 0BSD
4  */
5 
6 #include <iterator>
7 
8 #include <tsl/robin_set.h>
9 
10 #include "backend/x64/block_of_code.h"
11 #include "backend/x64/emit_x64.h"
12 #include "backend/x64/nzcv_util.h"
13 #include "backend/x64/perf_map.h"
14 #include "common/assert.h"
15 #include "common/bit_util.h"
16 #include "common/common_types.h"
17 #include "common/scope_exit.h"
18 #include "common/variant_util.h"
19 #include "frontend/ir/basic_block.h"
20 #include "frontend/ir/microinstruction.h"
21 #include "frontend/ir/opcodes.h"
22 
23 // TODO: Have ARM flags in host flags and not have them use up GPR registers unless necessary.
24 // TODO: Actually implement that proper instruction selector you've always wanted to sweetheart.
25 
26 namespace Dynarmic::Backend::X64 {
27 
28 using namespace Xbyak::util;
29 
EmitContext(RegAlloc & reg_alloc,IR::Block & block)30 EmitContext::EmitContext(RegAlloc& reg_alloc, IR::Block& block)
31     : reg_alloc(reg_alloc), block(block) {}
32 
GetInstOffset(IR::Inst * inst) const33 size_t EmitContext::GetInstOffset(IR::Inst* inst) const {
34     return static_cast<size_t>(std::distance(block.begin(), IR::Block::iterator(inst)));
35 }
36 
EraseInstruction(IR::Inst * inst)37 void EmitContext::EraseInstruction(IR::Inst* inst) {
38     block.Instructions().erase(inst);
39     inst->ClearArgs();
40 }
41 
EmitX64(BlockOfCode & code)42 EmitX64::EmitX64(BlockOfCode& code) : code(code) {
43     exception_handler.Register(code);
44 }
45 
46 EmitX64::~EmitX64() = default;
47 
GetBasicBlock(IR::LocationDescriptor descriptor) const48 std::optional<EmitX64::BlockDescriptor> EmitX64::GetBasicBlock(IR::LocationDescriptor descriptor) const {
49     const auto iter = block_descriptors.find(descriptor);
50     if (iter == block_descriptors.end()) {
51         return std::nullopt;
52     }
53     return iter->second;
54 }
55 
EmitVoid(EmitContext &,IR::Inst *)56 void EmitX64::EmitVoid(EmitContext&, IR::Inst*) {
57 }
58 
EmitBreakpoint(EmitContext &,IR::Inst *)59 void EmitX64::EmitBreakpoint(EmitContext&, IR::Inst*) {
60     code.int3();
61 }
62 
EmitIdentity(EmitContext & ctx,IR::Inst * inst)63 void EmitX64::EmitIdentity(EmitContext& ctx, IR::Inst* inst) {
64     auto args = ctx.reg_alloc.GetArgumentInfo(inst);
65     if (!args[0].IsImmediate()) {
66         ctx.reg_alloc.DefineValue(inst, args[0]);
67     }
68 }
69 
PushRSBHelper(Xbyak::Reg64 loc_desc_reg,Xbyak::Reg64 index_reg,IR::LocationDescriptor target)70 void EmitX64::PushRSBHelper(Xbyak::Reg64 loc_desc_reg, Xbyak::Reg64 index_reg, IR::LocationDescriptor target) {
71     using namespace Xbyak::util;
72 
73     const auto iter = block_descriptors.find(target);
74     CodePtr target_code_ptr = iter != block_descriptors.end()
75                             ? iter->second.entrypoint
76                             : code.GetReturnFromRunCodeAddress();
77 
78     code.mov(index_reg.cvt32(), dword[r15 + code.GetJitStateInfo().offsetof_rsb_ptr]);
79 
80     code.mov(loc_desc_reg, target.Value());
81 
82     patch_information[target].mov_rcx.emplace_back(code.getCurr());
83     EmitPatchMovRcx(target_code_ptr);
84 
85     code.mov(qword[r15 + index_reg * 8 + code.GetJitStateInfo().offsetof_rsb_location_descriptors], loc_desc_reg);
86     code.mov(qword[r15 + index_reg * 8 + code.GetJitStateInfo().offsetof_rsb_codeptrs], rcx);
87 
88     code.add(index_reg.cvt32(), 1);
89     code.and_(index_reg.cvt32(), u32(code.GetJitStateInfo().rsb_ptr_mask));
90     code.mov(dword[r15 + code.GetJitStateInfo().offsetof_rsb_ptr], index_reg.cvt32());
91 }
92 
EmitPushRSB(EmitContext & ctx,IR::Inst * inst)93 void EmitX64::EmitPushRSB(EmitContext& ctx, IR::Inst* inst) {
94     auto args = ctx.reg_alloc.GetArgumentInfo(inst);
95     ASSERT(args[0].IsImmediate());
96     const u64 unique_hash_of_target = args[0].GetImmediateU64();
97 
98     ctx.reg_alloc.ScratchGpr(HostLoc::RCX);
99     const Xbyak::Reg64 loc_desc_reg = ctx.reg_alloc.ScratchGpr();
100     const Xbyak::Reg64 index_reg = ctx.reg_alloc.ScratchGpr();
101 
102     PushRSBHelper(loc_desc_reg, index_reg, IR::LocationDescriptor{unique_hash_of_target});
103 }
104 
EmitGetCarryFromOp(EmitContext &,IR::Inst *)105 void EmitX64::EmitGetCarryFromOp(EmitContext&, IR::Inst*) {
106     ASSERT_MSG(false, "should never happen");
107 }
108 
EmitGetOverflowFromOp(EmitContext &,IR::Inst *)109 void EmitX64::EmitGetOverflowFromOp(EmitContext&, IR::Inst*) {
110     ASSERT_MSG(false, "should never happen");
111 }
112 
EmitGetGEFromOp(EmitContext &,IR::Inst *)113 void EmitX64::EmitGetGEFromOp(EmitContext&, IR::Inst*) {
114     ASSERT_MSG(false, "should never happen");
115 }
116 
EmitGetUpperFromOp(EmitContext &,IR::Inst *)117 void EmitX64::EmitGetUpperFromOp(EmitContext&, IR::Inst*) {
118     ASSERT_MSG(false, "should never happen");
119 }
120 
EmitGetLowerFromOp(EmitContext &,IR::Inst *)121 void EmitX64::EmitGetLowerFromOp(EmitContext&, IR::Inst*) {
122     ASSERT_MSG(false, "should never happen");
123 }
124 
EmitGetNZCVFromOp(EmitContext & ctx,IR::Inst * inst)125 void EmitX64::EmitGetNZCVFromOp(EmitContext& ctx, IR::Inst* inst) {
126     auto args = ctx.reg_alloc.GetArgumentInfo(inst);
127 
128     const int bitsize = [&]{
129         switch (args[0].GetType()) {
130         case IR::Type::U8:
131             return 8;
132         case IR::Type::U16:
133             return 16;
134         case IR::Type::U32:
135             return 32;
136         case IR::Type::U64:
137             return 64;
138         default:
139             UNREACHABLE();
140         }
141     }();
142 
143     const Xbyak::Reg64 nzcv = ctx.reg_alloc.ScratchGpr(HostLoc::RAX);
144     const Xbyak::Reg value = ctx.reg_alloc.UseGpr(args[0]).changeBit(bitsize);
145     code.cmp(value, 0);
146     code.lahf();
147     code.seto(code.al);
148     ctx.reg_alloc.DefineValue(inst, nzcv);
149 }
150 
EmitNZCVFromPackedFlags(EmitContext & ctx,IR::Inst * inst)151 void EmitX64::EmitNZCVFromPackedFlags(EmitContext& ctx, IR::Inst* inst) {
152     auto args = ctx.reg_alloc.GetArgumentInfo(inst);
153 
154     if (args[0].IsImmediate()) {
155         const Xbyak::Reg32 nzcv = ctx.reg_alloc.ScratchGpr().cvt32();
156         u32 value = 0;
157         value |= Common::Bit<31>(args[0].GetImmediateU32()) ? (1 << 15) : 0;
158         value |= Common::Bit<30>(args[0].GetImmediateU32()) ? (1 << 14) : 0;
159         value |= Common::Bit<29>(args[0].GetImmediateU32()) ? (1 << 8) : 0;
160         value |= Common::Bit<28>(args[0].GetImmediateU32()) ? (1 << 0) : 0;
161         code.mov(nzcv, value);
162         ctx.reg_alloc.DefineValue(inst, nzcv);
163     } else if (code.HasFastBMI2()) {
164         const Xbyak::Reg32 nzcv = ctx.reg_alloc.UseScratchGpr(args[0]).cvt32();
165         const Xbyak::Reg32 tmp = ctx.reg_alloc.ScratchGpr().cvt32();
166 
167         code.shr(nzcv, 28);
168         code.mov(tmp, NZCV::x64_mask);
169         code.pdep(nzcv, nzcv, tmp);
170 
171         ctx.reg_alloc.DefineValue(inst, nzcv);
172     } else {
173         const Xbyak::Reg32 nzcv = ctx.reg_alloc.UseScratchGpr(args[0]).cvt32();
174 
175         code.shr(nzcv, 28);
176         code.imul(nzcv, nzcv, NZCV::to_x64_multiplier);
177         code.and_(nzcv, NZCV::x64_mask);
178 
179         ctx.reg_alloc.DefineValue(inst, nzcv);
180     }
181 }
182 
EmitAddCycles(size_t cycles)183 void EmitX64::EmitAddCycles(size_t cycles) {
184     ASSERT(cycles < std::numeric_limits<u32>::max());
185     code.sub(qword[r15 + code.GetJitStateInfo().offsetof_cycles_remaining], static_cast<u32>(cycles));
186 }
187 
EmitCond(IR::Cond cond)188 Xbyak::Label EmitX64::EmitCond(IR::Cond cond) {
189     Xbyak::Label pass;
190 
191     code.mov(eax, dword[r15 + code.GetJitStateInfo().offsetof_cpsr_nzcv]);
192 
193     // sahf restores SF, ZF, CF
194     // add al, 0x7F restores OF
195 
196     switch (cond) {
197     case IR::Cond::EQ: //z
198         code.sahf();
199         code.jz(pass);
200         break;
201     case IR::Cond::NE: //!z
202         code.sahf();
203         code.jnz(pass);
204         break;
205     case IR::Cond::CS: //c
206         code.sahf();
207         code.jc(pass);
208         break;
209     case IR::Cond::CC: //!c
210         code.sahf();
211         code.jnc(pass);
212         break;
213     case IR::Cond::MI: //n
214         code.sahf();
215         code.js(pass);
216         break;
217     case IR::Cond::PL: //!n
218         code.sahf();
219         code.jns(pass);
220         break;
221     case IR::Cond::VS: //v
222         code.add(al, 0x7F);
223         code.jo(pass);
224         break;
225     case IR::Cond::VC: //!v
226         code.add(al, 0x7F);
227         code.jno(pass);
228         break;
229     case IR::Cond::HI: //c & !z
230         code.sahf();
231         code.cmc();
232         code.ja(pass);
233         break;
234     case IR::Cond::LS: //!c | z
235         code.sahf();
236         code.cmc();
237         code.jna(pass);
238         break;
239     case IR::Cond::GE: // n == v
240         code.add(al, 0x7F);
241         code.sahf();
242         code.jge(pass);
243         break;
244     case IR::Cond::LT: // n != v
245         code.add(al, 0x7F);
246         code.sahf();
247         code.jl(pass);
248         break;
249     case IR::Cond::GT: // !z & (n == v)
250         code.add(al, 0x7F);
251         code.sahf();
252         code.jg(pass);
253         break;
254     case IR::Cond::LE: // z | (n != v)
255         code.add(al, 0x7F);
256         code.sahf();
257         code.jle(pass);
258         break;
259     default:
260         ASSERT_MSG(false, "Unknown cond {}", static_cast<size_t>(cond));
261         break;
262     }
263 
264     return pass;
265 }
266 
RegisterBlock(const IR::LocationDescriptor & descriptor,CodePtr entrypoint,size_t size)267 EmitX64::BlockDescriptor EmitX64::RegisterBlock(const IR::LocationDescriptor& descriptor, CodePtr entrypoint, size_t size) {
268     PerfMapRegister(entrypoint, code.getCurr(), LocationDescriptorToFriendlyName(descriptor));
269     Patch(descriptor, entrypoint);
270 
271     BlockDescriptor block_desc{entrypoint, size};
272     block_descriptors.emplace(descriptor.Value(), block_desc);
273     return block_desc;
274 }
275 
EmitTerminal(IR::Terminal terminal,IR::LocationDescriptor initial_location,bool is_single_step)276 void EmitX64::EmitTerminal(IR::Terminal terminal, IR::LocationDescriptor initial_location, bool is_single_step) {
277     Common::VisitVariant<void>(terminal, [this, initial_location, is_single_step](auto x) {
278         using T = std::decay_t<decltype(x)>;
279         if constexpr (!std::is_same_v<T, IR::Term::Invalid>) {
280             this->EmitTerminalImpl(x, initial_location, is_single_step);
281         } else {
282             ASSERT_MSG(false, "Invalid terminal");
283         }
284     });
285 }
286 
Patch(const IR::LocationDescriptor & target_desc,CodePtr target_code_ptr)287 void EmitX64::Patch(const IR::LocationDescriptor& target_desc, CodePtr target_code_ptr) {
288     const CodePtr save_code_ptr = code.getCurr();
289     const PatchInformation& patch_info = patch_information[target_desc];
290 
291     for (CodePtr location : patch_info.jg) {
292         code.SetCodePtr(location);
293         EmitPatchJg(target_desc, target_code_ptr);
294     }
295 
296     for (CodePtr location : patch_info.jmp) {
297         code.SetCodePtr(location);
298         EmitPatchJmp(target_desc, target_code_ptr);
299     }
300 
301     for (CodePtr location : patch_info.mov_rcx) {
302         code.SetCodePtr(location);
303         EmitPatchMovRcx(target_code_ptr);
304     }
305 
306     code.SetCodePtr(save_code_ptr);
307 }
308 
Unpatch(const IR::LocationDescriptor & target_desc)309 void EmitX64::Unpatch(const IR::LocationDescriptor& target_desc) {
310     Patch(target_desc, nullptr);
311 }
312 
ClearCache()313 void EmitX64::ClearCache() {
314     block_descriptors.clear();
315     patch_information.clear();
316 
317     PerfMapClear();
318 }
319 
InvalidateBasicBlocks(const tsl::robin_set<IR::LocationDescriptor> & locations)320 void EmitX64::InvalidateBasicBlocks(const tsl::robin_set<IR::LocationDescriptor>& locations) {
321     code.EnableWriting();
322     SCOPE_EXIT { code.DisableWriting(); };
323 
324     for (const auto &descriptor : locations) {
325         const auto it = block_descriptors.find(descriptor);
326         if (it == block_descriptors.end()) {
327             continue;
328         }
329 
330         if (patch_information.count(descriptor)) {
331             Unpatch(descriptor);
332         }
333         block_descriptors.erase(it);
334     }
335 }
336 
337 } // namespace Dynarmic::Backend::X64
338