1 #include <algorithm>
2 #include "Jitter_CodeGen_AArch64.h"
3 #include "BitManip.h"
4
5 using namespace Jitter;
6
7 CAArch64Assembler::REGISTER32 CCodeGen_AArch64::g_registers[MAX_REGISTERS] =
8 {
9 CAArch64Assembler::w20,
10 CAArch64Assembler::w21,
11 CAArch64Assembler::w22,
12 CAArch64Assembler::w23,
13 CAArch64Assembler::w24,
14 CAArch64Assembler::w25,
15 CAArch64Assembler::w26,
16 CAArch64Assembler::w27,
17 CAArch64Assembler::w28,
18 };
19
20 CAArch64Assembler::REGISTERMD CCodeGen_AArch64::g_registersMd[MAX_MDREGISTERS] =
21 {
22 CAArch64Assembler::v4, CAArch64Assembler::v5, CAArch64Assembler::v6, CAArch64Assembler::v7,
23 CAArch64Assembler::v8, CAArch64Assembler::v9, CAArch64Assembler::v10, CAArch64Assembler::v11,
24 CAArch64Assembler::v12, CAArch64Assembler::v13, CAArch64Assembler::v14, CAArch64Assembler::v15,
25 CAArch64Assembler::v16, CAArch64Assembler::v17, CAArch64Assembler::v18, CAArch64Assembler::v19,
26 CAArch64Assembler::v20, CAArch64Assembler::v21, CAArch64Assembler::v22, CAArch64Assembler::v23,
27 CAArch64Assembler::v24, CAArch64Assembler::v25, CAArch64Assembler::v26, CAArch64Assembler::v27,
28 };
29
30 CAArch64Assembler::REGISTER32 CCodeGen_AArch64::g_tempRegisters[MAX_TEMP_REGS] =
31 {
32 CAArch64Assembler::w9,
33 CAArch64Assembler::w10,
34 CAArch64Assembler::w11,
35 CAArch64Assembler::w12,
36 CAArch64Assembler::w13,
37 CAArch64Assembler::w14,
38 CAArch64Assembler::w15
39 };
40
41 CAArch64Assembler::REGISTER64 CCodeGen_AArch64::g_tempRegisters64[MAX_TEMP_REGS] =
42 {
43 CAArch64Assembler::x9,
44 CAArch64Assembler::x10,
45 CAArch64Assembler::x11,
46 CAArch64Assembler::x12,
47 CAArch64Assembler::x13,
48 CAArch64Assembler::x14,
49 CAArch64Assembler::x15
50 };
51
52 CAArch64Assembler::REGISTERMD CCodeGen_AArch64::g_tempRegistersMd[MAX_TEMP_MD_REGS] =
53 {
54 CAArch64Assembler::v0,
55 CAArch64Assembler::v1,
56 CAArch64Assembler::v2,
57 CAArch64Assembler::v3,
58 };
59
60 CAArch64Assembler::REGISTER32 CCodeGen_AArch64::g_paramRegisters[MAX_PARAM_REGS] =
61 {
62 CAArch64Assembler::w0,
63 CAArch64Assembler::w1,
64 CAArch64Assembler::w2,
65 CAArch64Assembler::w3,
66 CAArch64Assembler::w4,
67 CAArch64Assembler::w5,
68 CAArch64Assembler::w6,
69 CAArch64Assembler::w7,
70 };
71
72 CAArch64Assembler::REGISTER64 CCodeGen_AArch64::g_paramRegisters64[MAX_PARAM_REGS] =
73 {
74 CAArch64Assembler::x0,
75 CAArch64Assembler::x1,
76 CAArch64Assembler::x2,
77 CAArch64Assembler::x3,
78 CAArch64Assembler::x4,
79 CAArch64Assembler::x5,
80 CAArch64Assembler::x6,
81 CAArch64Assembler::x7,
82 };
83
84 CAArch64Assembler::REGISTER64 CCodeGen_AArch64::g_baseRegister = CAArch64Assembler::x19;
85
isMask(uint32 value)86 static bool isMask(uint32 value)
87 {
88 return value && (((value + 1) & value) == 0);
89 }
90
isShiftedMask(uint32 value)91 static bool isShiftedMask(uint32 value)
92 {
93 return value && isMask((value - 1) | value);
94 }
95
96 template <typename AddSubOp>
Emit_AddSub_VarAnyVar(const STATEMENT & statement)97 void CCodeGen_AArch64::Emit_AddSub_VarAnyVar(const STATEMENT& statement)
98 {
99 auto dst = statement.dst->GetSymbol().get();
100 auto src1 = statement.src1->GetSymbol().get();
101 auto src2 = statement.src2->GetSymbol().get();
102
103 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
104 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
105 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
106 ((m_assembler).*(AddSubOp::OpReg()))(dstReg, src1Reg, src2Reg);
107 CommitSymbolRegister(dst, dstReg);
108 }
109
110 template <typename AddSubOp>
Emit_AddSub_VarVarCst(const STATEMENT & statement)111 void CCodeGen_AArch64::Emit_AddSub_VarVarCst(const STATEMENT& statement)
112 {
113 auto dst = statement.dst->GetSymbol().get();
114 auto src1 = statement.src1->GetSymbol().get();
115 auto src2 = statement.src2->GetSymbol().get();
116
117 assert(src2->m_type == SYM_CONSTANT);
118
119 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
120 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
121
122 ADDSUB_IMM_PARAMS addSubImmParams;
123 if(TryGetAddSubImmParams(src2->m_valueLow, addSubImmParams))
124 {
125 ((m_assembler).*(AddSubOp::OpImm()))(dstReg, src1Reg, addSubImmParams.imm, addSubImmParams.shiftType);
126 }
127 else if(TryGetAddSubImmParams(-static_cast<int32>(src2->m_valueLow), addSubImmParams))
128 {
129 ((m_assembler).*(AddSubOp::OpImmRev()))(dstReg, src1Reg, addSubImmParams.imm, addSubImmParams.shiftType);
130 }
131 else
132 {
133 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
134 ((m_assembler).*(AddSubOp::OpReg()))(dstReg, src1Reg, src2Reg);
135 }
136
137 CommitSymbolRegister(dst, dstReg);
138 }
139
140 template <typename ShiftOp>
Emit_Shift_VarAnyVar(const STATEMENT & statement)141 void CCodeGen_AArch64::Emit_Shift_VarAnyVar(const STATEMENT& statement)
142 {
143 auto dst = statement.dst->GetSymbol().get();
144 auto src1 = statement.src1->GetSymbol().get();
145 auto src2 = statement.src2->GetSymbol().get();
146
147 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
148 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
149 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
150 ((m_assembler).*(ShiftOp::OpReg()))(dstReg, src1Reg, src2Reg);
151 CommitSymbolRegister(dst, dstReg);
152 }
153
154 template <typename ShiftOp>
Emit_Shift_VarVarCst(const STATEMENT & statement)155 void CCodeGen_AArch64::Emit_Shift_VarVarCst(const STATEMENT& statement)
156 {
157 auto dst = statement.dst->GetSymbol().get();
158 auto src1 = statement.src1->GetSymbol().get();
159 auto src2 = statement.src2->GetSymbol().get();
160
161 assert(src2->m_type == SYM_CONSTANT);
162
163 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
164 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
165 ((m_assembler).*(ShiftOp::OpImm()))(dstReg, src1Reg, src2->m_valueLow);
166 CommitSymbolRegister(dst, dstReg);
167 }
168
169 template <typename LogicOp>
Emit_Logic_VarAnyVar(const STATEMENT & statement)170 void CCodeGen_AArch64::Emit_Logic_VarAnyVar(const STATEMENT& statement)
171 {
172 auto dst = statement.dst->GetSymbol().get();
173 auto src1 = statement.src1->GetSymbol().get();
174 auto src2 = statement.src2->GetSymbol().get();
175
176 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
177 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
178 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
179 ((m_assembler).*(LogicOp::OpReg()))(dstReg, src1Reg, src2Reg);
180 CommitSymbolRegister(dst, dstReg);
181 }
182
183 template <typename LogicOp>
Emit_Logic_VarVarCst(const STATEMENT & statement)184 void CCodeGen_AArch64::Emit_Logic_VarVarCst(const STATEMENT& statement)
185 {
186 auto dst = statement.dst->GetSymbol().get();
187 auto src1 = statement.src1->GetSymbol().get();
188 auto src2 = statement.src2->GetSymbol().get();
189
190 assert(src2->m_type == SYM_CONSTANT);
191 assert(src2->m_valueLow != 0);
192
193 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
194 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
195
196 LOGICAL_IMM_PARAMS logicalImmParams;
197 if(TryGetLogicalImmParams(src2->m_valueLow, logicalImmParams))
198 {
199 ((m_assembler).*(LogicOp::OpImm()))(dstReg, src1Reg,
200 logicalImmParams.n, logicalImmParams.immr, logicalImmParams.imms);
201 }
202 else
203 {
204 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
205 ((m_assembler).*(LogicOp::OpReg()))(dstReg, src1Reg, src2Reg);
206 }
207 CommitSymbolRegister(dst, dstReg);
208 }
209
210 template <bool isSigned>
Emit_Mul_Tmp64AnyAny(const STATEMENT & statement)211 void CCodeGen_AArch64::Emit_Mul_Tmp64AnyAny(const STATEMENT& statement)
212 {
213 auto dst = statement.dst->GetSymbol().get();
214 auto src1 = statement.src1->GetSymbol().get();
215 auto src2 = statement.src2->GetSymbol().get();
216
217 assert(dst->m_type == SYM_TEMPORARY64);
218
219 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
220 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
221 auto dstReg = GetNextTempRegister64();
222
223 if(isSigned)
224 {
225 m_assembler.Smull(dstReg, src1Reg, src2Reg);
226 }
227 else
228 {
229 m_assembler.Umull(dstReg, src1Reg, src2Reg);
230 }
231
232 m_assembler.Str(dstReg, CAArch64Assembler::xSP, dst->m_stackLocation);
233 }
234
235 template <bool isSigned>
Emit_Div_Tmp64AnyAny(const STATEMENT & statement)236 void CCodeGen_AArch64::Emit_Div_Tmp64AnyAny(const STATEMENT& statement)
237 {
238 auto dst = statement.dst->GetSymbol().get();
239 auto src1 = statement.src1->GetSymbol().get();
240 auto src2 = statement.src2->GetSymbol().get();
241
242 assert(dst->m_type == SYM_TEMPORARY64);
243
244 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
245 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
246 auto resReg = GetNextTempRegister();
247 auto modReg = GetNextTempRegister();
248
249 if(isSigned)
250 {
251 m_assembler.Sdiv(resReg, src1Reg, src2Reg);
252 }
253 else
254 {
255 m_assembler.Udiv(resReg, src1Reg, src2Reg);
256 }
257
258 m_assembler.Msub(modReg, resReg, src2Reg, src1Reg);
259
260 m_assembler.Str(resReg, CAArch64Assembler::xSP, dst->m_stackLocation + 0);
261 m_assembler.Str(modReg, CAArch64Assembler::xSP, dst->m_stackLocation + 4);
262 }
263
264 #define LOGIC_CONST_MATCHERS(LOGICOP_CST, LOGICOP) \
265 { LOGICOP_CST, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_Logic_VarVarCst<LOGICOP> }, \
266 { LOGICOP_CST, MATCH_VARIABLE, MATCH_ANY, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_Logic_VarAnyVar<LOGICOP> },
267
268 CCodeGen_AArch64::CONSTMATCHER CCodeGen_AArch64::g_constMatchers[] =
269 {
270 { OP_NOP, MATCH_NIL, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Nop },
271
272 { OP_MOV, MATCH_REGISTER, MATCH_REGISTER, MATCH_NIL, &CCodeGen_AArch64::Emit_Mov_RegReg },
273 { OP_MOV, MATCH_REGISTER, MATCH_MEMORY, MATCH_NIL, &CCodeGen_AArch64::Emit_Mov_RegMem },
274 { OP_MOV, MATCH_REGISTER, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Mov_RegCst },
275 { OP_MOV, MATCH_MEMORY, MATCH_REGISTER, MATCH_NIL, &CCodeGen_AArch64::Emit_Mov_MemReg },
276 { OP_MOV, MATCH_MEMORY, MATCH_MEMORY, MATCH_NIL, &CCodeGen_AArch64::Emit_Mov_MemMem },
277 { OP_MOV, MATCH_MEMORY, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Mov_MemCst },
278
279 { OP_NOT, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_NIL, &CCodeGen_AArch64::Emit_Not_VarVar },
280 { OP_LZC, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_NIL, &CCodeGen_AArch64::Emit_Lzc_VarVar },
281
282 { OP_RELTOREF, MATCH_TMP_REF, MATCH_CONSTANT, MATCH_ANY, &CCodeGen_AArch64::Emit_RelToRef_TmpCst },
283
284 { OP_ADDREF, MATCH_TMP_REF, MATCH_MEM_REF, MATCH_ANY, &CCodeGen_AArch64::Emit_AddRef_TmpMemAny },
285
286 { OP_LOADFROMREF, MATCH_VARIABLE, MATCH_MEM_REF, MATCH_NIL, &CCodeGen_AArch64::Emit_LoadFromRef_VarMem },
287
288 //Cannot use MATCH_ANY here because it will match SYM_RELATIVE128
289 { OP_STOREATREF, MATCH_NIL, MATCH_MEM_REF, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_StoreAtRef_MemAny },
290 { OP_STOREATREF, MATCH_NIL, MATCH_MEM_REF, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_StoreAtRef_MemAny },
291
292 { OP_PARAM, MATCH_NIL, MATCH_CONTEXT, MATCH_NIL, &CCodeGen_AArch64::Emit_Param_Ctx },
293 { OP_PARAM, MATCH_NIL, MATCH_REGISTER, MATCH_NIL, &CCodeGen_AArch64::Emit_Param_Reg },
294 { OP_PARAM, MATCH_NIL, MATCH_MEMORY, MATCH_NIL, &CCodeGen_AArch64::Emit_Param_Mem },
295 { OP_PARAM, MATCH_NIL, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Param_Cst },
296 { OP_PARAM, MATCH_NIL, MATCH_MEMORY64, MATCH_NIL, &CCodeGen_AArch64::Emit_Param_Mem64 },
297 { OP_PARAM, MATCH_NIL, MATCH_CONSTANT64, MATCH_NIL, &CCodeGen_AArch64::Emit_Param_Cst64 },
298 { OP_PARAM, MATCH_NIL, MATCH_REGISTER128, MATCH_NIL, &CCodeGen_AArch64::Emit_Param_Reg128 },
299 { OP_PARAM, MATCH_NIL, MATCH_MEMORY128, MATCH_NIL, &CCodeGen_AArch64::Emit_Param_Mem128 },
300
301 { OP_CALL, MATCH_NIL, MATCH_CONSTANTPTR, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_Call },
302
303 { OP_RETVAL, MATCH_REGISTER, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_RetVal_Reg },
304 { OP_RETVAL, MATCH_TEMPORARY, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_RetVal_Tmp },
305 { OP_RETVAL, MATCH_MEMORY64, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_RetVal_Mem64 },
306 { OP_RETVAL, MATCH_REGISTER128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_RetVal_Reg128 },
307 { OP_RETVAL, MATCH_MEMORY128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_RetVal_Mem128 },
308
309 { OP_JMP, MATCH_NIL, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Jmp },
310
311 { OP_CONDJMP, MATCH_NIL, MATCH_ANY, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_CondJmp_AnyVar },
312 { OP_CONDJMP, MATCH_NIL, MATCH_VARIABLE, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_CondJmp_VarCst },
313
314 { OP_CMP, MATCH_VARIABLE, MATCH_ANY, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_Cmp_VarAnyVar },
315 { OP_CMP, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_Cmp_VarVarCst },
316
317 { OP_SLL, MATCH_VARIABLE, MATCH_ANY, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_Shift_VarAnyVar<SHIFTOP_LSL> },
318 { OP_SRL, MATCH_VARIABLE, MATCH_ANY, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_Shift_VarAnyVar<SHIFTOP_LSR> },
319 { OP_SRA, MATCH_VARIABLE, MATCH_ANY, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_Shift_VarAnyVar<SHIFTOP_ASR> },
320
321 { OP_SLL, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_Shift_VarVarCst<SHIFTOP_LSL> },
322 { OP_SRL, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_Shift_VarVarCst<SHIFTOP_LSR> },
323 { OP_SRA, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_Shift_VarVarCst<SHIFTOP_ASR> },
324
325 LOGIC_CONST_MATCHERS(OP_AND, LOGICOP_AND)
326 LOGIC_CONST_MATCHERS(OP_OR, LOGICOP_OR )
327 LOGIC_CONST_MATCHERS(OP_XOR, LOGICOP_XOR)
328
329 { OP_ADD, MATCH_VARIABLE, MATCH_ANY, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_AddSub_VarAnyVar<ADDSUBOP_ADD> },
330 { OP_ADD, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_AddSub_VarVarCst<ADDSUBOP_ADD> },
331 { OP_SUB, MATCH_VARIABLE, MATCH_ANY, MATCH_VARIABLE, &CCodeGen_AArch64::Emit_AddSub_VarAnyVar<ADDSUBOP_SUB> },
332 { OP_SUB, MATCH_VARIABLE, MATCH_VARIABLE, MATCH_CONSTANT, &CCodeGen_AArch64::Emit_AddSub_VarVarCst<ADDSUBOP_SUB> },
333
334 { OP_MUL, MATCH_TEMPORARY64, MATCH_ANY, MATCH_ANY, &CCodeGen_AArch64::Emit_Mul_Tmp64AnyAny<false> },
335 { OP_MULS, MATCH_TEMPORARY64, MATCH_ANY, MATCH_ANY, &CCodeGen_AArch64::Emit_Mul_Tmp64AnyAny<true> },
336
337 { OP_DIV, MATCH_TEMPORARY64, MATCH_ANY, MATCH_ANY, &CCodeGen_AArch64::Emit_Div_Tmp64AnyAny<false> },
338 { OP_DIVS, MATCH_TEMPORARY64, MATCH_ANY, MATCH_ANY, &CCodeGen_AArch64::Emit_Div_Tmp64AnyAny<true> },
339
340 { OP_LABEL, MATCH_NIL, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::MarkLabel },
341 };
342
CCodeGen_AArch64()343 CCodeGen_AArch64::CCodeGen_AArch64()
344 {
345 const auto copyMatchers =
346 [this](const CONSTMATCHER* constMatchers)
347 {
348 for(auto* constMatcher = constMatchers; constMatcher->emitter != nullptr; constMatcher++)
349 {
350 MATCHER matcher;
351 matcher.op = constMatcher->op;
352 matcher.dstType = constMatcher->dstType;
353 matcher.src1Type = constMatcher->src1Type;
354 matcher.src2Type = constMatcher->src2Type;
355 matcher.emitter = std::bind(constMatcher->emitter, this, std::placeholders::_1);
356 m_matchers.insert(MatcherMapType::value_type(matcher.op, matcher));
357 }
358 };
359
360 copyMatchers(g_constMatchers);
361 copyMatchers(g_64ConstMatchers);
362 copyMatchers(g_fpuConstMatchers);
363 copyMatchers(g_mdConstMatchers);
364 }
365
~CCodeGen_AArch64()366 CCodeGen_AArch64::~CCodeGen_AArch64()
367 {
368
369 }
370
SetGenerateRelocatableCalls(bool generateRelocatableCalls)371 void CCodeGen_AArch64::SetGenerateRelocatableCalls(bool generateRelocatableCalls)
372 {
373 m_generateRelocatableCalls = generateRelocatableCalls;
374 }
375
GetAvailableRegisterCount() const376 unsigned int CCodeGen_AArch64::GetAvailableRegisterCount() const
377 {
378 return MAX_REGISTERS;
379 }
380
GetAvailableMdRegisterCount() const381 unsigned int CCodeGen_AArch64::GetAvailableMdRegisterCount() const
382 {
383 return MAX_MDREGISTERS;
384 }
385
CanHold128BitsReturnValueInRegisters() const386 bool CCodeGen_AArch64::CanHold128BitsReturnValueInRegisters() const
387 {
388 return true;
389 }
390
SetStream(Framework::CStream * stream)391 void CCodeGen_AArch64::SetStream(Framework::CStream* stream)
392 {
393 m_stream = stream;
394 m_assembler.SetStream(stream);
395 }
396
RegisterExternalSymbols(CObjectFile * objectFile) const397 void CCodeGen_AArch64::RegisterExternalSymbols(CObjectFile* objectFile) const
398 {
399
400 }
401
GenerateCode(const StatementList & statements,unsigned int stackSize)402 void CCodeGen_AArch64::GenerateCode(const StatementList& statements, unsigned int stackSize)
403 {
404 m_nextTempRegister = 0;
405 m_nextTempRegisterMd = 0;
406
407 //Align stack size (must be aligned on 16 bytes boundary)
408 stackSize = (stackSize + 0xF) & ~0xF;
409
410 uint16 registerSave = GetSavedRegisterList(GetRegisterUsage(statements));
411
412 Emit_Prolog(statements, stackSize, registerSave);
413
414 for(const auto& statement : statements)
415 {
416 bool found = false;
417 auto begin = m_matchers.lower_bound(statement.op);
418 auto end = m_matchers.upper_bound(statement.op);
419
420 for(auto matchIterator(begin); matchIterator != end; matchIterator++)
421 {
422 const MATCHER& matcher(matchIterator->second);
423 if(!SymbolMatches(matcher.dstType, statement.dst)) continue;
424 if(!SymbolMatches(matcher.src1Type, statement.src1)) continue;
425 if(!SymbolMatches(matcher.src2Type, statement.src2)) continue;
426 matcher.emitter(statement);
427 found = true;
428 break;
429 }
430 assert(found);
431 if(!found)
432 {
433 throw std::runtime_error("No suitable emitter found for statement.");
434 }
435 }
436
437 Emit_Epilog(stackSize, registerSave);
438
439 m_assembler.ResolveLabelReferences();
440 m_assembler.ClearLabels();
441 m_labels.clear();
442 }
443
GetMaxParamSpillSize(const StatementList & statements)444 uint32 CCodeGen_AArch64::GetMaxParamSpillSize(const StatementList& statements)
445 {
446 uint32 maxParamSpillSize = 0;
447 uint32 currParamSpillSize = 0;
448 for(const auto& statement : statements)
449 {
450 switch(statement.op)
451 {
452 case OP_PARAM:
453 case OP_PARAM_RET:
454 {
455 CSymbol* src1 = statement.src1->GetSymbol().get();
456 switch(src1->m_type)
457 {
458 case SYM_REGISTER128:
459 currParamSpillSize += 16;
460 break;
461 default:
462 break;
463 }
464 }
465 break;
466 case OP_CALL:
467 maxParamSpillSize = std::max<uint32>(currParamSpillSize, maxParamSpillSize);
468 currParamSpillSize = 0;
469 break;
470 default:
471 break;
472 }
473 }
474 return maxParamSpillSize;
475 }
476
GetNextTempRegister()477 CAArch64Assembler::REGISTER32 CCodeGen_AArch64::GetNextTempRegister()
478 {
479 auto result = g_tempRegisters[m_nextTempRegister];
480 m_nextTempRegister++;
481 m_nextTempRegister %= MAX_TEMP_REGS;
482 return result;
483 }
484
GetNextTempRegister64()485 CAArch64Assembler::REGISTER64 CCodeGen_AArch64::GetNextTempRegister64()
486 {
487 auto result = g_tempRegisters64[m_nextTempRegister];
488 m_nextTempRegister++;
489 m_nextTempRegister %= MAX_TEMP_REGS;
490 return result;
491 }
492
GetNextTempRegisterMd()493 CAArch64Assembler::REGISTERMD CCodeGen_AArch64::GetNextTempRegisterMd()
494 {
495 auto result = g_tempRegistersMd[m_nextTempRegisterMd];
496 m_nextTempRegisterMd++;
497 m_nextTempRegisterMd %= MAX_TEMP_MD_REGS;
498 return result;
499 }
500
LoadMemoryInRegister(CAArch64Assembler::REGISTER32 registerId,CSymbol * src)501 void CCodeGen_AArch64::LoadMemoryInRegister(CAArch64Assembler::REGISTER32 registerId, CSymbol* src)
502 {
503 switch(src->m_type)
504 {
505 case SYM_RELATIVE:
506 assert((src->m_valueLow & 0x03) == 0x00);
507 m_assembler.Ldr(registerId, g_baseRegister, src->m_valueLow);
508 break;
509 case SYM_TEMPORARY:
510 m_assembler.Ldr(registerId, CAArch64Assembler::xSP, src->m_stackLocation);
511 break;
512 default:
513 assert(0);
514 break;
515 }
516 }
517
StoreRegisterInMemory(CSymbol * dst,CAArch64Assembler::REGISTER32 registerId)518 void CCodeGen_AArch64::StoreRegisterInMemory(CSymbol* dst, CAArch64Assembler::REGISTER32 registerId)
519 {
520 switch(dst->m_type)
521 {
522 case SYM_RELATIVE:
523 assert((dst->m_valueLow & 0x03) == 0x00);
524 m_assembler.Str(registerId, g_baseRegister, dst->m_valueLow);
525 break;
526 case SYM_TEMPORARY:
527 m_assembler.Str(registerId, CAArch64Assembler::xSP, dst->m_stackLocation);
528 break;
529 default:
530 assert(0);
531 break;
532 }
533 }
534
LoadConstantInRegister(CAArch64Assembler::REGISTER32 registerId,uint32 constant)535 void CCodeGen_AArch64::LoadConstantInRegister(CAArch64Assembler::REGISTER32 registerId, uint32 constant)
536 {
537 if((constant & 0x0000FFFF) == constant)
538 {
539 m_assembler.Movz(registerId, static_cast<uint16>(constant & 0xFFFF), 0);
540 }
541 else if((constant & 0xFFFF0000) == constant)
542 {
543 m_assembler.Movz(registerId, static_cast<uint16>(constant >> 16), 1);
544 }
545 else if((~constant & 0x0000FFFF) == ~constant)
546 {
547 m_assembler.Movn(registerId, static_cast<uint16>(~constant & 0xFFFF), 0);
548 }
549 else if((~constant & 0xFFFF0000) == ~constant)
550 {
551 m_assembler.Movn(registerId, static_cast<uint16>(~constant >> 16), 1);
552 }
553 else
554 {
555 m_assembler.Movz(registerId, static_cast<uint16>(constant & 0xFFFF), 0);
556 m_assembler.Movk(registerId, static_cast<uint16>(constant >> 16), 1);
557 }
558 }
559
LoadMemoryReferenceInRegister(CAArch64Assembler::REGISTER64 registerId,CSymbol * src)560 void CCodeGen_AArch64::LoadMemoryReferenceInRegister(CAArch64Assembler::REGISTER64 registerId, CSymbol* src)
561 {
562 switch(src->m_type)
563 {
564 case SYM_REL_REFERENCE:
565 assert((src->m_valueLow & 0x07) == 0x00);
566 m_assembler.Ldr(registerId, g_baseRegister, src->m_valueLow);
567 break;
568 case SYM_TMP_REFERENCE:
569 m_assembler.Ldr(registerId, CAArch64Assembler::xSP, src->m_stackLocation);
570 break;
571 default:
572 assert(false);
573 break;
574 }
575 }
576
StoreRegisterInTemporaryReference(CSymbol * dst,CAArch64Assembler::REGISTER64 registerId)577 void CCodeGen_AArch64::StoreRegisterInTemporaryReference(CSymbol* dst, CAArch64Assembler::REGISTER64 registerId)
578 {
579 assert(dst->m_type == SYM_TMP_REFERENCE);
580 m_assembler.Str(registerId, CAArch64Assembler::xSP, dst->m_stackLocation);
581 }
582
PrepareSymbolRegisterDef(CSymbol * symbol,CAArch64Assembler::REGISTER32 preferedRegister)583 CAArch64Assembler::REGISTER32 CCodeGen_AArch64::PrepareSymbolRegisterDef(CSymbol* symbol, CAArch64Assembler::REGISTER32 preferedRegister)
584 {
585 switch(symbol->m_type)
586 {
587 case SYM_REGISTER:
588 assert(symbol->m_valueLow < MAX_REGISTERS);
589 return g_registers[symbol->m_valueLow];
590 break;
591 case SYM_TEMPORARY:
592 case SYM_RELATIVE:
593 return preferedRegister;
594 break;
595 default:
596 throw std::runtime_error("Invalid symbol type.");
597 break;
598 }
599 }
600
PrepareSymbolRegisterUse(CSymbol * symbol,CAArch64Assembler::REGISTER32 preferedRegister)601 CAArch64Assembler::REGISTER32 CCodeGen_AArch64::PrepareSymbolRegisterUse(CSymbol* symbol, CAArch64Assembler::REGISTER32 preferedRegister)
602 {
603 switch(symbol->m_type)
604 {
605 case SYM_REGISTER:
606 assert(symbol->m_valueLow < MAX_REGISTERS);
607 return g_registers[symbol->m_valueLow];
608 break;
609 case SYM_TEMPORARY:
610 case SYM_RELATIVE:
611 LoadMemoryInRegister(preferedRegister, symbol);
612 return preferedRegister;
613 break;
614 case SYM_CONSTANT:
615 LoadConstantInRegister(preferedRegister, symbol->m_valueLow);
616 return preferedRegister;
617 break;
618 default:
619 throw std::runtime_error("Invalid symbol type.");
620 break;
621 }
622 }
623
CommitSymbolRegister(CSymbol * symbol,CAArch64Assembler::REGISTER32 usedRegister)624 void CCodeGen_AArch64::CommitSymbolRegister(CSymbol* symbol, CAArch64Assembler::REGISTER32 usedRegister)
625 {
626 switch(symbol->m_type)
627 {
628 case SYM_REGISTER:
629 assert(usedRegister == g_registers[symbol->m_valueLow]);
630 break;
631 case SYM_TEMPORARY:
632 case SYM_RELATIVE:
633 StoreRegisterInMemory(symbol, usedRegister);
634 break;
635 default:
636 throw std::runtime_error("Invalid symbol type.");
637 break;
638 }
639 }
640
PrepareParam(PARAM_STATE & paramState)641 CAArch64Assembler::REGISTER32 CCodeGen_AArch64::PrepareParam(PARAM_STATE& paramState)
642 {
643 assert(!paramState.prepared);
644 paramState.prepared = true;
645 if(paramState.index < MAX_PARAM_REGS)
646 {
647 return g_paramRegisters[paramState.index];
648 }
649 else
650 {
651 assert(false);
652 return g_paramRegisters[0];
653 }
654 }
655
PrepareParam64(PARAM_STATE & paramState)656 CAArch64Assembler::REGISTER64 CCodeGen_AArch64::PrepareParam64(PARAM_STATE& paramState)
657 {
658 assert(!paramState.prepared);
659 paramState.prepared = true;
660 if(paramState.index < MAX_PARAM_REGS)
661 {
662 return g_paramRegisters64[paramState.index];
663 }
664 else
665 {
666 assert(false);
667 return g_paramRegisters64[0];
668 }
669 }
670
CommitParam(PARAM_STATE & paramState)671 void CCodeGen_AArch64::CommitParam(PARAM_STATE& paramState)
672 {
673 assert(paramState.prepared);
674 paramState.prepared = false;
675 if(paramState.index < MAX_PARAM_REGS)
676 {
677 //Nothing to do
678 }
679 else
680 {
681 assert(false);
682 }
683 paramState.index++;
684 }
685
CommitParam64(PARAM_STATE & paramState)686 void CCodeGen_AArch64::CommitParam64(PARAM_STATE& paramState)
687 {
688 assert(paramState.prepared);
689 paramState.prepared = false;
690 if(paramState.index < MAX_PARAM_REGS)
691 {
692 //Nothing to do
693 }
694 else
695 {
696 assert(false);
697 }
698 paramState.index++;
699 }
700
TryGetAddSubImmParams(uint32 imm,ADDSUB_IMM_PARAMS & params)701 bool CCodeGen_AArch64::TryGetAddSubImmParams(uint32 imm, ADDSUB_IMM_PARAMS& params)
702 {
703 if((imm & 0xFFF) == imm)
704 {
705 params.imm = static_cast<uint16>(imm);
706 params.shiftType = CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0;
707 return true;
708 }
709 else if((imm & 0xFFF000) == imm)
710 {
711 params.imm = static_cast<uint16>(imm >> 12);
712 params.shiftType = CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL12;
713 return true;
714 }
715 return false;
716 }
717
TryGetAddSub64ImmParams(uint64 imm,ADDSUB_IMM_PARAMS & params)718 bool CCodeGen_AArch64::TryGetAddSub64ImmParams(uint64 imm, ADDSUB_IMM_PARAMS& params)
719 {
720 if((imm & 0xFFF) == imm)
721 {
722 params.imm = static_cast<uint16>(imm);
723 params.shiftType = CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0;
724 return true;
725 }
726 else if((imm & 0xFFF000) == imm)
727 {
728 params.imm = static_cast<uint16>(imm >> 12);
729 params.shiftType = CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL12;
730 return true;
731 }
732 return false;
733 }
734
TryGetLogicalImmParams(uint32 imm,LOGICAL_IMM_PARAMS & params)735 bool CCodeGen_AArch64::TryGetLogicalImmParams(uint32 imm, LOGICAL_IMM_PARAMS& params)
736 {
737 //Algorithm from LLVM, 'processLogicalImmediate' function
738
739 if((imm == 0) || (imm == ~0))
740 {
741 return false;
742 }
743
744 int size = 32;
745 while(true)
746 {
747 size /= 2;
748 uint32 mask = (1 << size) - 1;
749 if((imm & mask) != ((imm >> size) & mask))
750 {
751 size *= 2;
752 break;
753 }
754 if(size > 2) break;
755 }
756
757 uint32 cto = 0, i = 0;
758 uint32 mask = (~0) >> (32 - size);
759 imm &= mask;
760
761 if(isShiftedMask(imm))
762 {
763 i = __builtin_ctz(imm);
764 cto = __builtin_ctz(~(imm >> i));
765 }
766 else
767 {
768 imm |= ~mask;
769 if(!isShiftedMask(~imm))
770 {
771 return false;
772 }
773 uint32 clo = __builtin_clz(~imm);
774 i = 32 - clo;
775 cto = clo + __builtin_ctz(~imm) - (32 - size);
776 }
777
778 assert(size > i);
779 params.immr = (size - i) & (size - 1);
780
781 uint32 nimms = ~(size - 1) << 1;
782 nimms |= (cto - 1);
783
784 params.n = ((nimms >> 6) & 1) ^ 1;
785
786 params.imms = nimms & 0x3F;
787
788 return true;
789 }
790
GetSavedRegisterList(uint32 registerUsage)791 uint16 CCodeGen_AArch64::GetSavedRegisterList(uint32 registerUsage)
792 {
793 uint16 registerSave = 0;
794 for(unsigned int i = 0; i < MAX_REGISTERS; i++)
795 {
796 if((1 << i) & registerUsage)
797 {
798 registerSave |= (1 << (g_registers[i] / 2));
799 }
800 }
801 registerSave |= (1 << (g_baseRegister / 2));
802 return registerSave;
803 }
804
Emit_Prolog(const StatementList & statements,uint32 stackSize,uint16 registerSave)805 void CCodeGen_AArch64::Emit_Prolog(const StatementList& statements, uint32 stackSize, uint16 registerSave)
806 {
807 uint32 maxParamSpillSize = GetMaxParamSpillSize(statements);
808 m_assembler.Stp_PreIdx(CAArch64Assembler::x29, CAArch64Assembler::x30, CAArch64Assembler::xSP, -16);
809 //Preserve saved registers
810 for(uint32 i = 0; i < 16; i++)
811 {
812 if(registerSave & (1 << i))
813 {
814 auto reg0 = static_cast<CAArch64Assembler::REGISTER64>((i * 2) + 0);
815 auto reg1 = static_cast<CAArch64Assembler::REGISTER64>((i * 2) + 1);
816 m_assembler.Stp_PreIdx(reg0, reg1, CAArch64Assembler::xSP, -16);
817 }
818 }
819 m_assembler.Mov_Sp(CAArch64Assembler::x29, CAArch64Assembler::xSP);
820 uint32 totalStackAlloc = stackSize + maxParamSpillSize;
821 m_paramSpillBase = stackSize;
822 if(totalStackAlloc != 0)
823 {
824 m_assembler.Sub(CAArch64Assembler::xSP, CAArch64Assembler::xSP, totalStackAlloc, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
825 }
826 m_assembler.Mov(g_baseRegister, CAArch64Assembler::x0);
827 }
828
Emit_Epilog(uint32 stackSize,uint16 registerSave)829 void CCodeGen_AArch64::Emit_Epilog(uint32 stackSize, uint16 registerSave)
830 {
831 m_assembler.Mov_Sp(CAArch64Assembler::xSP, CAArch64Assembler::x29);
832 //Restore saved registers
833 for(int32 i = 15; i >= 0; i--)
834 {
835 if(registerSave & (1 << i))
836 {
837 auto reg0 = static_cast<CAArch64Assembler::REGISTER64>((i * 2) + 0);
838 auto reg1 = static_cast<CAArch64Assembler::REGISTER64>((i * 2) + 1);
839 m_assembler.Ldp_PostIdx(reg0, reg1, CAArch64Assembler::xSP, 16);
840 }
841 }
842 m_assembler.Ldp_PostIdx(CAArch64Assembler::x29, CAArch64Assembler::x30, CAArch64Assembler::xSP, 16);
843 m_assembler.Ret();
844 }
845
GetLabel(uint32 blockId)846 CAArch64Assembler::LABEL CCodeGen_AArch64::GetLabel(uint32 blockId)
847 {
848 CAArch64Assembler::LABEL result;
849 auto labelIterator(m_labels.find(blockId));
850 if(labelIterator == m_labels.end())
851 {
852 result = m_assembler.CreateLabel();
853 m_labels[blockId] = result;
854 }
855 else
856 {
857 result = labelIterator->second;
858 }
859 return result;
860 }
861
MarkLabel(const STATEMENT & statement)862 void CCodeGen_AArch64::MarkLabel(const STATEMENT& statement)
863 {
864 auto label = GetLabel(statement.jmpBlock);
865 m_assembler.MarkLabel(label);
866 }
867
Emit_Nop(const STATEMENT &)868 void CCodeGen_AArch64::Emit_Nop(const STATEMENT&)
869 {
870
871 }
872
Emit_Mov_RegReg(const STATEMENT & statement)873 void CCodeGen_AArch64::Emit_Mov_RegReg(const STATEMENT& statement)
874 {
875 auto dst = statement.dst->GetSymbol().get();
876 auto src1 = statement.src1->GetSymbol().get();
877
878 m_assembler.Mov(g_registers[dst->m_valueLow], g_registers[src1->m_valueLow]);
879 }
880
Emit_Mov_RegMem(const STATEMENT & statement)881 void CCodeGen_AArch64::Emit_Mov_RegMem(const STATEMENT& statement)
882 {
883 auto dst = statement.dst->GetSymbol().get();
884 auto src1 = statement.src1->GetSymbol().get();
885
886 LoadMemoryInRegister(g_registers[dst->m_valueLow], src1);
887 }
888
Emit_Mov_RegCst(const STATEMENT & statement)889 void CCodeGen_AArch64::Emit_Mov_RegCst(const STATEMENT& statement)
890 {
891 auto dst = statement.dst->GetSymbol().get();
892 auto src1 = statement.src1->GetSymbol().get();
893
894 assert(dst->m_type == SYM_REGISTER);
895 assert(src1->m_type == SYM_CONSTANT);
896
897 LoadConstantInRegister(g_registers[dst->m_valueLow], src1->m_valueLow);
898 }
899
Emit_Mov_MemReg(const STATEMENT & statement)900 void CCodeGen_AArch64::Emit_Mov_MemReg(const STATEMENT& statement)
901 {
902 auto dst = statement.dst->GetSymbol().get();
903 auto src1 = statement.src1->GetSymbol().get();
904
905 assert(src1->m_type == SYM_REGISTER);
906
907 StoreRegisterInMemory(dst, g_registers[src1->m_valueLow]);
908 }
909
Emit_Mov_MemMem(const STATEMENT & statement)910 void CCodeGen_AArch64::Emit_Mov_MemMem(const STATEMENT& statement)
911 {
912 auto dst = statement.dst->GetSymbol().get();
913 auto src1 = statement.src1->GetSymbol().get();
914
915 auto tmpReg = GetNextTempRegister();
916 LoadMemoryInRegister(tmpReg, src1);
917 StoreRegisterInMemory(dst, tmpReg);
918 }
919
Emit_Mov_MemCst(const STATEMENT & statement)920 void CCodeGen_AArch64::Emit_Mov_MemCst(const STATEMENT& statement)
921 {
922 auto dst = statement.dst->GetSymbol().get();
923 auto src1 = statement.src1->GetSymbol().get();
924
925 assert(src1->m_type == SYM_CONSTANT);
926
927 auto tmpReg = GetNextTempRegister();
928 LoadConstantInRegister(tmpReg, src1->m_valueLow);
929 StoreRegisterInMemory(dst, tmpReg);
930 }
931
Emit_Not_VarVar(const STATEMENT & statement)932 void CCodeGen_AArch64::Emit_Not_VarVar(const STATEMENT& statement)
933 {
934 auto dst = statement.dst->GetSymbol().get();
935 auto src1 = statement.src1->GetSymbol().get();
936
937 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
938 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
939 m_assembler.Mvn(dstReg, src1Reg);
940 CommitSymbolRegister(dst, dstReg);
941 }
942
Emit_Lzc_VarVar(const STATEMENT & statement)943 void CCodeGen_AArch64::Emit_Lzc_VarVar(const STATEMENT& statement)
944 {
945 auto dst = statement.dst->GetSymbol().get();
946 auto src1 = statement.src1->GetSymbol().get();
947
948 auto dstRegister = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
949 auto src1Register = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
950
951 auto set32Label = m_assembler.CreateLabel();
952 auto startCountLabel = m_assembler.CreateLabel();
953 auto doneLabel = m_assembler.CreateLabel();
954
955 m_assembler.Mov(dstRegister, src1Register);
956 m_assembler.Tst(dstRegister, dstRegister);
957 m_assembler.BCc(CAArch64Assembler::CONDITION_EQ, set32Label);
958 m_assembler.BCc(CAArch64Assembler::CONDITION_PL, startCountLabel);
959
960 //reverse:
961 m_assembler.Mvn(dstRegister, dstRegister);
962 m_assembler.Tst(dstRegister, dstRegister);
963 m_assembler.BCc(CAArch64Assembler::CONDITION_EQ, set32Label);
964
965 //startCount:
966 m_assembler.MarkLabel(startCountLabel);
967 m_assembler.Clz(dstRegister, dstRegister);
968 m_assembler.Sub(dstRegister, dstRegister, 1, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
969 m_assembler.BCc(CAArch64Assembler::CONDITION_AL, doneLabel);
970
971 //set32:
972 m_assembler.MarkLabel(set32Label);
973 LoadConstantInRegister(dstRegister, 0x1F);
974
975 //done
976 m_assembler.MarkLabel(doneLabel);
977
978 CommitSymbolRegister(dst, dstRegister);
979 }
980
Emit_RelToRef_TmpCst(const STATEMENT & statement)981 void CCodeGen_AArch64::Emit_RelToRef_TmpCst(const STATEMENT& statement)
982 {
983 auto dst = statement.dst->GetSymbol().get();
984 auto src1 = statement.src1->GetSymbol().get();
985
986 assert(src1->m_type == SYM_CONSTANT);
987
988 auto tmpReg = GetNextTempRegister64();
989
990 ADDSUB_IMM_PARAMS addSubImmParams;
991 if(TryGetAddSubImmParams(src1->m_valueLow, addSubImmParams))
992 {
993 m_assembler.Add(tmpReg, g_baseRegister, addSubImmParams.imm, addSubImmParams.shiftType);
994 }
995 else
996 {
997 assert(false);
998 }
999
1000 StoreRegisterInTemporaryReference(dst, tmpReg);
1001 }
1002
Emit_AddRef_TmpMemAny(const STATEMENT & statement)1003 void CCodeGen_AArch64::Emit_AddRef_TmpMemAny(const STATEMENT& statement)
1004 {
1005 auto dst = statement.dst->GetSymbol().get();
1006 auto src1 = statement.src1->GetSymbol().get();
1007 auto src2 = statement.src2->GetSymbol().get();
1008
1009 auto tmpReg = GetNextTempRegister64();
1010 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
1011
1012 LoadMemoryReferenceInRegister(tmpReg, src1);
1013 m_assembler.Add(tmpReg, tmpReg, static_cast<CAArch64Assembler::REGISTER64>(src2Reg));
1014 StoreRegisterInTemporaryReference(dst, tmpReg);
1015 }
1016
Emit_LoadFromRef_VarMem(const STATEMENT & statement)1017 void CCodeGen_AArch64::Emit_LoadFromRef_VarMem(const STATEMENT& statement)
1018 {
1019 auto dst = statement.dst->GetSymbol().get();
1020 auto src1 = statement.src1->GetSymbol().get();
1021
1022 auto addressReg = GetNextTempRegister64();
1023 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
1024
1025 LoadMemoryReferenceInRegister(addressReg, src1);
1026 m_assembler.Ldr(dstReg, addressReg, 0);
1027
1028 CommitSymbolRegister(dst, dstReg);
1029 }
1030
Emit_StoreAtRef_MemAny(const STATEMENT & statement)1031 void CCodeGen_AArch64::Emit_StoreAtRef_MemAny(const STATEMENT& statement)
1032 {
1033 auto src1 = statement.src1->GetSymbol().get();
1034 auto src2 = statement.src2->GetSymbol().get();
1035
1036 assert(src1->m_type == SYM_TMP_REFERENCE);
1037
1038 auto addressReg = GetNextTempRegister64();
1039 auto valueReg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
1040
1041 LoadMemoryReferenceInRegister(addressReg, src1);
1042 m_assembler.Str(valueReg, addressReg, 0);
1043 }
1044
Emit_Param_Ctx(const STATEMENT & statement)1045 void CCodeGen_AArch64::Emit_Param_Ctx(const STATEMENT& statement)
1046 {
1047 auto src1 = statement.src1->GetSymbol().get();
1048
1049 assert(src1->m_type == SYM_CONTEXT);
1050
1051 m_params.push_back(
1052 [this] (PARAM_STATE& paramState)
1053 {
1054 auto paramReg = PrepareParam64(paramState);
1055 m_assembler.Mov(paramReg, g_baseRegister);
1056 CommitParam64(paramState);
1057 }
1058 );
1059 }
1060
Emit_Param_Reg(const STATEMENT & statement)1061 void CCodeGen_AArch64::Emit_Param_Reg(const STATEMENT& statement)
1062 {
1063 auto src1 = statement.src1->GetSymbol().get();
1064
1065 assert(src1->m_type == SYM_REGISTER);
1066
1067 m_params.push_back(
1068 [this, src1] (PARAM_STATE& paramState)
1069 {
1070 auto paramReg = PrepareParam(paramState);
1071 m_assembler.Mov(paramReg, g_registers[src1->m_valueLow]);
1072 CommitParam(paramState);
1073 }
1074 );
1075 }
1076
Emit_Param_Mem(const STATEMENT & statement)1077 void CCodeGen_AArch64::Emit_Param_Mem(const STATEMENT& statement)
1078 {
1079 auto src1 = statement.src1->GetSymbol().get();
1080
1081 m_params.push_back(
1082 [this, src1] (PARAM_STATE& paramState)
1083 {
1084 auto paramReg = PrepareParam(paramState);
1085 LoadMemoryInRegister(paramReg, src1);
1086 CommitParam(paramState);
1087 }
1088 );
1089 }
1090
Emit_Param_Cst(const STATEMENT & statement)1091 void CCodeGen_AArch64::Emit_Param_Cst(const STATEMENT& statement)
1092 {
1093 auto src1 = statement.src1->GetSymbol().get();
1094
1095 m_params.push_back(
1096 [this, src1] (PARAM_STATE& paramState)
1097 {
1098 auto paramReg = PrepareParam(paramState);
1099 LoadConstantInRegister(paramReg, src1->m_valueLow);
1100 CommitParam(paramState);
1101 }
1102 );
1103 }
1104
Emit_Param_Mem64(const STATEMENT & statement)1105 void CCodeGen_AArch64::Emit_Param_Mem64(const STATEMENT& statement)
1106 {
1107 auto src1 = statement.src1->GetSymbol().get();
1108
1109 m_params.push_back(
1110 [this, src1] (PARAM_STATE& paramState)
1111 {
1112 auto paramReg = PrepareParam64(paramState);
1113 LoadMemory64InRegister(paramReg, src1);
1114 CommitParam64(paramState);
1115 }
1116 );
1117 }
1118
Emit_Param_Cst64(const STATEMENT & statement)1119 void CCodeGen_AArch64::Emit_Param_Cst64(const STATEMENT& statement)
1120 {
1121 auto src1 = statement.src1->GetSymbol().get();
1122
1123 m_params.push_back(
1124 [this, src1] (PARAM_STATE& paramState)
1125 {
1126 auto paramReg = PrepareParam64(paramState);
1127 LoadConstant64InRegister(paramReg, src1->GetConstant64());
1128 CommitParam64(paramState);
1129 }
1130 );
1131 }
1132
Emit_Param_Reg128(const STATEMENT & statement)1133 void CCodeGen_AArch64::Emit_Param_Reg128(const STATEMENT& statement)
1134 {
1135 auto src1 = statement.src1->GetSymbol().get();
1136
1137 m_params.push_back(
1138 [this, src1] (PARAM_STATE& paramState)
1139 {
1140 auto paramReg = PrepareParam64(paramState);
1141 uint32 paramBase = m_paramSpillBase + paramState.spillOffset;
1142 m_assembler.Add(paramReg, CAArch64Assembler::xSP, paramBase, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
1143 m_assembler.Str_1q(g_registersMd[src1->m_valueLow], CAArch64Assembler::xSP, paramBase);
1144 paramState.spillOffset += 0x10;
1145 CommitParam64(paramState);
1146 }
1147 );
1148 }
1149
Emit_Param_Mem128(const STATEMENT & statement)1150 void CCodeGen_AArch64::Emit_Param_Mem128(const STATEMENT& statement)
1151 {
1152 auto src1 = statement.src1->GetSymbol().get();
1153
1154 m_params.push_back(
1155 [this, src1] (PARAM_STATE& paramState)
1156 {
1157 auto paramReg = PrepareParam64(paramState);
1158 LoadMemory128AddressInRegister(paramReg, src1);
1159 CommitParam64(paramState);
1160 }
1161 );
1162 }
1163
Emit_Call(const STATEMENT & statement)1164 void CCodeGen_AArch64::Emit_Call(const STATEMENT& statement)
1165 {
1166 auto src1 = statement.src1->GetSymbol().get();
1167 auto src2 = statement.src2->GetSymbol().get();
1168
1169 assert(src1->m_type == SYM_CONSTANTPTR);
1170 assert(src2->m_type == SYM_CONSTANT);
1171
1172 unsigned int paramCount = src2->m_valueLow;
1173 PARAM_STATE paramState;
1174
1175 for(unsigned int i = 0; i < paramCount; i++)
1176 {
1177 auto emitter(m_params.back());
1178 m_params.pop_back();
1179 emitter(paramState);
1180 }
1181
1182 if(m_generateRelocatableCalls)
1183 {
1184 if(m_externalSymbolReferencedHandler)
1185 {
1186 auto position = m_stream->GetLength();
1187 m_externalSymbolReferencedHandler(src1->GetConstantPtr(), position);
1188 }
1189 m_assembler.Bl(0);
1190 }
1191 else
1192 {
1193 auto fctAddressReg = GetNextTempRegister64();
1194 LoadConstant64InRegister(fctAddressReg, src1->GetConstantPtr());
1195 m_assembler.Blr(fctAddressReg);
1196 }
1197 }
1198
Emit_RetVal_Reg(const STATEMENT & statement)1199 void CCodeGen_AArch64::Emit_RetVal_Reg(const STATEMENT& statement)
1200 {
1201 auto dst = statement.dst->GetSymbol().get();
1202 assert(dst->m_type == SYM_REGISTER);
1203 m_assembler.Mov(g_registers[dst->m_valueLow], CAArch64Assembler::w0);
1204 }
1205
Emit_RetVal_Tmp(const STATEMENT & statement)1206 void CCodeGen_AArch64::Emit_RetVal_Tmp(const STATEMENT& statement)
1207 {
1208 auto dst = statement.dst->GetSymbol().get();
1209 assert(dst->m_type == SYM_TEMPORARY);
1210 StoreRegisterInMemory(dst, CAArch64Assembler::w0);
1211 }
1212
Emit_RetVal_Mem64(const STATEMENT & statement)1213 void CCodeGen_AArch64::Emit_RetVal_Mem64(const STATEMENT& statement)
1214 {
1215 auto dst = statement.dst->GetSymbol().get();
1216 StoreRegisterInMemory64(dst, CAArch64Assembler::x0);
1217 }
1218
Emit_RetVal_Reg128(const STATEMENT & statement)1219 void CCodeGen_AArch64::Emit_RetVal_Reg128(const STATEMENT& statement)
1220 {
1221 auto dst = statement.dst->GetSymbol().get();
1222
1223 m_assembler.Ins_1d(g_registersMd[dst->m_valueLow], 0, CAArch64Assembler::x0);
1224 m_assembler.Ins_1d(g_registersMd[dst->m_valueLow], 1, CAArch64Assembler::x1);
1225 }
1226
Emit_RetVal_Mem128(const STATEMENT & statement)1227 void CCodeGen_AArch64::Emit_RetVal_Mem128(const STATEMENT& statement)
1228 {
1229 auto dst = statement.dst->GetSymbol().get();
1230
1231 auto dstAddrReg = GetNextTempRegister64();
1232
1233 LoadMemory128AddressInRegister(dstAddrReg, dst);
1234 m_assembler.Str(CAArch64Assembler::x0, dstAddrReg, 0);
1235 m_assembler.Str(CAArch64Assembler::x1, dstAddrReg, 8);
1236 }
1237
Emit_Jmp(const STATEMENT & statement)1238 void CCodeGen_AArch64::Emit_Jmp(const STATEMENT& statement)
1239 {
1240 m_assembler.B(GetLabel(statement.jmpBlock));
1241 }
1242
Emit_CondJmp(const STATEMENT & statement)1243 void CCodeGen_AArch64::Emit_CondJmp(const STATEMENT& statement)
1244 {
1245 auto label(GetLabel(statement.jmpBlock));
1246
1247 switch(statement.jmpCondition)
1248 {
1249 case CONDITION_EQ:
1250 m_assembler.BCc(CAArch64Assembler::CONDITION_EQ, label);
1251 break;
1252 case CONDITION_NE:
1253 m_assembler.BCc(CAArch64Assembler::CONDITION_NE, label);
1254 break;
1255 case CONDITION_LT:
1256 m_assembler.BCc(CAArch64Assembler::CONDITION_LT, label);
1257 break;
1258 case CONDITION_LE:
1259 m_assembler.BCc(CAArch64Assembler::CONDITION_LE, label);
1260 break;
1261 case CONDITION_GT:
1262 m_assembler.BCc(CAArch64Assembler::CONDITION_GT, label);
1263 break;
1264 case CONDITION_GE:
1265 m_assembler.BCc(CAArch64Assembler::CONDITION_GE, label);
1266 break;
1267 default:
1268 assert(0);
1269 break;
1270 }
1271 }
1272
Emit_CondJmp_AnyVar(const STATEMENT & statement)1273 void CCodeGen_AArch64::Emit_CondJmp_AnyVar(const STATEMENT& statement)
1274 {
1275 auto src1 = statement.src1->GetSymbol().get();
1276 auto src2 = statement.src2->GetSymbol().get();
1277
1278 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
1279 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
1280 m_assembler.Cmp(src1Reg, src2Reg);
1281 Emit_CondJmp(statement);
1282 }
1283
Emit_CondJmp_VarCst(const STATEMENT & statement)1284 void CCodeGen_AArch64::Emit_CondJmp_VarCst(const STATEMENT& statement)
1285 {
1286 auto src1 = statement.src1->GetSymbol().get();
1287 auto src2 = statement.src2->GetSymbol().get();
1288
1289 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
1290 assert(src2->m_type == SYM_CONSTANT);
1291
1292 ADDSUB_IMM_PARAMS addSubImmParams;
1293 if(TryGetAddSubImmParams(src2->m_valueLow, addSubImmParams))
1294 {
1295 m_assembler.Cmp(src1Reg, addSubImmParams.imm, addSubImmParams.shiftType);
1296 }
1297 else if(TryGetAddSubImmParams(-static_cast<int32>(src2->m_valueLow), addSubImmParams))
1298 {
1299 m_assembler.Cmn(src1Reg, addSubImmParams.imm, addSubImmParams.shiftType);
1300 }
1301 else
1302 {
1303 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
1304 m_assembler.Cmp(src1Reg, src2Reg);
1305 }
1306
1307 Emit_CondJmp(statement);
1308 }
1309
Cmp_GetFlag(CAArch64Assembler::REGISTER32 registerId,Jitter::CONDITION condition)1310 void CCodeGen_AArch64::Cmp_GetFlag(CAArch64Assembler::REGISTER32 registerId, Jitter::CONDITION condition)
1311 {
1312 switch(condition)
1313 {
1314 case CONDITION_EQ:
1315 m_assembler.Cset(registerId, CAArch64Assembler::CONDITION_EQ);
1316 break;
1317 case CONDITION_NE:
1318 m_assembler.Cset(registerId, CAArch64Assembler::CONDITION_NE);
1319 break;
1320 case CONDITION_LT:
1321 m_assembler.Cset(registerId, CAArch64Assembler::CONDITION_LT);
1322 break;
1323 case CONDITION_LE:
1324 m_assembler.Cset(registerId, CAArch64Assembler::CONDITION_LE);
1325 break;
1326 case CONDITION_GT:
1327 m_assembler.Cset(registerId, CAArch64Assembler::CONDITION_GT);
1328 break;
1329 case CONDITION_BL:
1330 m_assembler.Cset(registerId, CAArch64Assembler::CONDITION_CC);
1331 break;
1332 case CONDITION_BE:
1333 m_assembler.Cset(registerId, CAArch64Assembler::CONDITION_LS);
1334 break;
1335 case CONDITION_AB:
1336 m_assembler.Cset(registerId, CAArch64Assembler::CONDITION_HI);
1337 break;
1338 default:
1339 assert(0);
1340 break;
1341 }
1342 }
1343
Emit_Cmp_VarAnyVar(const STATEMENT & statement)1344 void CCodeGen_AArch64::Emit_Cmp_VarAnyVar(const STATEMENT& statement)
1345 {
1346 auto dst = statement.dst->GetSymbol().get();
1347 auto src1 = statement.src1->GetSymbol().get();
1348 auto src2 = statement.src2->GetSymbol().get();
1349
1350 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
1351 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
1352 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
1353 m_assembler.Cmp(src1Reg, src2Reg);
1354 Cmp_GetFlag(dstReg, statement.jmpCondition);
1355 CommitSymbolRegister(dst, dstReg);
1356 }
1357
Emit_Cmp_VarVarCst(const STATEMENT & statement)1358 void CCodeGen_AArch64::Emit_Cmp_VarVarCst(const STATEMENT& statement)
1359 {
1360 auto dst = statement.dst->GetSymbol().get();
1361 auto src1 = statement.src1->GetSymbol().get();
1362 auto src2 = statement.src2->GetSymbol().get();
1363
1364 assert(src2->m_type == SYM_CONSTANT);
1365
1366 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
1367 auto src1Reg = PrepareSymbolRegisterUse(src1, GetNextTempRegister());
1368
1369 ADDSUB_IMM_PARAMS addSubImmParams;
1370 if(TryGetAddSubImmParams(src2->m_valueLow, addSubImmParams))
1371 {
1372 m_assembler.Cmp(src1Reg, addSubImmParams.imm, addSubImmParams.shiftType);
1373 }
1374 else if(TryGetAddSubImmParams(-static_cast<int32>(src2->m_valueLow), addSubImmParams))
1375 {
1376 m_assembler.Cmn(src1Reg, addSubImmParams.imm, addSubImmParams.shiftType);
1377 }
1378 else
1379 {
1380 auto src2Reg = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
1381 m_assembler.Cmp(src1Reg, src2Reg);
1382 }
1383
1384 Cmp_GetFlag(dstReg, statement.jmpCondition);
1385 CommitSymbolRegister(dst, dstReg);
1386 }
1387