1 #include "Jitter_CodeGen_AArch64.h"
2 #include <stdexcept>
3
4 using namespace Jitter;
5
LoadMemory128InRegister(CAArch64Assembler::REGISTERMD dstReg,CSymbol * symbol)6 void CCodeGen_AArch64::LoadMemory128InRegister(CAArch64Assembler::REGISTERMD dstReg, CSymbol* symbol)
7 {
8 switch(symbol->m_type)
9 {
10 case SYM_RELATIVE128:
11 m_assembler.Ldr_1q(dstReg, g_baseRegister, symbol->m_valueLow);
12 break;
13 case SYM_TEMPORARY128:
14 m_assembler.Ldr_1q(dstReg, CAArch64Assembler::xSP, symbol->m_stackLocation);
15 break;
16 default:
17 assert(0);
18 break;
19 }
20 }
21
StoreRegisterInMemory128(CSymbol * symbol,CAArch64Assembler::REGISTERMD srcReg)22 void CCodeGen_AArch64::StoreRegisterInMemory128(CSymbol* symbol, CAArch64Assembler::REGISTERMD srcReg)
23 {
24 switch(symbol->m_type)
25 {
26 case SYM_RELATIVE128:
27 m_assembler.Str_1q(srcReg, g_baseRegister, symbol->m_valueLow);
28 break;
29 case SYM_TEMPORARY128:
30 m_assembler.Str_1q(srcReg, CAArch64Assembler::xSP, symbol->m_stackLocation);
31 break;
32 default:
33 assert(0);
34 break;
35 }
36 }
37
LoadMemory128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg,CSymbol * symbol,uint32 offset)38 void CCodeGen_AArch64::LoadMemory128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg, CSymbol* symbol, uint32 offset)
39 {
40 switch(symbol->m_type)
41 {
42 case SYM_RELATIVE128:
43 LoadRelative128AddressInRegister(dstReg, symbol, offset);
44 break;
45 case SYM_TEMPORARY128:
46 LoadTemporary128AddressInRegister(dstReg, symbol, offset);
47 break;
48 default:
49 assert(0);
50 break;
51 }
52 }
53
LoadRelative128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg,CSymbol * symbol,uint32 offset)54 void CCodeGen_AArch64::LoadRelative128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg, CSymbol* symbol, uint32 offset)
55 {
56 assert(symbol->m_type == SYM_RELATIVE128);
57
58 uint32 totalOffset = symbol->m_valueLow + offset;
59 assert(totalOffset < 0x1000);
60 m_assembler.Add(dstReg, g_baseRegister, totalOffset, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
61 }
62
LoadTemporary128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg,CSymbol * symbol,uint32 offset)63 void CCodeGen_AArch64::LoadTemporary128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg, CSymbol* symbol, uint32 offset)
64 {
65 assert(symbol->m_type == SYM_TEMPORARY128);
66
67 uint32 totalOffset = symbol->m_stackLocation + offset;
68 assert(totalOffset < 0x1000);
69 m_assembler.Add(dstReg, CAArch64Assembler::xSP, totalOffset, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
70 }
71
LoadTemporary256ElementAddressInRegister(CAArch64Assembler::REGISTER64 dstReg,CSymbol * symbol,uint32 offset)72 void CCodeGen_AArch64::LoadTemporary256ElementAddressInRegister(CAArch64Assembler::REGISTER64 dstReg, CSymbol* symbol, uint32 offset)
73 {
74 assert(symbol->m_type == SYM_TEMPORARY256);
75
76 uint32 totalOffset = symbol->m_stackLocation + offset;
77 assert(totalOffset < 0x1000);
78 m_assembler.Add(dstReg, CAArch64Assembler::xSP, totalOffset, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
79 }
80
PrepareSymbolRegisterDefMd(CSymbol * symbol,CAArch64Assembler::REGISTERMD preferedRegister)81 CAArch64Assembler::REGISTERMD CCodeGen_AArch64::PrepareSymbolRegisterDefMd(CSymbol* symbol, CAArch64Assembler::REGISTERMD preferedRegister)
82 {
83 switch(symbol->m_type)
84 {
85 case SYM_REGISTER128:
86 assert(symbol->m_valueLow < MAX_MDREGISTERS);
87 return g_registersMd[symbol->m_valueLow];
88 break;
89 case SYM_TEMPORARY128:
90 case SYM_RELATIVE128:
91 return preferedRegister;
92 break;
93 default:
94 throw std::runtime_error("Invalid symbol type.");
95 break;
96 }
97 }
98
PrepareSymbolRegisterUseMd(CSymbol * symbol,CAArch64Assembler::REGISTERMD preferedRegister)99 CAArch64Assembler::REGISTERMD CCodeGen_AArch64::PrepareSymbolRegisterUseMd(CSymbol* symbol, CAArch64Assembler::REGISTERMD preferedRegister)
100 {
101 switch(symbol->m_type)
102 {
103 case SYM_REGISTER128:
104 assert(symbol->m_valueLow < MAX_MDREGISTERS);
105 return g_registersMd[symbol->m_valueLow];
106 break;
107 case SYM_TEMPORARY128:
108 case SYM_RELATIVE128:
109 LoadMemory128InRegister(preferedRegister, symbol);
110 return preferedRegister;
111 break;
112 default:
113 throw std::runtime_error("Invalid symbol type.");
114 break;
115 }
116 }
117
CommitSymbolRegisterMd(CSymbol * symbol,CAArch64Assembler::REGISTERMD usedRegister)118 void CCodeGen_AArch64::CommitSymbolRegisterMd(CSymbol* symbol, CAArch64Assembler::REGISTERMD usedRegister)
119 {
120 switch(symbol->m_type)
121 {
122 case SYM_REGISTER128:
123 assert(usedRegister == g_registersMd[symbol->m_valueLow]);
124 break;
125 case SYM_TEMPORARY128:
126 case SYM_RELATIVE128:
127 StoreRegisterInMemory128(symbol, usedRegister);
128 break;
129 default:
130 throw std::runtime_error("Invalid symbol type.");
131 break;
132 }
133 }
134
135 template <typename MDOP>
Emit_Md_VarVar(const STATEMENT & statement)136 void CCodeGen_AArch64::Emit_Md_VarVar(const STATEMENT& statement)
137 {
138 auto dst = statement.dst->GetSymbol().get();
139 auto src1 = statement.src1->GetSymbol().get();
140
141 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
142 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
143
144 ((m_assembler).*(MDOP::OpReg()))(dstReg, src1Reg);
145
146 CommitSymbolRegisterMd(dst, dstReg);
147 }
148
149 template <typename MDOP>
Emit_Md_VarVarVar(const STATEMENT & statement)150 void CCodeGen_AArch64::Emit_Md_VarVarVar(const STATEMENT& statement)
151 {
152 auto dst = statement.dst->GetSymbol().get();
153 auto src1 = statement.src1->GetSymbol().get();
154 auto src2 = statement.src2->GetSymbol().get();
155
156 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
157 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
158 auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
159
160 ((m_assembler).*(MDOP::OpReg()))(dstReg, src1Reg, src2Reg);
161
162 CommitSymbolRegisterMd(dst, dstReg);
163 }
164
165 template <typename MDOP>
Emit_Md_VarVarVarRev(const STATEMENT & statement)166 void CCodeGen_AArch64::Emit_Md_VarVarVarRev(const STATEMENT& statement)
167 {
168 auto dst = statement.dst->GetSymbol().get();
169 auto src1 = statement.src1->GetSymbol().get();
170 auto src2 = statement.src2->GetSymbol().get();
171
172 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
173 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
174 auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
175
176 ((m_assembler).*(MDOP::OpReg()))(dstReg, src2Reg, src1Reg);
177
178 CommitSymbolRegisterMd(dst, dstReg);
179 }
180
181 template <typename MDSHIFTOP>
Emit_Md_Shift_VarVarCst(const STATEMENT & statement)182 void CCodeGen_AArch64::Emit_Md_Shift_VarVarCst(const STATEMENT& statement)
183 {
184 auto dst = statement.dst->GetSymbol().get();
185 auto src1 = statement.src1->GetSymbol().get();
186 auto src2 = statement.src2->GetSymbol().get();
187
188 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
189 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
190
191 ((m_assembler).*(MDSHIFTOP::OpReg()))(dstReg, src1Reg, src2->m_valueLow);
192
193 CommitSymbolRegisterMd(dst, dstReg);
194 }
195
Emit_Md_MakeSz_VarVar(const STATEMENT & statement)196 void CCodeGen_AArch64::Emit_Md_MakeSz_VarVar(const STATEMENT& statement)
197 {
198 auto dst = statement.dst->GetSymbol().get();
199 auto src1 = statement.src1->GetSymbol().get();
200
201 m_nextTempRegisterMd = 0;
202
203 auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
204 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
205
206 auto signReg = GetNextTempRegisterMd();
207 auto zeroReg = GetNextTempRegisterMd();
208 auto cstReg = GetNextTempRegisterMd();
209
210 assert(zeroReg == signReg + 1);
211
212 m_assembler.Cmltz_4s(signReg, src1Reg);
213 m_assembler.Fcmeqz_4s(zeroReg, src1Reg);
214
215 LITERAL128 lit1(0x0004080C1014181CUL, 0xFFFFFFFFFFFFFFFFUL);
216 LITERAL128 lit2(0x8040201008040201UL, 0x0000000000000000UL);
217
218 m_assembler.Ldr_Pc(cstReg, lit1);
219 m_assembler.Tbl(signReg, signReg, cstReg);
220 m_assembler.Ldr_Pc(cstReg, lit2);
221 m_assembler.And_16b(signReg, signReg, cstReg);
222 m_assembler.Uaddlv_16b(signReg, signReg);
223 m_assembler.Umov_1s(dstReg, signReg, 0);
224
225 CommitSymbolRegister(dst, dstReg);
226 }
227
Emit_Md_Mov_RegReg(const STATEMENT & statement)228 void CCodeGen_AArch64::Emit_Md_Mov_RegReg(const STATEMENT& statement)
229 {
230 auto dst = statement.dst->GetSymbol().get();
231 auto src1 = statement.src1->GetSymbol().get();
232
233 assert(!dst->Equals(src1));
234
235 m_assembler.Mov(g_registersMd[dst->m_valueLow], g_registersMd[src1->m_valueLow]);
236 }
237
Emit_Md_Mov_RegMem(const STATEMENT & statement)238 void CCodeGen_AArch64::Emit_Md_Mov_RegMem(const STATEMENT& statement)
239 {
240 auto dst = statement.dst->GetSymbol().get();
241 auto src1 = statement.src1->GetSymbol().get();
242
243 LoadMemory128InRegister(g_registersMd[dst->m_valueLow], src1);
244 }
245
Emit_Md_Mov_MemReg(const STATEMENT & statement)246 void CCodeGen_AArch64::Emit_Md_Mov_MemReg(const STATEMENT& statement)
247 {
248 auto dst = statement.dst->GetSymbol().get();
249 auto src1 = statement.src1->GetSymbol().get();
250
251 StoreRegisterInMemory128(dst, g_registersMd[src1->m_valueLow]);
252 }
253
Emit_Md_Mov_MemMem(const STATEMENT & statement)254 void CCodeGen_AArch64::Emit_Md_Mov_MemMem(const STATEMENT& statement)
255 {
256 auto dst = statement.dst->GetSymbol().get();
257 auto src1 = statement.src1->GetSymbol().get();
258
259 auto tmpReg = GetNextTempRegisterMd();
260
261 LoadMemory128InRegister(tmpReg, src1);
262 StoreRegisterInMemory128(dst, tmpReg);
263 }
264
Emit_Md_CmpLtS_VarVarVar(const STATEMENT & statement)265 void CCodeGen_AArch64::Emit_Md_CmpLtS_VarVarVar(const STATEMENT& statement)
266 {
267 auto dst = statement.dst->GetSymbol().get();
268 auto src1 = statement.src1->GetSymbol().get();
269 auto src2 = statement.src2->GetSymbol().get();
270
271 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
272 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
273 auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
274
275 m_assembler.Fcmge_4s(dstReg, src1Reg, src2Reg);
276 m_assembler.Mvn_16b(dstReg, dstReg);
277
278 CommitSymbolRegisterMd(dst, dstReg);
279 }
280
Emit_Md_LoadFromRef_VarVar(const STATEMENT & statement)281 void CCodeGen_AArch64::Emit_Md_LoadFromRef_VarVar(const STATEMENT& statement)
282 {
283 auto dst = statement.dst->GetSymbol().get();
284 auto src1 = statement.src1->GetSymbol().get();
285
286 auto src1AddrReg = PrepareSymbolRegisterUseRef(src1, GetNextTempRegister64());
287 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
288
289 m_assembler.Ldr_1q(dstReg, src1AddrReg, 0);
290
291 CommitSymbolRegisterMd(dst, dstReg);
292 }
293
Emit_Md_StoreAtRef_VarVar(const STATEMENT & statement)294 void CCodeGen_AArch64::Emit_Md_StoreAtRef_VarVar(const STATEMENT& statement)
295 {
296 auto src1 = statement.src1->GetSymbol().get();
297 auto src2 = statement.src2->GetSymbol().get();
298
299 auto src1AddrReg = PrepareSymbolRegisterUseRef(src1, GetNextTempRegister64());
300 auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
301
302 m_assembler.Str_1q(src2Reg, src1AddrReg, 0);
303 }
304
Emit_Md_MovMasked_VarVarVar(const STATEMENT & statement)305 void CCodeGen_AArch64::Emit_Md_MovMasked_VarVarVar(const STATEMENT& statement)
306 {
307 auto dst = statement.dst->GetSymbol().get();
308 auto src1 = statement.src1->GetSymbol().get();
309 auto src2 = statement.src2->GetSymbol().get();
310
311 assert(dst->Equals(src1));
312
313 auto mask = static_cast<uint8>(statement.jmpCondition);
314
315 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
316 auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
317
318 for(unsigned int i = 0; i < 4; i++)
319 {
320 if(mask & (1 << i))
321 {
322 m_assembler.Ins_1s(src1Reg, i, src2Reg, i);
323 }
324 }
325
326 //This is only valid if dst == src1
327 CommitSymbolRegisterMd(dst, src1Reg);
328 }
329
Emit_Md_Expand_VarReg(const STATEMENT & statement)330 void CCodeGen_AArch64::Emit_Md_Expand_VarReg(const STATEMENT& statement)
331 {
332 auto dst = statement.dst->GetSymbol().get();
333 auto src1 = statement.src1->GetSymbol().get();
334
335 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
336
337 m_assembler.Dup_4s(dstReg, g_registers[src1->m_valueLow]);
338
339 CommitSymbolRegisterMd(dst, dstReg);
340 }
341
Emit_Md_Expand_VarMem(const STATEMENT & statement)342 void CCodeGen_AArch64::Emit_Md_Expand_VarMem(const STATEMENT& statement)
343 {
344 auto dst = statement.dst->GetSymbol().get();
345 auto src1 = statement.src1->GetSymbol().get();
346
347 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
348 auto src1Reg = GetNextTempRegister();
349
350 LoadMemoryInRegister(src1Reg, src1);
351
352 m_assembler.Dup_4s(dstReg, src1Reg);
353
354 CommitSymbolRegisterMd(dst, dstReg);
355 }
356
Emit_Md_Expand_VarCst(const STATEMENT & statement)357 void CCodeGen_AArch64::Emit_Md_Expand_VarCst(const STATEMENT& statement)
358 {
359 auto dst = statement.dst->GetSymbol().get();
360 auto src1 = statement.src1->GetSymbol().get();
361
362 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
363 auto src1Reg = GetNextTempRegister();
364
365 LoadConstantInRegister(src1Reg, src1->m_valueLow);
366
367 m_assembler.Dup_4s(dstReg, src1Reg);
368
369 CommitSymbolRegisterMd(dst, dstReg);
370 }
371
Emit_Md_PackHB_VarVarVar(const STATEMENT & statement)372 void CCodeGen_AArch64::Emit_Md_PackHB_VarVarVar(const STATEMENT& statement)
373 {
374 auto dst = statement.dst->GetSymbol().get();
375 auto src1 = statement.src1->GetSymbol().get();
376 auto src2 = statement.src2->GetSymbol().get();
377
378 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
379 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
380 auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
381
382 if(dstReg == src1Reg)
383 {
384 auto tmpReg = GetNextTempRegisterMd();
385 m_assembler.Xtn1_8b(tmpReg, src2Reg);
386 m_assembler.Xtn2_16b(tmpReg, src1Reg);
387 m_assembler.Mov(dstReg, tmpReg);
388 }
389 else
390 {
391 m_assembler.Xtn1_8b(dstReg, src2Reg);
392 m_assembler.Xtn2_16b(dstReg, src1Reg);
393 }
394
395 CommitSymbolRegisterMd(dst, dstReg);
396 }
397
Emit_Md_PackWH_VarVarVar(const STATEMENT & statement)398 void CCodeGen_AArch64::Emit_Md_PackWH_VarVarVar(const STATEMENT& statement)
399 {
400 auto dst = statement.dst->GetSymbol().get();
401 auto src1 = statement.src1->GetSymbol().get();
402 auto src2 = statement.src2->GetSymbol().get();
403
404 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
405 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
406 auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
407
408 if(dstReg == src1Reg)
409 {
410 auto tmpReg = GetNextTempRegisterMd();
411 m_assembler.Xtn1_4h(tmpReg, src2Reg);
412 m_assembler.Xtn2_8h(tmpReg, src1Reg);
413 m_assembler.Mov(dstReg, tmpReg);
414 }
415 else
416 {
417 m_assembler.Xtn1_4h(dstReg, src2Reg);
418 m_assembler.Xtn2_8h(dstReg, src1Reg);
419 }
420
421 CommitSymbolRegisterMd(dst, dstReg);
422 }
423
Emit_MergeTo256_MemVarVar(const STATEMENT & statement)424 void CCodeGen_AArch64::Emit_MergeTo256_MemVarVar(const STATEMENT& statement)
425 {
426 auto dst = statement.dst->GetSymbol().get();
427 auto src1 = statement.src1->GetSymbol().get();
428 auto src2 = statement.src2->GetSymbol().get();
429
430 assert(dst->m_type == SYM_TEMPORARY256);
431
432 auto dstLoAddrReg = GetNextTempRegister64();
433 auto dstHiAddrReg = GetNextTempRegister64();
434 auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
435 auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
436
437 LoadTemporary256ElementAddressInRegister(dstLoAddrReg, dst, 0x00);
438 LoadTemporary256ElementAddressInRegister(dstHiAddrReg, dst, 0x10);
439
440 m_assembler.St1_4s(src1Reg, dstLoAddrReg);
441 m_assembler.St1_4s(src2Reg, dstHiAddrReg);
442 }
443
Emit_Md_Srl256_VarMemCst(const STATEMENT & statement)444 void CCodeGen_AArch64::Emit_Md_Srl256_VarMemCst(const STATEMENT& statement)
445 {
446 auto dst = statement.dst->GetSymbol().get();
447 auto src1 = statement.src1->GetSymbol().get();
448 auto src2 = statement.src2->GetSymbol().get();
449
450 assert(src1->m_type == SYM_TEMPORARY256);
451 assert(src2->m_type == SYM_CONSTANT);
452
453 auto src1AddrReg = GetNextTempRegister64();
454 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
455
456 uint32 offset = (src2->m_valueLow & 0x7F) / 8;
457 LoadTemporary256ElementAddressInRegister(src1AddrReg, src1, offset);
458
459 m_assembler.Ld1_4s(dstReg, src1AddrReg);
460
461 CommitSymbolRegisterMd(dst, dstReg);
462 }
463
Emit_Md_Srl256_VarMemVar(const STATEMENT & statement)464 void CCodeGen_AArch64::Emit_Md_Srl256_VarMemVar(const STATEMENT& statement)
465 {
466 auto dst = statement.dst->GetSymbol().get();
467 auto src1 = statement.src1->GetSymbol().get();
468 auto src2 = statement.src2->GetSymbol().get();
469
470 assert(src1->m_type == SYM_TEMPORARY256);
471
472 auto offsetRegister = GetNextTempRegister();
473 auto src1AddrReg = GetNextTempRegister64();
474 auto src2Register = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
475
476 auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
477
478 LoadTemporary256ElementAddressInRegister(src1AddrReg, src1, 0);
479
480 //Compute offset and modify address
481 LOGICAL_IMM_PARAMS logicalImmParams;
482 bool result = TryGetLogicalImmParams(0x7F, logicalImmParams);
483 assert(result);
484 m_assembler.And(offsetRegister, src2Register, logicalImmParams.n, logicalImmParams.immr, logicalImmParams.imms);
485 m_assembler.Lsr(offsetRegister, offsetRegister, 3);
486 m_assembler.Add(src1AddrReg, src1AddrReg, static_cast<CAArch64Assembler::REGISTER64>(offsetRegister));
487
488 m_assembler.Ld1_4s(dstReg, src1AddrReg);
489
490 CommitSymbolRegisterMd(dst, dstReg);
491 }
492
493 CCodeGen_AArch64::CONSTMATCHER CCodeGen_AArch64::g_mdConstMatchers[] =
494 {
495 { OP_MD_ADD_B, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDB> },
496 { OP_MD_ADD_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDH> },
497 { OP_MD_ADD_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDW> },
498
499 { OP_MD_ADDUS_B, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDBUS> },
500 { OP_MD_ADDUS_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDHUS> },
501 { OP_MD_ADDUS_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDWUS> },
502
503 { OP_MD_ADDSS_B, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDBSS> },
504 { OP_MD_ADDSS_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDHSS> },
505 { OP_MD_ADDSS_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDWSS> },
506
507 { OP_MD_SUB_B, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBB> },
508 { OP_MD_SUB_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBH> },
509 { OP_MD_SUB_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBW> },
510
511 { OP_MD_SUBUS_B, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBBUS> },
512 { OP_MD_SUBUS_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBHUS> },
513 { OP_MD_SUBUS_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBWUS> },
514
515 { OP_MD_SUBSS_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBHSS> },
516 { OP_MD_SUBSS_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBWSS> },
517
518 { OP_MD_CMPEQ_B, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPEQB> },
519 { OP_MD_CMPEQ_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPEQH> },
520 { OP_MD_CMPEQ_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPEQW> },
521
522 { OP_MD_CMPGT_B, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPGTB> },
523 { OP_MD_CMPGT_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPGTH> },
524 { OP_MD_CMPGT_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPGTW> },
525
526 { OP_MD_MIN_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MINH> },
527 { OP_MD_MIN_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MINW> },
528
529 { OP_MD_MAX_H, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MAXH> },
530 { OP_MD_MAX_W, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MAXW> },
531
532 { OP_MD_ADD_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDS> },
533 { OP_MD_SUB_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBS> },
534 { OP_MD_MUL_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MULS> },
535 { OP_MD_DIV_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_DIVS> },
536
537 { OP_MD_ABS_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVar<MDOP_ABSS> },
538 { OP_MD_MIN_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MINS> },
539 { OP_MD_MAX_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MAXS> },
540
541 { OP_MD_CMPLT_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_CmpLtS_VarVarVar },
542 { OP_MD_CMPGT_S, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPGTS> },
543
544 { OP_MD_AND, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_AND> },
545 { OP_MD_OR, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_OR> },
546 { OP_MD_XOR, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_XOR> },
547 { OP_MD_NOT, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVar<MDOP_NOT> },
548
549 { OP_MD_SLLH, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SLLH> },
550 { OP_MD_SLLW, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SLLW> },
551
552 { OP_MD_SRLH, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SRLH> },
553 { OP_MD_SRLW, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SRLW> },
554
555 { OP_MD_SRAH, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SRAH> },
556 { OP_MD_SRAW, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SRAW> },
557
558 { OP_MD_SRL256, MATCH_VARIABLE128, MATCH_MEMORY256, MATCH_VARIABLE, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Srl256_VarMemVar },
559 { OP_MD_SRL256, MATCH_VARIABLE128, MATCH_MEMORY256, MATCH_CONSTANT, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Srl256_VarMemCst },
560
561 { OP_MD_MAKESZ, MATCH_VARIABLE, MATCH_VARIABLE128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_MakeSz_VarVar },
562
563 { OP_MD_TOSINGLE, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVar<MDOP_TOSINGLE> },
564 { OP_MD_TOWORD_TRUNCATE, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVar<MDOP_TOWORD> },
565
566 { OP_LOADFROMREF, MATCH_VARIABLE128, MATCH_VAR_REF, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_LoadFromRef_VarVar },
567 { OP_STOREATREF, MATCH_NIL, MATCH_VAR_REF, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_StoreAtRef_VarVar },
568
569 { OP_MD_MOV_MASKED, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_MovMasked_VarVarVar },
570
571 { OP_MD_EXPAND, MATCH_VARIABLE128, MATCH_REGISTER, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Expand_VarReg },
572 { OP_MD_EXPAND, MATCH_VARIABLE128, MATCH_MEMORY, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Expand_VarMem },
573 { OP_MD_EXPAND, MATCH_VARIABLE128, MATCH_CONSTANT, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Expand_VarCst },
574
575 { OP_MD_PACK_HB, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_PackHB_VarVarVar },
576 { OP_MD_PACK_WH, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_PackWH_VarVarVar },
577
578 { OP_MD_UNPACK_LOWER_BH, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_LOWER_BH> },
579 { OP_MD_UNPACK_LOWER_HW, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_LOWER_HW> },
580 { OP_MD_UNPACK_LOWER_WD, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_LOWER_WD> },
581
582 { OP_MD_UNPACK_UPPER_BH, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_UPPER_BH> },
583 { OP_MD_UNPACK_UPPER_HW, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_UPPER_HW> },
584 { OP_MD_UNPACK_UPPER_WD, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_UPPER_WD> },
585
586 { OP_MOV, MATCH_REGISTER128, MATCH_REGISTER128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Mov_RegReg },
587 { OP_MOV, MATCH_REGISTER128, MATCH_MEMORY128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Mov_RegMem },
588 { OP_MOV, MATCH_MEMORY128, MATCH_REGISTER128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Mov_MemReg },
589 { OP_MOV, MATCH_MEMORY128, MATCH_MEMORY128, MATCH_NIL, MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Mov_MemMem },
590
591 { OP_MERGETO256, MATCH_MEMORY256, MATCH_VARIABLE128, MATCH_VARIABLE128, MATCH_NIL, &CCodeGen_AArch64::Emit_MergeTo256_MemVarVar },
592
593 { OP_MOV, MATCH_NIL, MATCH_NIL, MATCH_NIL, MATCH_NIL, nullptr },
594 };
595