1 #include "Jitter_CodeGen_AArch64.h"
2 #include <stdexcept>
3 
4 using namespace Jitter;
5 
LoadMemory128InRegister(CAArch64Assembler::REGISTERMD dstReg,CSymbol * symbol)6 void CCodeGen_AArch64::LoadMemory128InRegister(CAArch64Assembler::REGISTERMD dstReg, CSymbol* symbol)
7 {
8 	switch(symbol->m_type)
9 	{
10 	case SYM_RELATIVE128:
11 		m_assembler.Ldr_1q(dstReg, g_baseRegister, symbol->m_valueLow);
12 		break;
13 	case SYM_TEMPORARY128:
14 		m_assembler.Ldr_1q(dstReg, CAArch64Assembler::xSP, symbol->m_stackLocation);
15 		break;
16 	default:
17 		assert(0);
18 		break;
19 	}
20 }
21 
StoreRegisterInMemory128(CSymbol * symbol,CAArch64Assembler::REGISTERMD srcReg)22 void CCodeGen_AArch64::StoreRegisterInMemory128(CSymbol* symbol, CAArch64Assembler::REGISTERMD srcReg)
23 {
24 	switch(symbol->m_type)
25 	{
26 	case SYM_RELATIVE128:
27 		m_assembler.Str_1q(srcReg, g_baseRegister, symbol->m_valueLow);
28 		break;
29 	case SYM_TEMPORARY128:
30 		m_assembler.Str_1q(srcReg, CAArch64Assembler::xSP, symbol->m_stackLocation);
31 		break;
32 	default:
33 		assert(0);
34 		break;
35 	}
36 }
37 
LoadMemory128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg,CSymbol * symbol,uint32 offset)38 void CCodeGen_AArch64::LoadMemory128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg, CSymbol* symbol, uint32 offset)
39 {
40 	switch(symbol->m_type)
41 	{
42 	case SYM_RELATIVE128:
43 		LoadRelative128AddressInRegister(dstReg, symbol, offset);
44 		break;
45 	case SYM_TEMPORARY128:
46 		LoadTemporary128AddressInRegister(dstReg, symbol, offset);
47 		break;
48 	default:
49 		assert(0);
50 		break;
51 	}
52 }
53 
LoadRelative128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg,CSymbol * symbol,uint32 offset)54 void CCodeGen_AArch64::LoadRelative128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg, CSymbol* symbol, uint32 offset)
55 {
56 	assert(symbol->m_type == SYM_RELATIVE128);
57 
58 	uint32 totalOffset = symbol->m_valueLow + offset;
59 	assert(totalOffset < 0x1000);
60 	m_assembler.Add(dstReg, g_baseRegister, totalOffset, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
61 }
62 
LoadTemporary128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg,CSymbol * symbol,uint32 offset)63 void CCodeGen_AArch64::LoadTemporary128AddressInRegister(CAArch64Assembler::REGISTER64 dstReg, CSymbol* symbol, uint32 offset)
64 {
65 	assert(symbol->m_type == SYM_TEMPORARY128);
66 
67 	uint32 totalOffset = symbol->m_stackLocation + offset;
68 	assert(totalOffset < 0x1000);
69 	m_assembler.Add(dstReg, CAArch64Assembler::xSP, totalOffset, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
70 }
71 
LoadTemporary256ElementAddressInRegister(CAArch64Assembler::REGISTER64 dstReg,CSymbol * symbol,uint32 offset)72 void CCodeGen_AArch64::LoadTemporary256ElementAddressInRegister(CAArch64Assembler::REGISTER64 dstReg, CSymbol* symbol, uint32 offset)
73 {
74 	assert(symbol->m_type == SYM_TEMPORARY256);
75 
76 	uint32 totalOffset = symbol->m_stackLocation + offset;
77 	assert(totalOffset < 0x1000);
78 	m_assembler.Add(dstReg, CAArch64Assembler::xSP, totalOffset, CAArch64Assembler::ADDSUB_IMM_SHIFT_LSL0);
79 }
80 
PrepareSymbolRegisterDefMd(CSymbol * symbol,CAArch64Assembler::REGISTERMD preferedRegister)81 CAArch64Assembler::REGISTERMD CCodeGen_AArch64::PrepareSymbolRegisterDefMd(CSymbol* symbol, CAArch64Assembler::REGISTERMD preferedRegister)
82 {
83 	switch(symbol->m_type)
84 	{
85 	case SYM_REGISTER128:
86 		assert(symbol->m_valueLow < MAX_MDREGISTERS);
87 		return g_registersMd[symbol->m_valueLow];
88 		break;
89 	case SYM_TEMPORARY128:
90 	case SYM_RELATIVE128:
91 		return preferedRegister;
92 		break;
93 	default:
94 		throw std::runtime_error("Invalid symbol type.");
95 		break;
96 	}
97 }
98 
PrepareSymbolRegisterUseMd(CSymbol * symbol,CAArch64Assembler::REGISTERMD preferedRegister)99 CAArch64Assembler::REGISTERMD CCodeGen_AArch64::PrepareSymbolRegisterUseMd(CSymbol* symbol, CAArch64Assembler::REGISTERMD preferedRegister)
100 {
101 	switch(symbol->m_type)
102 	{
103 	case SYM_REGISTER128:
104 		assert(symbol->m_valueLow < MAX_MDREGISTERS);
105 		return g_registersMd[symbol->m_valueLow];
106 		break;
107 	case SYM_TEMPORARY128:
108 	case SYM_RELATIVE128:
109 		LoadMemory128InRegister(preferedRegister, symbol);
110 		return preferedRegister;
111 		break;
112 	default:
113 		throw std::runtime_error("Invalid symbol type.");
114 		break;
115 	}
116 }
117 
CommitSymbolRegisterMd(CSymbol * symbol,CAArch64Assembler::REGISTERMD usedRegister)118 void CCodeGen_AArch64::CommitSymbolRegisterMd(CSymbol* symbol, CAArch64Assembler::REGISTERMD usedRegister)
119 {
120 	switch(symbol->m_type)
121 	{
122 	case SYM_REGISTER128:
123 		assert(usedRegister == g_registersMd[symbol->m_valueLow]);
124 		break;
125 	case SYM_TEMPORARY128:
126 	case SYM_RELATIVE128:
127 		StoreRegisterInMemory128(symbol, usedRegister);
128 		break;
129 	default:
130 		throw std::runtime_error("Invalid symbol type.");
131 		break;
132 	}
133 }
134 
135 template <typename MDOP>
Emit_Md_VarVar(const STATEMENT & statement)136 void CCodeGen_AArch64::Emit_Md_VarVar(const STATEMENT& statement)
137 {
138 	auto dst = statement.dst->GetSymbol().get();
139 	auto src1 = statement.src1->GetSymbol().get();
140 
141 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
142 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
143 
144 	((m_assembler).*(MDOP::OpReg()))(dstReg, src1Reg);
145 
146 	CommitSymbolRegisterMd(dst, dstReg);
147 }
148 
149 template <typename MDOP>
Emit_Md_VarVarVar(const STATEMENT & statement)150 void CCodeGen_AArch64::Emit_Md_VarVarVar(const STATEMENT& statement)
151 {
152 	auto dst = statement.dst->GetSymbol().get();
153 	auto src1 = statement.src1->GetSymbol().get();
154 	auto src2 = statement.src2->GetSymbol().get();
155 
156 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
157 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
158 	auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
159 
160 	((m_assembler).*(MDOP::OpReg()))(dstReg, src1Reg, src2Reg);
161 
162 	CommitSymbolRegisterMd(dst, dstReg);
163 }
164 
165 template <typename MDOP>
Emit_Md_VarVarVarRev(const STATEMENT & statement)166 void CCodeGen_AArch64::Emit_Md_VarVarVarRev(const STATEMENT& statement)
167 {
168 	auto dst = statement.dst->GetSymbol().get();
169 	auto src1 = statement.src1->GetSymbol().get();
170 	auto src2 = statement.src2->GetSymbol().get();
171 
172 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
173 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
174 	auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
175 
176 	((m_assembler).*(MDOP::OpReg()))(dstReg, src2Reg, src1Reg);
177 
178 	CommitSymbolRegisterMd(dst, dstReg);
179 }
180 
181 template <typename MDSHIFTOP>
Emit_Md_Shift_VarVarCst(const STATEMENT & statement)182 void CCodeGen_AArch64::Emit_Md_Shift_VarVarCst(const STATEMENT& statement)
183 {
184 	auto dst = statement.dst->GetSymbol().get();
185 	auto src1 = statement.src1->GetSymbol().get();
186 	auto src2 = statement.src2->GetSymbol().get();
187 
188 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
189 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
190 
191 	((m_assembler).*(MDSHIFTOP::OpReg()))(dstReg, src1Reg, src2->m_valueLow);
192 
193 	CommitSymbolRegisterMd(dst, dstReg);
194 }
195 
Emit_Md_MakeSz_VarVar(const STATEMENT & statement)196 void CCodeGen_AArch64::Emit_Md_MakeSz_VarVar(const STATEMENT& statement)
197 {
198 	auto dst = statement.dst->GetSymbol().get();
199 	auto src1 = statement.src1->GetSymbol().get();
200 
201 	m_nextTempRegisterMd = 0;
202 
203 	auto dstReg = PrepareSymbolRegisterDef(dst, GetNextTempRegister());
204 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
205 
206 	auto signReg = GetNextTempRegisterMd();
207 	auto zeroReg = GetNextTempRegisterMd();
208 	auto cstReg = GetNextTempRegisterMd();
209 
210 	assert(zeroReg == signReg + 1);
211 
212 	m_assembler.Cmltz_4s(signReg, src1Reg);
213 	m_assembler.Fcmeqz_4s(zeroReg, src1Reg);
214 
215 	LITERAL128 lit1(0x0004080C1014181CUL, 0xFFFFFFFFFFFFFFFFUL);
216 	LITERAL128 lit2(0x8040201008040201UL, 0x0000000000000000UL);
217 
218 	m_assembler.Ldr_Pc(cstReg, lit1);
219 	m_assembler.Tbl(signReg, signReg, cstReg);
220 	m_assembler.Ldr_Pc(cstReg, lit2);
221 	m_assembler.And_16b(signReg, signReg, cstReg);
222 	m_assembler.Uaddlv_16b(signReg, signReg);
223 	m_assembler.Umov_1s(dstReg, signReg, 0);
224 
225 	CommitSymbolRegister(dst, dstReg);
226 }
227 
Emit_Md_Mov_RegReg(const STATEMENT & statement)228 void CCodeGen_AArch64::Emit_Md_Mov_RegReg(const STATEMENT& statement)
229 {
230 	auto dst = statement.dst->GetSymbol().get();
231 	auto src1 = statement.src1->GetSymbol().get();
232 
233 	assert(!dst->Equals(src1));
234 
235 	m_assembler.Mov(g_registersMd[dst->m_valueLow], g_registersMd[src1->m_valueLow]);
236 }
237 
Emit_Md_Mov_RegMem(const STATEMENT & statement)238 void CCodeGen_AArch64::Emit_Md_Mov_RegMem(const STATEMENT& statement)
239 {
240 	auto dst = statement.dst->GetSymbol().get();
241 	auto src1 = statement.src1->GetSymbol().get();
242 
243 	LoadMemory128InRegister(g_registersMd[dst->m_valueLow], src1);
244 }
245 
Emit_Md_Mov_MemReg(const STATEMENT & statement)246 void CCodeGen_AArch64::Emit_Md_Mov_MemReg(const STATEMENT& statement)
247 {
248 	auto dst = statement.dst->GetSymbol().get();
249 	auto src1 = statement.src1->GetSymbol().get();
250 
251 	StoreRegisterInMemory128(dst, g_registersMd[src1->m_valueLow]);
252 }
253 
Emit_Md_Mov_MemMem(const STATEMENT & statement)254 void CCodeGen_AArch64::Emit_Md_Mov_MemMem(const STATEMENT& statement)
255 {
256 	auto dst = statement.dst->GetSymbol().get();
257 	auto src1 = statement.src1->GetSymbol().get();
258 
259 	auto tmpReg = GetNextTempRegisterMd();
260 
261 	LoadMemory128InRegister(tmpReg, src1);
262 	StoreRegisterInMemory128(dst, tmpReg);
263 }
264 
Emit_Md_CmpLtS_VarVarVar(const STATEMENT & statement)265 void CCodeGen_AArch64::Emit_Md_CmpLtS_VarVarVar(const STATEMENT& statement)
266 {
267 	auto dst = statement.dst->GetSymbol().get();
268 	auto src1 = statement.src1->GetSymbol().get();
269 	auto src2 = statement.src2->GetSymbol().get();
270 
271 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
272 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
273 	auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
274 
275 	m_assembler.Fcmge_4s(dstReg, src1Reg, src2Reg);
276 	m_assembler.Mvn_16b(dstReg, dstReg);
277 
278 	CommitSymbolRegisterMd(dst, dstReg);
279 }
280 
Emit_Md_LoadFromRef_VarVar(const STATEMENT & statement)281 void CCodeGen_AArch64::Emit_Md_LoadFromRef_VarVar(const STATEMENT& statement)
282 {
283 	auto dst = statement.dst->GetSymbol().get();
284 	auto src1 = statement.src1->GetSymbol().get();
285 
286 	auto src1AddrReg = PrepareSymbolRegisterUseRef(src1, GetNextTempRegister64());
287 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
288 
289 	m_assembler.Ldr_1q(dstReg, src1AddrReg, 0);
290 
291 	CommitSymbolRegisterMd(dst, dstReg);
292 }
293 
Emit_Md_StoreAtRef_VarVar(const STATEMENT & statement)294 void CCodeGen_AArch64::Emit_Md_StoreAtRef_VarVar(const STATEMENT& statement)
295 {
296 	auto src1 = statement.src1->GetSymbol().get();
297 	auto src2 = statement.src2->GetSymbol().get();
298 
299 	auto src1AddrReg = PrepareSymbolRegisterUseRef(src1, GetNextTempRegister64());
300 	auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
301 
302 	m_assembler.Str_1q(src2Reg, src1AddrReg, 0);
303 }
304 
Emit_Md_MovMasked_VarVarVar(const STATEMENT & statement)305 void CCodeGen_AArch64::Emit_Md_MovMasked_VarVarVar(const STATEMENT& statement)
306 {
307 	auto dst = statement.dst->GetSymbol().get();
308 	auto src1 = statement.src1->GetSymbol().get();
309 	auto src2 = statement.src2->GetSymbol().get();
310 
311 	assert(dst->Equals(src1));
312 
313 	auto mask = static_cast<uint8>(statement.jmpCondition);
314 
315 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
316 	auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
317 
318 	for(unsigned int i = 0; i < 4; i++)
319 	{
320 		if(mask & (1 << i))
321 		{
322 			m_assembler.Ins_1s(src1Reg, i, src2Reg, i);
323 		}
324 	}
325 
326 	//This is only valid if dst == src1
327 	CommitSymbolRegisterMd(dst, src1Reg);
328 }
329 
Emit_Md_Expand_VarReg(const STATEMENT & statement)330 void CCodeGen_AArch64::Emit_Md_Expand_VarReg(const STATEMENT& statement)
331 {
332 	auto dst = statement.dst->GetSymbol().get();
333 	auto src1 = statement.src1->GetSymbol().get();
334 
335 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
336 
337 	m_assembler.Dup_4s(dstReg, g_registers[src1->m_valueLow]);
338 
339 	CommitSymbolRegisterMd(dst, dstReg);
340 }
341 
Emit_Md_Expand_VarMem(const STATEMENT & statement)342 void CCodeGen_AArch64::Emit_Md_Expand_VarMem(const STATEMENT& statement)
343 {
344 	auto dst = statement.dst->GetSymbol().get();
345 	auto src1 = statement.src1->GetSymbol().get();
346 
347 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
348 	auto src1Reg = GetNextTempRegister();
349 
350 	LoadMemoryInRegister(src1Reg, src1);
351 
352 	m_assembler.Dup_4s(dstReg, src1Reg);
353 
354 	CommitSymbolRegisterMd(dst, dstReg);
355 }
356 
Emit_Md_Expand_VarCst(const STATEMENT & statement)357 void CCodeGen_AArch64::Emit_Md_Expand_VarCst(const STATEMENT& statement)
358 {
359 	auto dst = statement.dst->GetSymbol().get();
360 	auto src1 = statement.src1->GetSymbol().get();
361 
362 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
363 	auto src1Reg = GetNextTempRegister();
364 
365 	LoadConstantInRegister(src1Reg, src1->m_valueLow);
366 
367 	m_assembler.Dup_4s(dstReg, src1Reg);
368 
369 	CommitSymbolRegisterMd(dst, dstReg);
370 }
371 
Emit_Md_PackHB_VarVarVar(const STATEMENT & statement)372 void CCodeGen_AArch64::Emit_Md_PackHB_VarVarVar(const STATEMENT& statement)
373 {
374 	auto dst = statement.dst->GetSymbol().get();
375 	auto src1 = statement.src1->GetSymbol().get();
376 	auto src2 = statement.src2->GetSymbol().get();
377 
378 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
379 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
380 	auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
381 
382 	if(dstReg == src1Reg)
383 	{
384 		auto tmpReg = GetNextTempRegisterMd();
385 		m_assembler.Xtn1_8b(tmpReg, src2Reg);
386 		m_assembler.Xtn2_16b(tmpReg, src1Reg);
387 		m_assembler.Mov(dstReg, tmpReg);
388 	}
389 	else
390 	{
391 		m_assembler.Xtn1_8b(dstReg, src2Reg);
392 		m_assembler.Xtn2_16b(dstReg, src1Reg);
393 	}
394 
395 	CommitSymbolRegisterMd(dst, dstReg);
396 }
397 
Emit_Md_PackWH_VarVarVar(const STATEMENT & statement)398 void CCodeGen_AArch64::Emit_Md_PackWH_VarVarVar(const STATEMENT& statement)
399 {
400 	auto dst = statement.dst->GetSymbol().get();
401 	auto src1 = statement.src1->GetSymbol().get();
402 	auto src2 = statement.src2->GetSymbol().get();
403 
404 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
405 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
406 	auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
407 
408 	if(dstReg == src1Reg)
409 	{
410 		auto tmpReg = GetNextTempRegisterMd();
411 		m_assembler.Xtn1_4h(tmpReg, src2Reg);
412 		m_assembler.Xtn2_8h(tmpReg, src1Reg);
413 		m_assembler.Mov(dstReg, tmpReg);
414 	}
415 	else
416 	{
417 		m_assembler.Xtn1_4h(dstReg, src2Reg);
418 		m_assembler.Xtn2_8h(dstReg, src1Reg);
419 	}
420 
421 	CommitSymbolRegisterMd(dst, dstReg);
422 }
423 
Emit_MergeTo256_MemVarVar(const STATEMENT & statement)424 void CCodeGen_AArch64::Emit_MergeTo256_MemVarVar(const STATEMENT& statement)
425 {
426 	auto dst = statement.dst->GetSymbol().get();
427 	auto src1 = statement.src1->GetSymbol().get();
428 	auto src2 = statement.src2->GetSymbol().get();
429 
430 	assert(dst->m_type == SYM_TEMPORARY256);
431 
432 	auto dstLoAddrReg = GetNextTempRegister64();
433 	auto dstHiAddrReg = GetNextTempRegister64();
434 	auto src1Reg = PrepareSymbolRegisterUseMd(src1, GetNextTempRegisterMd());
435 	auto src2Reg = PrepareSymbolRegisterUseMd(src2, GetNextTempRegisterMd());
436 
437 	LoadTemporary256ElementAddressInRegister(dstLoAddrReg, dst, 0x00);
438 	LoadTemporary256ElementAddressInRegister(dstHiAddrReg, dst, 0x10);
439 
440 	m_assembler.St1_4s(src1Reg, dstLoAddrReg);
441 	m_assembler.St1_4s(src2Reg, dstHiAddrReg);
442 }
443 
Emit_Md_Srl256_VarMemCst(const STATEMENT & statement)444 void CCodeGen_AArch64::Emit_Md_Srl256_VarMemCst(const STATEMENT& statement)
445 {
446 	auto dst = statement.dst->GetSymbol().get();
447 	auto src1 = statement.src1->GetSymbol().get();
448 	auto src2 = statement.src2->GetSymbol().get();
449 
450 	assert(src1->m_type == SYM_TEMPORARY256);
451 	assert(src2->m_type == SYM_CONSTANT);
452 
453 	auto src1AddrReg = GetNextTempRegister64();
454 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
455 
456 	uint32 offset = (src2->m_valueLow & 0x7F) / 8;
457 	LoadTemporary256ElementAddressInRegister(src1AddrReg, src1, offset);
458 
459 	m_assembler.Ld1_4s(dstReg, src1AddrReg);
460 
461 	CommitSymbolRegisterMd(dst, dstReg);
462 }
463 
Emit_Md_Srl256_VarMemVar(const STATEMENT & statement)464 void CCodeGen_AArch64::Emit_Md_Srl256_VarMemVar(const STATEMENT& statement)
465 {
466 	auto dst = statement.dst->GetSymbol().get();
467 	auto src1 = statement.src1->GetSymbol().get();
468 	auto src2 = statement.src2->GetSymbol().get();
469 
470 	assert(src1->m_type == SYM_TEMPORARY256);
471 
472 	auto offsetRegister = GetNextTempRegister();
473 	auto src1AddrReg = GetNextTempRegister64();
474 	auto src2Register = PrepareSymbolRegisterUse(src2, GetNextTempRegister());
475 
476 	auto dstReg = PrepareSymbolRegisterDefMd(dst, GetNextTempRegisterMd());
477 
478 	LoadTemporary256ElementAddressInRegister(src1AddrReg, src1, 0);
479 
480 	//Compute offset and modify address
481 	LOGICAL_IMM_PARAMS logicalImmParams;
482 	bool result = TryGetLogicalImmParams(0x7F, logicalImmParams);
483 	assert(result);
484 	m_assembler.And(offsetRegister, src2Register, logicalImmParams.n, logicalImmParams.immr, logicalImmParams.imms);
485 	m_assembler.Lsr(offsetRegister, offsetRegister, 3);
486 	m_assembler.Add(src1AddrReg, src1AddrReg, static_cast<CAArch64Assembler::REGISTER64>(offsetRegister));
487 
488 	m_assembler.Ld1_4s(dstReg, src1AddrReg);
489 
490 	CommitSymbolRegisterMd(dst, dstReg);
491 }
492 
493 CCodeGen_AArch64::CONSTMATCHER CCodeGen_AArch64::g_mdConstMatchers[] =
494 {
495 	{ OP_MD_ADD_B,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDB>                  },
496 	{ OP_MD_ADD_H,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDH>                  },
497 	{ OP_MD_ADD_W,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDW>                  },
498 
499 	{ OP_MD_ADDUS_B,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDBUS>                },
500 	{ OP_MD_ADDUS_H,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDHUS>                },
501 	{ OP_MD_ADDUS_W,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDWUS>                },
502 
503 	{ OP_MD_ADDSS_B,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDBSS>                },
504 	{ OP_MD_ADDSS_H,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDHSS>                },
505 	{ OP_MD_ADDSS_W,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDWSS>                },
506 
507 	{ OP_MD_SUB_B,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBB>                  },
508 	{ OP_MD_SUB_H,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBH>                  },
509 	{ OP_MD_SUB_W,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBW>                  },
510 
511 	{ OP_MD_SUBUS_B,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBBUS>                },
512 	{ OP_MD_SUBUS_H,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBHUS>                },
513 	{ OP_MD_SUBUS_W,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBWUS>                },
514 
515 	{ OP_MD_SUBSS_H,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBHSS>                },
516 	{ OP_MD_SUBSS_W,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBWSS>                },
517 
518 	{ OP_MD_CMPEQ_B,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPEQB>                },
519 	{ OP_MD_CMPEQ_H,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPEQH>                },
520 	{ OP_MD_CMPEQ_W,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPEQW>                },
521 
522 	{ OP_MD_CMPGT_B,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPGTB>                },
523 	{ OP_MD_CMPGT_H,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPGTH>                },
524 	{ OP_MD_CMPGT_W,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPGTW>                },
525 
526 	{ OP_MD_MIN_H,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MINH>                  },
527 	{ OP_MD_MIN_W,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MINW>                  },
528 
529 	{ OP_MD_MAX_H,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MAXH>                  },
530 	{ OP_MD_MAX_W,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MAXW>                  },
531 
532 	{ OP_MD_ADD_S,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_ADDS>                  },
533 	{ OP_MD_SUB_S,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_SUBS>                  },
534 	{ OP_MD_MUL_S,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MULS>                  },
535 	{ OP_MD_DIV_S,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_DIVS>                  },
536 
537 	{ OP_MD_ABS_S,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVar<MDOP_ABSS>                     },
538 	{ OP_MD_MIN_S,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MINS>                  },
539 	{ OP_MD_MAX_S,              MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_MAXS>                  },
540 
541 	{ OP_MD_CMPLT_S,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_CmpLtS_VarVarVar                      },
542 	{ OP_MD_CMPGT_S,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_CMPGTS>                },
543 
544 	{ OP_MD_AND,                MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_AND>                   },
545 	{ OP_MD_OR,                 MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_OR>                    },
546 	{ OP_MD_XOR,                MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVar<MDOP_XOR>                   },
547 	{ OP_MD_NOT,                MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVar<MDOP_NOT>                      },
548 
549 	{ OP_MD_SLLH,               MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_CONSTANT,         MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SLLH>            },
550 	{ OP_MD_SLLW,               MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_CONSTANT,         MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SLLW>            },
551 
552 	{ OP_MD_SRLH,               MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_CONSTANT,         MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SRLH>            },
553 	{ OP_MD_SRLW,               MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_CONSTANT,         MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SRLW>            },
554 
555 	{ OP_MD_SRAH,               MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_CONSTANT,         MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SRAH>            },
556 	{ OP_MD_SRAW,               MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_CONSTANT,         MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Shift_VarVarCst<MDOP_SRAW>            },
557 
558 	{ OP_MD_SRL256,             MATCH_VARIABLE128,    MATCH_MEMORY256,      MATCH_VARIABLE,         MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Srl256_VarMemVar                      },
559 	{ OP_MD_SRL256,             MATCH_VARIABLE128,    MATCH_MEMORY256,      MATCH_CONSTANT,         MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Srl256_VarMemCst                      },
560 
561 	{ OP_MD_MAKESZ,             MATCH_VARIABLE,       MATCH_VARIABLE128,    MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_MakeSz_VarVar                         },
562 
563 	{ OP_MD_TOSINGLE,           MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVar<MDOP_TOSINGLE>                 },
564 	{ OP_MD_TOWORD_TRUNCATE,    MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVar<MDOP_TOWORD>                   },
565 
566 	{ OP_LOADFROMREF,           MATCH_VARIABLE128,    MATCH_VAR_REF,        MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_LoadFromRef_VarVar                    },
567 	{ OP_STOREATREF,            MATCH_NIL,            MATCH_VAR_REF,        MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_StoreAtRef_VarVar                     },
568 
569 	{ OP_MD_MOV_MASKED,         MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_MovMasked_VarVarVar                   },
570 
571 	{ OP_MD_EXPAND,             MATCH_VARIABLE128,    MATCH_REGISTER,       MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Expand_VarReg                         },
572 	{ OP_MD_EXPAND,             MATCH_VARIABLE128,    MATCH_MEMORY,         MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Expand_VarMem                         },
573 	{ OP_MD_EXPAND,             MATCH_VARIABLE128,    MATCH_CONSTANT,       MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Expand_VarCst                         },
574 
575 	{ OP_MD_PACK_HB,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_PackHB_VarVarVar                      },
576 	{ OP_MD_PACK_WH,            MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_PackWH_VarVarVar                      },
577 
578 	{ OP_MD_UNPACK_LOWER_BH,    MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_LOWER_BH>    },
579 	{ OP_MD_UNPACK_LOWER_HW,    MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_LOWER_HW>    },
580 	{ OP_MD_UNPACK_LOWER_WD,    MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_LOWER_WD>    },
581 
582 	{ OP_MD_UNPACK_UPPER_BH,    MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_UPPER_BH>    },
583 	{ OP_MD_UNPACK_UPPER_HW,    MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_UPPER_HW>    },
584 	{ OP_MD_UNPACK_UPPER_WD,    MATCH_VARIABLE128,    MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_Md_VarVarVarRev<MDOP_UNPACK_UPPER_WD>    },
585 
586 	{ OP_MOV,                   MATCH_REGISTER128,    MATCH_REGISTER128,    MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Mov_RegReg                            },
587 	{ OP_MOV,                   MATCH_REGISTER128,    MATCH_MEMORY128,      MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Mov_RegMem                            },
588 	{ OP_MOV,                   MATCH_MEMORY128,      MATCH_REGISTER128,    MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Mov_MemReg                            },
589 	{ OP_MOV,                   MATCH_MEMORY128,      MATCH_MEMORY128,      MATCH_NIL,              MATCH_NIL, &CCodeGen_AArch64::Emit_Md_Mov_MemMem                            },
590 
591 	{ OP_MERGETO256,            MATCH_MEMORY256,      MATCH_VARIABLE128,    MATCH_VARIABLE128,      MATCH_NIL, &CCodeGen_AArch64::Emit_MergeTo256_MemVarVar                     },
592 
593 	{ OP_MOV,                   MATCH_NIL,            MATCH_NIL,            MATCH_NIL,              MATCH_NIL, nullptr                                                          },
594 };
595