1/* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25#include "../tcg-ldst.c.inc" 26#include "../tcg-pool.c.inc" 27 28#ifdef CONFIG_DEBUG_TCG 29static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { 30#if TCG_TARGET_REG_BITS == 64 31 "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi", 32#else 33 "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi", 34#endif 35 "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", 36 "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7", 37#if TCG_TARGET_REG_BITS == 64 38 "%xmm8", "%xmm9", "%xmm10", "%xmm11", 39 "%xmm12", "%xmm13", "%xmm14", "%xmm15", 40#endif 41}; 42#endif 43 44static const int tcg_target_reg_alloc_order[] = { 45#if TCG_TARGET_REG_BITS == 64 46 TCG_REG_RBP, 47 TCG_REG_RBX, 48 TCG_REG_R12, 49 TCG_REG_R13, 50 TCG_REG_R14, 51 TCG_REG_R15, 52 TCG_REG_R10, 53 TCG_REG_R11, 54 TCG_REG_R9, 55 TCG_REG_R8, 56 TCG_REG_RCX, 57 TCG_REG_RDX, 58 TCG_REG_RSI, 59 TCG_REG_RDI, 60 TCG_REG_RAX, 61#else 62 TCG_REG_EBX, 63 TCG_REG_ESI, 64 TCG_REG_EDI, 65 TCG_REG_EBP, 66 TCG_REG_ECX, 67 TCG_REG_EDX, 68 TCG_REG_EAX, 69#endif 70 TCG_REG_XMM0, 71 TCG_REG_XMM1, 72 TCG_REG_XMM2, 73 TCG_REG_XMM3, 74 TCG_REG_XMM4, 75 TCG_REG_XMM5, 76#ifndef _WIN64 77 /* The Win64 ABI has xmm6-xmm15 as caller-saves, and we do not save 78 any of them. Therefore only allow xmm0-xmm5 to be allocated. */ 79 TCG_REG_XMM6, 80 TCG_REG_XMM7, 81#if TCG_TARGET_REG_BITS == 64 82 TCG_REG_XMM8, 83 TCG_REG_XMM9, 84 TCG_REG_XMM10, 85 TCG_REG_XMM11, 86 TCG_REG_XMM12, 87 TCG_REG_XMM13, 88 TCG_REG_XMM14, 89 TCG_REG_XMM15, 90#endif 91#endif 92}; 93 94#define TCG_TMP_VEC TCG_REG_XMM5 95 96static const int tcg_target_call_iarg_regs[] = { 97#if TCG_TARGET_REG_BITS == 64 98#if defined(_WIN64) 99 TCG_REG_RCX, 100 TCG_REG_RDX, 101#else 102 TCG_REG_RDI, 103 TCG_REG_RSI, 104 TCG_REG_RDX, 105 TCG_REG_RCX, 106#endif 107 TCG_REG_R8, 108 TCG_REG_R9, 109#else 110 /* 32 bit mode uses stack based calling convention (GCC default). */ 111#endif 112}; 113 114static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) 115{ 116 switch (kind) { 117 case TCG_CALL_RET_NORMAL: 118 tcg_debug_assert(slot >= 0 && slot <= 1); 119 return slot ? TCG_REG_EDX : TCG_REG_EAX; 120#ifdef _WIN64 121 case TCG_CALL_RET_BY_VEC: 122 tcg_debug_assert(slot == 0); 123 return TCG_REG_XMM0; 124#endif 125 default: 126 g_assert_not_reached(); 127 } 128} 129 130/* Constants we accept. */ 131#define TCG_CT_CONST_S32 0x100 132#define TCG_CT_CONST_U32 0x200 133#define TCG_CT_CONST_I32 0x400 134#define TCG_CT_CONST_WSZ 0x800 135 136/* Registers used with L constraint, which are the first argument 137 registers on x86_64, and two random call clobbered registers on 138 i386. */ 139#if TCG_TARGET_REG_BITS == 64 140# define TCG_REG_L0 tcg_target_call_iarg_regs[0] 141# define TCG_REG_L1 tcg_target_call_iarg_regs[1] 142#else 143# define TCG_REG_L0 TCG_REG_EAX 144# define TCG_REG_L1 TCG_REG_EDX 145#endif 146 147#define ALL_BYTEH_REGS 0x0000000fu 148#if TCG_TARGET_REG_BITS == 64 149# define ALL_GENERAL_REGS 0x0000ffffu 150# define ALL_VECTOR_REGS 0xffff0000u 151# define ALL_BYTEL_REGS ALL_GENERAL_REGS 152#else 153# define ALL_GENERAL_REGS 0x000000ffu 154# define ALL_VECTOR_REGS 0x00ff0000u 155# define ALL_BYTEL_REGS ALL_BYTEH_REGS 156#endif 157#ifdef CONFIG_SOFTMMU 158# define SOFTMMU_RESERVE_REGS ((1 << TCG_REG_L0) | (1 << TCG_REG_L1)) 159#else 160# define SOFTMMU_RESERVE_REGS 0 161#endif 162 163/* For 64-bit, we always know that CMOV is available. */ 164#if TCG_TARGET_REG_BITS == 64 165# define have_cmov true 166#else 167# define have_cmov (cpuinfo & CPUINFO_CMOV) 168#endif 169#define have_bmi2 (cpuinfo & CPUINFO_BMI2) 170#define have_lzcnt (cpuinfo & CPUINFO_LZCNT) 171 172static const tcg_insn_unit *tb_ret_addr; 173 174static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 175 intptr_t value, intptr_t addend) 176{ 177 value += addend; 178 switch(type) { 179 case R_386_PC32: 180 value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr); 181 if (value != (int32_t)value) { 182 return false; 183 } 184 /* FALLTHRU */ 185 case R_386_32: 186 tcg_patch32(code_ptr, value); 187 break; 188 case R_386_PC8: 189 value -= (uintptr_t)tcg_splitwx_to_rx(code_ptr); 190 if (value != (int8_t)value) { 191 return false; 192 } 193 tcg_patch8(code_ptr, value); 194 break; 195 default: 196 g_assert_not_reached(); 197 } 198 return true; 199} 200 201/* test if a constant matches the constraint */ 202static bool tcg_target_const_match(int64_t val, TCGType type, int ct) 203{ 204 if (ct & TCG_CT_CONST) { 205 return 1; 206 } 207 if (type == TCG_TYPE_I32) { 208 if (ct & (TCG_CT_CONST_S32 | TCG_CT_CONST_U32 | TCG_CT_CONST_I32)) { 209 return 1; 210 } 211 } else { 212 if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) { 213 return 1; 214 } 215 if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) { 216 return 1; 217 } 218 if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) { 219 return 1; 220 } 221 } 222 if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) { 223 return 1; 224 } 225 return 0; 226} 227 228# define LOWREGMASK(x) ((x) & 7) 229 230#define P_EXT 0x100 /* 0x0f opcode prefix */ 231#define P_EXT38 0x200 /* 0x0f 0x38 opcode prefix */ 232#define P_DATA16 0x400 /* 0x66 opcode prefix */ 233#define P_VEXW 0x1000 /* Set VEX.W = 1 */ 234#if TCG_TARGET_REG_BITS == 64 235# define P_REXW P_VEXW /* Set REX.W = 1; match VEXW */ 236# define P_REXB_R 0x2000 /* REG field as byte register */ 237# define P_REXB_RM 0x4000 /* R/M field as byte register */ 238# define P_GS 0x8000 /* gs segment override */ 239#else 240# define P_REXW 0 241# define P_REXB_R 0 242# define P_REXB_RM 0 243# define P_GS 0 244#endif 245#define P_EXT3A 0x10000 /* 0x0f 0x3a opcode prefix */ 246#define P_SIMDF3 0x20000 /* 0xf3 opcode prefix */ 247#define P_SIMDF2 0x40000 /* 0xf2 opcode prefix */ 248#define P_VEXL 0x80000 /* Set VEX.L = 1 */ 249#define P_EVEX 0x100000 /* Requires EVEX encoding */ 250 251#define OPC_ARITH_EvIz (0x81) 252#define OPC_ARITH_EvIb (0x83) 253#define OPC_ARITH_GvEv (0x03) /* ... plus (ARITH_FOO << 3) */ 254#define OPC_ANDN (0xf2 | P_EXT38) 255#define OPC_ADD_GvEv (OPC_ARITH_GvEv | (ARITH_ADD << 3)) 256#define OPC_AND_GvEv (OPC_ARITH_GvEv | (ARITH_AND << 3)) 257#define OPC_BLENDPS (0x0c | P_EXT3A | P_DATA16) 258#define OPC_BSF (0xbc | P_EXT) 259#define OPC_BSR (0xbd | P_EXT) 260#define OPC_BSWAP (0xc8 | P_EXT) 261#define OPC_CALL_Jz (0xe8) 262#define OPC_CMOVCC (0x40 | P_EXT) /* ... plus condition code */ 263#define OPC_CMP_GvEv (OPC_ARITH_GvEv | (ARITH_CMP << 3)) 264#define OPC_DEC_r32 (0x48) 265#define OPC_IMUL_GvEv (0xaf | P_EXT) 266#define OPC_IMUL_GvEvIb (0x6b) 267#define OPC_IMUL_GvEvIz (0x69) 268#define OPC_INC_r32 (0x40) 269#define OPC_JCC_long (0x80 | P_EXT) /* ... plus condition code */ 270#define OPC_JCC_short (0x70) /* ... plus condition code */ 271#define OPC_JMP_long (0xe9) 272#define OPC_JMP_short (0xeb) 273#define OPC_LEA (0x8d) 274#define OPC_LZCNT (0xbd | P_EXT | P_SIMDF3) 275#define OPC_MOVB_EvGv (0x88) /* stores, more or less */ 276#define OPC_MOVL_EvGv (0x89) /* stores, more or less */ 277#define OPC_MOVL_GvEv (0x8b) /* loads, more or less */ 278#define OPC_MOVB_EvIz (0xc6) 279#define OPC_MOVL_EvIz (0xc7) 280#define OPC_MOVL_Iv (0xb8) 281#define OPC_MOVBE_GyMy (0xf0 | P_EXT38) 282#define OPC_MOVBE_MyGy (0xf1 | P_EXT38) 283#define OPC_MOVD_VyEy (0x6e | P_EXT | P_DATA16) 284#define OPC_MOVD_EyVy (0x7e | P_EXT | P_DATA16) 285#define OPC_MOVDDUP (0x12 | P_EXT | P_SIMDF2) 286#define OPC_MOVDQA_VxWx (0x6f | P_EXT | P_DATA16) 287#define OPC_MOVDQA_WxVx (0x7f | P_EXT | P_DATA16) 288#define OPC_MOVDQU_VxWx (0x6f | P_EXT | P_SIMDF3) 289#define OPC_MOVDQU_WxVx (0x7f | P_EXT | P_SIMDF3) 290#define OPC_MOVQ_VqWq (0x7e | P_EXT | P_SIMDF3) 291#define OPC_MOVQ_WqVq (0xd6 | P_EXT | P_DATA16) 292#define OPC_MOVSBL (0xbe | P_EXT) 293#define OPC_MOVSWL (0xbf | P_EXT) 294#define OPC_MOVSLQ (0x63 | P_REXW) 295#define OPC_MOVZBL (0xb6 | P_EXT) 296#define OPC_MOVZWL (0xb7 | P_EXT) 297#define OPC_PABSB (0x1c | P_EXT38 | P_DATA16) 298#define OPC_PABSW (0x1d | P_EXT38 | P_DATA16) 299#define OPC_PABSD (0x1e | P_EXT38 | P_DATA16) 300#define OPC_VPABSQ (0x1f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 301#define OPC_PACKSSDW (0x6b | P_EXT | P_DATA16) 302#define OPC_PACKSSWB (0x63 | P_EXT | P_DATA16) 303#define OPC_PACKUSDW (0x2b | P_EXT38 | P_DATA16) 304#define OPC_PACKUSWB (0x67 | P_EXT | P_DATA16) 305#define OPC_PADDB (0xfc | P_EXT | P_DATA16) 306#define OPC_PADDW (0xfd | P_EXT | P_DATA16) 307#define OPC_PADDD (0xfe | P_EXT | P_DATA16) 308#define OPC_PADDQ (0xd4 | P_EXT | P_DATA16) 309#define OPC_PADDSB (0xec | P_EXT | P_DATA16) 310#define OPC_PADDSW (0xed | P_EXT | P_DATA16) 311#define OPC_PADDUB (0xdc | P_EXT | P_DATA16) 312#define OPC_PADDUW (0xdd | P_EXT | P_DATA16) 313#define OPC_PAND (0xdb | P_EXT | P_DATA16) 314#define OPC_PANDN (0xdf | P_EXT | P_DATA16) 315#define OPC_PBLENDW (0x0e | P_EXT3A | P_DATA16) 316#define OPC_PCMPEQB (0x74 | P_EXT | P_DATA16) 317#define OPC_PCMPEQW (0x75 | P_EXT | P_DATA16) 318#define OPC_PCMPEQD (0x76 | P_EXT | P_DATA16) 319#define OPC_PCMPEQQ (0x29 | P_EXT38 | P_DATA16) 320#define OPC_PCMPGTB (0x64 | P_EXT | P_DATA16) 321#define OPC_PCMPGTW (0x65 | P_EXT | P_DATA16) 322#define OPC_PCMPGTD (0x66 | P_EXT | P_DATA16) 323#define OPC_PCMPGTQ (0x37 | P_EXT38 | P_DATA16) 324#define OPC_PEXTRD (0x16 | P_EXT3A | P_DATA16) 325#define OPC_PINSRD (0x22 | P_EXT3A | P_DATA16) 326#define OPC_PMAXSB (0x3c | P_EXT38 | P_DATA16) 327#define OPC_PMAXSW (0xee | P_EXT | P_DATA16) 328#define OPC_PMAXSD (0x3d | P_EXT38 | P_DATA16) 329#define OPC_VPMAXSQ (0x3d | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 330#define OPC_PMAXUB (0xde | P_EXT | P_DATA16) 331#define OPC_PMAXUW (0x3e | P_EXT38 | P_DATA16) 332#define OPC_PMAXUD (0x3f | P_EXT38 | P_DATA16) 333#define OPC_VPMAXUQ (0x3f | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 334#define OPC_PMINSB (0x38 | P_EXT38 | P_DATA16) 335#define OPC_PMINSW (0xea | P_EXT | P_DATA16) 336#define OPC_PMINSD (0x39 | P_EXT38 | P_DATA16) 337#define OPC_VPMINSQ (0x39 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 338#define OPC_PMINUB (0xda | P_EXT | P_DATA16) 339#define OPC_PMINUW (0x3a | P_EXT38 | P_DATA16) 340#define OPC_PMINUD (0x3b | P_EXT38 | P_DATA16) 341#define OPC_VPMINUQ (0x3b | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 342#define OPC_PMOVSXBW (0x20 | P_EXT38 | P_DATA16) 343#define OPC_PMOVSXWD (0x23 | P_EXT38 | P_DATA16) 344#define OPC_PMOVSXDQ (0x25 | P_EXT38 | P_DATA16) 345#define OPC_PMOVZXBW (0x30 | P_EXT38 | P_DATA16) 346#define OPC_PMOVZXWD (0x33 | P_EXT38 | P_DATA16) 347#define OPC_PMOVZXDQ (0x35 | P_EXT38 | P_DATA16) 348#define OPC_PMULLW (0xd5 | P_EXT | P_DATA16) 349#define OPC_PMULLD (0x40 | P_EXT38 | P_DATA16) 350#define OPC_VPMULLQ (0x40 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 351#define OPC_POR (0xeb | P_EXT | P_DATA16) 352#define OPC_PSHUFB (0x00 | P_EXT38 | P_DATA16) 353#define OPC_PSHUFD (0x70 | P_EXT | P_DATA16) 354#define OPC_PSHUFLW (0x70 | P_EXT | P_SIMDF2) 355#define OPC_PSHUFHW (0x70 | P_EXT | P_SIMDF3) 356#define OPC_PSHIFTW_Ib (0x71 | P_EXT | P_DATA16) /* /2 /6 /4 */ 357#define OPC_PSHIFTD_Ib (0x72 | P_EXT | P_DATA16) /* /1 /2 /6 /4 */ 358#define OPC_PSHIFTQ_Ib (0x73 | P_EXT | P_DATA16) /* /2 /6 /4 */ 359#define OPC_PSLLW (0xf1 | P_EXT | P_DATA16) 360#define OPC_PSLLD (0xf2 | P_EXT | P_DATA16) 361#define OPC_PSLLQ (0xf3 | P_EXT | P_DATA16) 362#define OPC_PSRAW (0xe1 | P_EXT | P_DATA16) 363#define OPC_PSRAD (0xe2 | P_EXT | P_DATA16) 364#define OPC_VPSRAQ (0xe2 | P_EXT | P_DATA16 | P_VEXW | P_EVEX) 365#define OPC_PSRLW (0xd1 | P_EXT | P_DATA16) 366#define OPC_PSRLD (0xd2 | P_EXT | P_DATA16) 367#define OPC_PSRLQ (0xd3 | P_EXT | P_DATA16) 368#define OPC_PSUBB (0xf8 | P_EXT | P_DATA16) 369#define OPC_PSUBW (0xf9 | P_EXT | P_DATA16) 370#define OPC_PSUBD (0xfa | P_EXT | P_DATA16) 371#define OPC_PSUBQ (0xfb | P_EXT | P_DATA16) 372#define OPC_PSUBSB (0xe8 | P_EXT | P_DATA16) 373#define OPC_PSUBSW (0xe9 | P_EXT | P_DATA16) 374#define OPC_PSUBUB (0xd8 | P_EXT | P_DATA16) 375#define OPC_PSUBUW (0xd9 | P_EXT | P_DATA16) 376#define OPC_PUNPCKLBW (0x60 | P_EXT | P_DATA16) 377#define OPC_PUNPCKLWD (0x61 | P_EXT | P_DATA16) 378#define OPC_PUNPCKLDQ (0x62 | P_EXT | P_DATA16) 379#define OPC_PUNPCKLQDQ (0x6c | P_EXT | P_DATA16) 380#define OPC_PUNPCKHBW (0x68 | P_EXT | P_DATA16) 381#define OPC_PUNPCKHWD (0x69 | P_EXT | P_DATA16) 382#define OPC_PUNPCKHDQ (0x6a | P_EXT | P_DATA16) 383#define OPC_PUNPCKHQDQ (0x6d | P_EXT | P_DATA16) 384#define OPC_PXOR (0xef | P_EXT | P_DATA16) 385#define OPC_POP_r32 (0x58) 386#define OPC_POPCNT (0xb8 | P_EXT | P_SIMDF3) 387#define OPC_PUSH_r32 (0x50) 388#define OPC_PUSH_Iv (0x68) 389#define OPC_PUSH_Ib (0x6a) 390#define OPC_RET (0xc3) 391#define OPC_SETCC (0x90 | P_EXT | P_REXB_RM) /* ... plus cc */ 392#define OPC_SHIFT_1 (0xd1) 393#define OPC_SHIFT_Ib (0xc1) 394#define OPC_SHIFT_cl (0xd3) 395#define OPC_SARX (0xf7 | P_EXT38 | P_SIMDF3) 396#define OPC_SHUFPS (0xc6 | P_EXT) 397#define OPC_SHLX (0xf7 | P_EXT38 | P_DATA16) 398#define OPC_SHRX (0xf7 | P_EXT38 | P_SIMDF2) 399#define OPC_SHRD_Ib (0xac | P_EXT) 400#define OPC_TESTL (0x85) 401#define OPC_TZCNT (0xbc | P_EXT | P_SIMDF3) 402#define OPC_UD2 (0x0b | P_EXT) 403#define OPC_VPBLENDD (0x02 | P_EXT3A | P_DATA16) 404#define OPC_VPBLENDVB (0x4c | P_EXT3A | P_DATA16) 405#define OPC_VPINSRB (0x20 | P_EXT3A | P_DATA16) 406#define OPC_VPINSRW (0xc4 | P_EXT | P_DATA16) 407#define OPC_VBROADCASTSS (0x18 | P_EXT38 | P_DATA16) 408#define OPC_VBROADCASTSD (0x19 | P_EXT38 | P_DATA16) 409#define OPC_VPBROADCASTB (0x78 | P_EXT38 | P_DATA16) 410#define OPC_VPBROADCASTW (0x79 | P_EXT38 | P_DATA16) 411#define OPC_VPBROADCASTD (0x58 | P_EXT38 | P_DATA16) 412#define OPC_VPBROADCASTQ (0x59 | P_EXT38 | P_DATA16) 413#define OPC_VPERMQ (0x00 | P_EXT3A | P_DATA16 | P_VEXW) 414#define OPC_VPERM2I128 (0x46 | P_EXT3A | P_DATA16 | P_VEXL) 415#define OPC_VPROLVD (0x15 | P_EXT38 | P_DATA16 | P_EVEX) 416#define OPC_VPROLVQ (0x15 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 417#define OPC_VPRORVD (0x14 | P_EXT38 | P_DATA16 | P_EVEX) 418#define OPC_VPRORVQ (0x14 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 419#define OPC_VPSHLDW (0x70 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) 420#define OPC_VPSHLDD (0x71 | P_EXT3A | P_DATA16 | P_EVEX) 421#define OPC_VPSHLDQ (0x71 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) 422#define OPC_VPSHLDVW (0x70 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 423#define OPC_VPSHLDVD (0x71 | P_EXT38 | P_DATA16 | P_EVEX) 424#define OPC_VPSHLDVQ (0x71 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 425#define OPC_VPSHRDVW (0x72 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 426#define OPC_VPSHRDVD (0x73 | P_EXT38 | P_DATA16 | P_EVEX) 427#define OPC_VPSHRDVQ (0x73 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 428#define OPC_VPSLLVW (0x12 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 429#define OPC_VPSLLVD (0x47 | P_EXT38 | P_DATA16) 430#define OPC_VPSLLVQ (0x47 | P_EXT38 | P_DATA16 | P_VEXW) 431#define OPC_VPSRAVW (0x11 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 432#define OPC_VPSRAVD (0x46 | P_EXT38 | P_DATA16) 433#define OPC_VPSRAVQ (0x46 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 434#define OPC_VPSRLVW (0x10 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) 435#define OPC_VPSRLVD (0x45 | P_EXT38 | P_DATA16) 436#define OPC_VPSRLVQ (0x45 | P_EXT38 | P_DATA16 | P_VEXW) 437#define OPC_VPTERNLOGQ (0x25 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) 438#define OPC_VZEROUPPER (0x77 | P_EXT) 439#define OPC_XCHG_ax_r32 (0x90) 440#define OPC_XCHG_EvGv (0x87) 441 442#define OPC_GRP3_Eb (0xf6) 443#define OPC_GRP3_Ev (0xf7) 444#define OPC_GRP5 (0xff) 445#define OPC_GRP14 (0x73 | P_EXT | P_DATA16) 446 447/* Group 1 opcode extensions for 0x80-0x83. 448 These are also used as modifiers for OPC_ARITH. */ 449#define ARITH_ADD 0 450#define ARITH_OR 1 451#define ARITH_ADC 2 452#define ARITH_SBB 3 453#define ARITH_AND 4 454#define ARITH_SUB 5 455#define ARITH_XOR 6 456#define ARITH_CMP 7 457 458/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3. */ 459#define SHIFT_ROL 0 460#define SHIFT_ROR 1 461#define SHIFT_SHL 4 462#define SHIFT_SHR 5 463#define SHIFT_SAR 7 464 465/* Group 3 opcode extensions for 0xf6, 0xf7. To be used with OPC_GRP3. */ 466#define EXT3_TESTi 0 467#define EXT3_NOT 2 468#define EXT3_NEG 3 469#define EXT3_MUL 4 470#define EXT3_IMUL 5 471#define EXT3_DIV 6 472#define EXT3_IDIV 7 473 474/* Group 5 opcode extensions for 0xff. To be used with OPC_GRP5. */ 475#define EXT5_INC_Ev 0 476#define EXT5_DEC_Ev 1 477#define EXT5_CALLN_Ev 2 478#define EXT5_JMPN_Ev 4 479 480/* Condition codes to be added to OPC_JCC_{long,short}. */ 481#define JCC_JMP (-1) 482#define JCC_JO 0x0 483#define JCC_JNO 0x1 484#define JCC_JB 0x2 485#define JCC_JAE 0x3 486#define JCC_JE 0x4 487#define JCC_JNE 0x5 488#define JCC_JBE 0x6 489#define JCC_JA 0x7 490#define JCC_JS 0x8 491#define JCC_JNS 0x9 492#define JCC_JP 0xa 493#define JCC_JNP 0xb 494#define JCC_JL 0xc 495#define JCC_JGE 0xd 496#define JCC_JLE 0xe 497#define JCC_JG 0xf 498 499static const uint8_t tcg_cond_to_jcc[] = { 500 [TCG_COND_EQ] = JCC_JE, 501 [TCG_COND_NE] = JCC_JNE, 502 [TCG_COND_LT] = JCC_JL, 503 [TCG_COND_GE] = JCC_JGE, 504 [TCG_COND_LE] = JCC_JLE, 505 [TCG_COND_GT] = JCC_JG, 506 [TCG_COND_LTU] = JCC_JB, 507 [TCG_COND_GEU] = JCC_JAE, 508 [TCG_COND_LEU] = JCC_JBE, 509 [TCG_COND_GTU] = JCC_JA, 510}; 511 512#if TCG_TARGET_REG_BITS == 64 513static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x) 514{ 515 int rex; 516 517 if (opc & P_GS) { 518 tcg_out8(s, 0x65); 519 } 520 if (opc & P_DATA16) { 521 /* We should never be asking for both 16 and 64-bit operation. */ 522 tcg_debug_assert((opc & P_REXW) == 0); 523 tcg_out8(s, 0x66); 524 } 525 if (opc & P_SIMDF3) { 526 tcg_out8(s, 0xf3); 527 } else if (opc & P_SIMDF2) { 528 tcg_out8(s, 0xf2); 529 } 530 531 rex = 0; 532 rex |= (opc & P_REXW) ? 0x8 : 0x0; /* REX.W */ 533 rex |= (r & 8) >> 1; /* REX.R */ 534 rex |= (x & 8) >> 2; /* REX.X */ 535 rex |= (rm & 8) >> 3; /* REX.B */ 536 537 /* P_REXB_{R,RM} indicates that the given register is the low byte. 538 For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do, 539 as otherwise the encoding indicates %[abcd]h. Note that the values 540 that are ORed in merely indicate that the REX byte must be present; 541 those bits get discarded in output. */ 542 rex |= opc & (r >= 4 ? P_REXB_R : 0); 543 rex |= opc & (rm >= 4 ? P_REXB_RM : 0); 544 545 if (rex) { 546 tcg_out8(s, (uint8_t)(rex | 0x40)); 547 } 548 549 if (opc & (P_EXT | P_EXT38 | P_EXT3A)) { 550 tcg_out8(s, 0x0f); 551 if (opc & P_EXT38) { 552 tcg_out8(s, 0x38); 553 } else if (opc & P_EXT3A) { 554 tcg_out8(s, 0x3a); 555 } 556 } 557 558 tcg_out8(s, opc); 559} 560#else 561static void tcg_out_opc(TCGContext *s, int opc) 562{ 563 if (opc & P_DATA16) { 564 tcg_out8(s, 0x66); 565 } 566 if (opc & P_SIMDF3) { 567 tcg_out8(s, 0xf3); 568 } else if (opc & P_SIMDF2) { 569 tcg_out8(s, 0xf2); 570 } 571 if (opc & (P_EXT | P_EXT38 | P_EXT3A)) { 572 tcg_out8(s, 0x0f); 573 if (opc & P_EXT38) { 574 tcg_out8(s, 0x38); 575 } else if (opc & P_EXT3A) { 576 tcg_out8(s, 0x3a); 577 } 578 } 579 tcg_out8(s, opc); 580} 581/* Discard the register arguments to tcg_out_opc early, so as not to penalize 582 the 32-bit compilation paths. This method works with all versions of gcc, 583 whereas relying on optimization may not be able to exclude them. */ 584#define tcg_out_opc(s, opc, r, rm, x) (tcg_out_opc)(s, opc) 585#endif 586 587static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm) 588{ 589 tcg_out_opc(s, opc, r, rm, 0); 590 tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); 591} 592 593static void tcg_out_vex_opc(TCGContext *s, int opc, int r, int v, 594 int rm, int index) 595{ 596 int tmp; 597 598 /* Use the two byte form if possible, which cannot encode 599 VEX.W, VEX.B, VEX.X, or an m-mmmm field other than P_EXT. */ 600 if ((opc & (P_EXT | P_EXT38 | P_EXT3A | P_VEXW)) == P_EXT 601 && ((rm | index) & 8) == 0) { 602 /* Two byte VEX prefix. */ 603 tcg_out8(s, 0xc5); 604 605 tmp = (r & 8 ? 0 : 0x80); /* VEX.R */ 606 } else { 607 /* Three byte VEX prefix. */ 608 tcg_out8(s, 0xc4); 609 610 /* VEX.m-mmmm */ 611 if (opc & P_EXT3A) { 612 tmp = 3; 613 } else if (opc & P_EXT38) { 614 tmp = 2; 615 } else if (opc & P_EXT) { 616 tmp = 1; 617 } else { 618 g_assert_not_reached(); 619 } 620 tmp |= (r & 8 ? 0 : 0x80); /* VEX.R */ 621 tmp |= (index & 8 ? 0 : 0x40); /* VEX.X */ 622 tmp |= (rm & 8 ? 0 : 0x20); /* VEX.B */ 623 tcg_out8(s, tmp); 624 625 tmp = (opc & P_VEXW ? 0x80 : 0); /* VEX.W */ 626 } 627 628 tmp |= (opc & P_VEXL ? 0x04 : 0); /* VEX.L */ 629 /* VEX.pp */ 630 if (opc & P_DATA16) { 631 tmp |= 1; /* 0x66 */ 632 } else if (opc & P_SIMDF3) { 633 tmp |= 2; /* 0xf3 */ 634 } else if (opc & P_SIMDF2) { 635 tmp |= 3; /* 0xf2 */ 636 } 637 tmp |= (~v & 15) << 3; /* VEX.vvvv */ 638 tcg_out8(s, tmp); 639 tcg_out8(s, opc); 640} 641 642static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v, 643 int rm, int index) 644{ 645 /* The entire 4-byte evex prefix; with R' and V' set. */ 646 uint32_t p = 0x08041062; 647 int mm, pp; 648 649 tcg_debug_assert(have_avx512vl); 650 651 /* EVEX.mm */ 652 if (opc & P_EXT3A) { 653 mm = 3; 654 } else if (opc & P_EXT38) { 655 mm = 2; 656 } else if (opc & P_EXT) { 657 mm = 1; 658 } else { 659 g_assert_not_reached(); 660 } 661 662 /* EVEX.pp */ 663 if (opc & P_DATA16) { 664 pp = 1; /* 0x66 */ 665 } else if (opc & P_SIMDF3) { 666 pp = 2; /* 0xf3 */ 667 } else if (opc & P_SIMDF2) { 668 pp = 3; /* 0xf2 */ 669 } else { 670 pp = 0; 671 } 672 673 p = deposit32(p, 8, 2, mm); 674 p = deposit32(p, 13, 1, (rm & 8) == 0); /* EVEX.RXB.B */ 675 p = deposit32(p, 14, 1, (index & 8) == 0); /* EVEX.RXB.X */ 676 p = deposit32(p, 15, 1, (r & 8) == 0); /* EVEX.RXB.R */ 677 p = deposit32(p, 16, 2, pp); 678 p = deposit32(p, 19, 4, ~v); 679 p = deposit32(p, 23, 1, (opc & P_VEXW) != 0); 680 p = deposit32(p, 29, 2, (opc & P_VEXL) != 0); 681 682 tcg_out32(s, p); 683 tcg_out8(s, opc); 684} 685 686static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm) 687{ 688 if (opc & P_EVEX) { 689 tcg_out_evex_opc(s, opc, r, v, rm, 0); 690 } else { 691 tcg_out_vex_opc(s, opc, r, v, rm, 0); 692 } 693 tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); 694} 695 696/* Output an opcode with a full "rm + (index<<shift) + offset" address mode. 697 We handle either RM and INDEX missing with a negative value. In 64-bit 698 mode for absolute addresses, ~RM is the size of the immediate operand 699 that will follow the instruction. */ 700 701static void tcg_out_sib_offset(TCGContext *s, int r, int rm, int index, 702 int shift, intptr_t offset) 703{ 704 int mod, len; 705 706 if (index < 0 && rm < 0) { 707 if (TCG_TARGET_REG_BITS == 64) { 708 /* Try for a rip-relative addressing mode. This has replaced 709 the 32-bit-mode absolute addressing encoding. */ 710 intptr_t pc = (intptr_t)s->code_ptr + 5 + ~rm; 711 intptr_t disp = offset - pc; 712 if (disp == (int32_t)disp) { 713 tcg_out8(s, (LOWREGMASK(r) << 3) | 5); 714 tcg_out32(s, disp); 715 return; 716 } 717 718 /* Try for an absolute address encoding. This requires the 719 use of the MODRM+SIB encoding and is therefore larger than 720 rip-relative addressing. */ 721 if (offset == (int32_t)offset) { 722 tcg_out8(s, (LOWREGMASK(r) << 3) | 4); 723 tcg_out8(s, (4 << 3) | 5); 724 tcg_out32(s, offset); 725 return; 726 } 727 728 /* ??? The memory isn't directly addressable. */ 729 g_assert_not_reached(); 730 } else { 731 /* Absolute address. */ 732 tcg_out8(s, (r << 3) | 5); 733 tcg_out32(s, offset); 734 return; 735 } 736 } 737 738 /* Find the length of the immediate addend. Note that the encoding 739 that would be used for (%ebp) indicates absolute addressing. */ 740 if (rm < 0) { 741 mod = 0, len = 4, rm = 5; 742 } else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) { 743 mod = 0, len = 0; 744 } else if (offset == (int8_t)offset) { 745 mod = 0x40, len = 1; 746 } else { 747 mod = 0x80, len = 4; 748 } 749 750 /* Use a single byte MODRM format if possible. Note that the encoding 751 that would be used for %esp is the escape to the two byte form. */ 752 if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) { 753 /* Single byte MODRM format. */ 754 tcg_out8(s, mod | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); 755 } else { 756 /* Two byte MODRM+SIB format. */ 757 758 /* Note that the encoding that would place %esp into the index 759 field indicates no index register. In 64-bit mode, the REX.X 760 bit counts, so %r12 can be used as the index. */ 761 if (index < 0) { 762 index = 4; 763 } else { 764 tcg_debug_assert(index != TCG_REG_ESP); 765 } 766 767 tcg_out8(s, mod | (LOWREGMASK(r) << 3) | 4); 768 tcg_out8(s, (shift << 6) | (LOWREGMASK(index) << 3) | LOWREGMASK(rm)); 769 } 770 771 if (len == 1) { 772 tcg_out8(s, offset); 773 } else if (len == 4) { 774 tcg_out32(s, offset); 775 } 776} 777 778static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm, 779 int index, int shift, intptr_t offset) 780{ 781 tcg_out_opc(s, opc, r, rm < 0 ? 0 : rm, index < 0 ? 0 : index); 782 tcg_out_sib_offset(s, r, rm, index, shift, offset); 783} 784 785static void tcg_out_vex_modrm_sib_offset(TCGContext *s, int opc, int r, int v, 786 int rm, int index, int shift, 787 intptr_t offset) 788{ 789 tcg_out_vex_opc(s, opc, r, v, rm < 0 ? 0 : rm, index < 0 ? 0 : index); 790 tcg_out_sib_offset(s, r, rm, index, shift, offset); 791} 792 793/* A simplification of the above with no index or shift. */ 794static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r, 795 int rm, intptr_t offset) 796{ 797 tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset); 798} 799 800static inline void tcg_out_vex_modrm_offset(TCGContext *s, int opc, int r, 801 int v, int rm, intptr_t offset) 802{ 803 tcg_out_vex_modrm_sib_offset(s, opc, r, v, rm, -1, 0, offset); 804} 805 806/* Output an opcode with an expected reference to the constant pool. */ 807static inline void tcg_out_modrm_pool(TCGContext *s, int opc, int r) 808{ 809 tcg_out_opc(s, opc, r, 0, 0); 810 /* Absolute for 32-bit, pc-relative for 64-bit. */ 811 tcg_out8(s, LOWREGMASK(r) << 3 | 5); 812 tcg_out32(s, 0); 813} 814 815/* Output an opcode with an expected reference to the constant pool. */ 816static inline void tcg_out_vex_modrm_pool(TCGContext *s, int opc, int r) 817{ 818 tcg_out_vex_opc(s, opc, r, 0, 0, 0); 819 /* Absolute for 32-bit, pc-relative for 64-bit. */ 820 tcg_out8(s, LOWREGMASK(r) << 3 | 5); 821 tcg_out32(s, 0); 822} 823 824/* Generate dest op= src. Uses the same ARITH_* codes as tgen_arithi. */ 825static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src) 826{ 827 /* Propagate an opcode prefix, such as P_REXW. */ 828 int ext = subop & ~0x7; 829 subop &= 0x7; 830 831 tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src); 832} 833 834static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 835{ 836 int rexw = 0; 837 838 if (arg == ret) { 839 return true; 840 } 841 switch (type) { 842 case TCG_TYPE_I64: 843 rexw = P_REXW; 844 /* fallthru */ 845 case TCG_TYPE_I32: 846 if (ret < 16) { 847 if (arg < 16) { 848 tcg_out_modrm(s, OPC_MOVL_GvEv + rexw, ret, arg); 849 } else { 850 tcg_out_vex_modrm(s, OPC_MOVD_EyVy + rexw, arg, 0, ret); 851 } 852 } else { 853 if (arg < 16) { 854 tcg_out_vex_modrm(s, OPC_MOVD_VyEy + rexw, ret, 0, arg); 855 } else { 856 tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg); 857 } 858 } 859 break; 860 861 case TCG_TYPE_V64: 862 tcg_debug_assert(ret >= 16 && arg >= 16); 863 tcg_out_vex_modrm(s, OPC_MOVQ_VqWq, ret, 0, arg); 864 break; 865 case TCG_TYPE_V128: 866 tcg_debug_assert(ret >= 16 && arg >= 16); 867 tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx, ret, 0, arg); 868 break; 869 case TCG_TYPE_V256: 870 tcg_debug_assert(ret >= 16 && arg >= 16); 871 tcg_out_vex_modrm(s, OPC_MOVDQA_VxWx | P_VEXL, ret, 0, arg); 872 break; 873 874 default: 875 g_assert_not_reached(); 876 } 877 return true; 878} 879 880static const int avx2_dup_insn[4] = { 881 OPC_VPBROADCASTB, OPC_VPBROADCASTW, 882 OPC_VPBROADCASTD, OPC_VPBROADCASTQ, 883}; 884 885static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 886 TCGReg r, TCGReg a) 887{ 888 if (have_avx2) { 889 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0); 890 tcg_out_vex_modrm(s, avx2_dup_insn[vece] + vex_l, r, 0, a); 891 } else { 892 switch (vece) { 893 case MO_8: 894 /* ??? With zero in a register, use PSHUFB. */ 895 tcg_out_vex_modrm(s, OPC_PUNPCKLBW, r, a, a); 896 a = r; 897 /* FALLTHRU */ 898 case MO_16: 899 tcg_out_vex_modrm(s, OPC_PUNPCKLWD, r, a, a); 900 a = r; 901 /* FALLTHRU */ 902 case MO_32: 903 tcg_out_vex_modrm(s, OPC_PSHUFD, r, 0, a); 904 /* imm8 operand: all output lanes selected from input lane 0. */ 905 tcg_out8(s, 0); 906 break; 907 case MO_64: 908 tcg_out_vex_modrm(s, OPC_PUNPCKLQDQ, r, a, a); 909 break; 910 default: 911 g_assert_not_reached(); 912 } 913 } 914 return true; 915} 916 917static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 918 TCGReg r, TCGReg base, intptr_t offset) 919{ 920 if (have_avx2) { 921 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0); 922 tcg_out_vex_modrm_offset(s, avx2_dup_insn[vece] + vex_l, 923 r, 0, base, offset); 924 } else { 925 switch (vece) { 926 case MO_64: 927 tcg_out_vex_modrm_offset(s, OPC_MOVDDUP, r, 0, base, offset); 928 break; 929 case MO_32: 930 tcg_out_vex_modrm_offset(s, OPC_VBROADCASTSS, r, 0, base, offset); 931 break; 932 case MO_16: 933 tcg_out_vex_modrm_offset(s, OPC_VPINSRW, r, r, base, offset); 934 tcg_out8(s, 0); /* imm8 */ 935 tcg_out_dup_vec(s, type, vece, r, r); 936 break; 937 case MO_8: 938 tcg_out_vex_modrm_offset(s, OPC_VPINSRB, r, r, base, offset); 939 tcg_out8(s, 0); /* imm8 */ 940 tcg_out_dup_vec(s, type, vece, r, r); 941 break; 942 default: 943 g_assert_not_reached(); 944 } 945 } 946 return true; 947} 948 949static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 950 TCGReg ret, int64_t arg) 951{ 952 int vex_l = (type == TCG_TYPE_V256 ? P_VEXL : 0); 953 954 if (arg == 0) { 955 tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret); 956 return; 957 } 958 if (arg == -1) { 959 tcg_out_vex_modrm(s, OPC_PCMPEQB + vex_l, ret, ret, ret); 960 return; 961 } 962 963 if (TCG_TARGET_REG_BITS == 32 && vece < MO_64) { 964 if (have_avx2) { 965 tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTD + vex_l, ret); 966 } else { 967 tcg_out_vex_modrm_pool(s, OPC_VBROADCASTSS, ret); 968 } 969 new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0); 970 } else { 971 if (type == TCG_TYPE_V64) { 972 tcg_out_vex_modrm_pool(s, OPC_MOVQ_VqWq, ret); 973 } else if (have_avx2) { 974 tcg_out_vex_modrm_pool(s, OPC_VPBROADCASTQ + vex_l, ret); 975 } else { 976 tcg_out_vex_modrm_pool(s, OPC_MOVDDUP, ret); 977 } 978 if (TCG_TARGET_REG_BITS == 64) { 979 new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4); 980 } else { 981 new_pool_l2(s, R_386_32, s->code_ptr - 4, 0, arg, arg >> 32); 982 } 983 } 984} 985 986static void tcg_out_movi_vec(TCGContext *s, TCGType type, 987 TCGReg ret, tcg_target_long arg) 988{ 989 if (arg == 0) { 990 tcg_out_vex_modrm(s, OPC_PXOR, ret, ret, ret); 991 return; 992 } 993 if (arg == -1) { 994 tcg_out_vex_modrm(s, OPC_PCMPEQB, ret, ret, ret); 995 return; 996 } 997 998 int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW); 999 tcg_out_vex_modrm_pool(s, OPC_MOVD_VyEy + rexw, ret); 1000 if (TCG_TARGET_REG_BITS == 64) { 1001 new_pool_label(s, arg, R_386_PC32, s->code_ptr - 4, -4); 1002 } else { 1003 new_pool_label(s, arg, R_386_32, s->code_ptr - 4, 0); 1004 } 1005} 1006 1007static void tcg_out_movi_int(TCGContext *s, TCGType type, 1008 TCGReg ret, tcg_target_long arg) 1009{ 1010 tcg_target_long diff; 1011 1012 if (arg == 0) { 1013 tgen_arithr(s, ARITH_XOR, ret, ret); 1014 return; 1015 } 1016 if (arg == (uint32_t)arg || type == TCG_TYPE_I32) { 1017 tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0); 1018 tcg_out32(s, arg); 1019 return; 1020 } 1021 if (arg == (int32_t)arg) { 1022 tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret); 1023 tcg_out32(s, arg); 1024 return; 1025 } 1026 1027 /* Try a 7 byte pc-relative lea before the 10 byte movq. */ 1028 diff = tcg_pcrel_diff(s, (const void *)arg) - 7; 1029 if (diff == (int32_t)diff) { 1030 tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0); 1031 tcg_out8(s, (LOWREGMASK(ret) << 3) | 5); 1032 tcg_out32(s, diff); 1033 return; 1034 } 1035 1036 tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0); 1037 tcg_out64(s, arg); 1038} 1039 1040static void tcg_out_movi(TCGContext *s, TCGType type, 1041 TCGReg ret, tcg_target_long arg) 1042{ 1043 switch (type) { 1044 case TCG_TYPE_I32: 1045#if TCG_TARGET_REG_BITS == 64 1046 case TCG_TYPE_I64: 1047#endif 1048 if (ret < 16) { 1049 tcg_out_movi_int(s, type, ret, arg); 1050 } else { 1051 tcg_out_movi_vec(s, type, ret, arg); 1052 } 1053 break; 1054 default: 1055 g_assert_not_reached(); 1056 } 1057} 1058 1059static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2) 1060{ 1061 int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; 1062 tcg_out_modrm(s, OPC_XCHG_EvGv + rexw, r1, r2); 1063 return true; 1064} 1065 1066static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, 1067 tcg_target_long imm) 1068{ 1069 /* This function is only used for passing structs by reference. */ 1070 tcg_debug_assert(imm == (int32_t)imm); 1071 tcg_out_modrm_offset(s, OPC_LEA | P_REXW, rd, rs, imm); 1072} 1073 1074static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val) 1075{ 1076 if (val == (int8_t)val) { 1077 tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0); 1078 tcg_out8(s, val); 1079 } else if (val == (int32_t)val) { 1080 tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0); 1081 tcg_out32(s, val); 1082 } else { 1083 g_assert_not_reached(); 1084 } 1085} 1086 1087static inline void tcg_out_mb(TCGContext *s, TCGArg a0) 1088{ 1089 /* Given the strength of x86 memory ordering, we only need care for 1090 store-load ordering. Experimentally, "lock orl $0,0(%esp)" is 1091 faster than "mfence", so don't bother with the sse insn. */ 1092 if (a0 & TCG_MO_ST_LD) { 1093 tcg_out8(s, 0xf0); 1094 tcg_out_modrm_offset(s, OPC_ARITH_EvIb, ARITH_OR, TCG_REG_ESP, 0); 1095 tcg_out8(s, 0); 1096 } 1097} 1098 1099static inline void tcg_out_push(TCGContext *s, int reg) 1100{ 1101 tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0); 1102} 1103 1104static inline void tcg_out_pop(TCGContext *s, int reg) 1105{ 1106 tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0); 1107} 1108 1109static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, 1110 TCGReg arg1, intptr_t arg2) 1111{ 1112 switch (type) { 1113 case TCG_TYPE_I32: 1114 if (ret < 16) { 1115 tcg_out_modrm_offset(s, OPC_MOVL_GvEv, ret, arg1, arg2); 1116 } else { 1117 tcg_out_vex_modrm_offset(s, OPC_MOVD_VyEy, ret, 0, arg1, arg2); 1118 } 1119 break; 1120 case TCG_TYPE_I64: 1121 if (ret < 16) { 1122 tcg_out_modrm_offset(s, OPC_MOVL_GvEv | P_REXW, ret, arg1, arg2); 1123 break; 1124 } 1125 /* FALLTHRU */ 1126 case TCG_TYPE_V64: 1127 /* There is no instruction that can validate 8-byte alignment. */ 1128 tcg_debug_assert(ret >= 16); 1129 tcg_out_vex_modrm_offset(s, OPC_MOVQ_VqWq, ret, 0, arg1, arg2); 1130 break; 1131 case TCG_TYPE_V128: 1132 /* 1133 * The gvec infrastructure is asserts that v128 vector loads 1134 * and stores use a 16-byte aligned offset. Validate that the 1135 * final pointer is aligned by using an insn that will SIGSEGV. 1136 */ 1137 tcg_debug_assert(ret >= 16); 1138 tcg_out_vex_modrm_offset(s, OPC_MOVDQA_VxWx, ret, 0, arg1, arg2); 1139 break; 1140 case TCG_TYPE_V256: 1141 /* 1142 * The gvec infrastructure only requires 16-byte alignment, 1143 * so here we must use an unaligned load. 1144 */ 1145 tcg_debug_assert(ret >= 16); 1146 tcg_out_vex_modrm_offset(s, OPC_MOVDQU_VxWx | P_VEXL, 1147 ret, 0, arg1, arg2); 1148 break; 1149 default: 1150 g_assert_not_reached(); 1151 } 1152} 1153 1154static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, 1155 TCGReg arg1, intptr_t arg2) 1156{ 1157 switch (type) { 1158 case TCG_TYPE_I32: 1159 if (arg < 16) { 1160 tcg_out_modrm_offset(s, OPC_MOVL_EvGv, arg, arg1, arg2); 1161 } else { 1162 tcg_out_vex_modrm_offset(s, OPC_MOVD_EyVy, arg, 0, arg1, arg2); 1163 } 1164 break; 1165 case TCG_TYPE_I64: 1166 if (arg < 16) { 1167 tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_REXW, arg, arg1, arg2); 1168 break; 1169 } 1170 /* FALLTHRU */ 1171 case TCG_TYPE_V64: 1172 /* There is no instruction that can validate 8-byte alignment. */ 1173 tcg_debug_assert(arg >= 16); 1174 tcg_out_vex_modrm_offset(s, OPC_MOVQ_WqVq, arg, 0, arg1, arg2); 1175 break; 1176 case TCG_TYPE_V128: 1177 /* 1178 * The gvec infrastructure is asserts that v128 vector loads 1179 * and stores use a 16-byte aligned offset. Validate that the 1180 * final pointer is aligned by using an insn that will SIGSEGV. 1181 * 1182 * This specific instance is also used by TCG_CALL_RET_BY_VEC, 1183 * for _WIN64, which must have SSE2 but may not have AVX. 1184 */ 1185 tcg_debug_assert(arg >= 16); 1186 if (have_avx1) { 1187 tcg_out_vex_modrm_offset(s, OPC_MOVDQA_WxVx, arg, 0, arg1, arg2); 1188 } else { 1189 tcg_out_modrm_offset(s, OPC_MOVDQA_WxVx, arg, arg1, arg2); 1190 } 1191 break; 1192 case TCG_TYPE_V256: 1193 /* 1194 * The gvec infrastructure only requires 16-byte alignment, 1195 * so here we must use an unaligned store. 1196 */ 1197 tcg_debug_assert(arg >= 16); 1198 tcg_out_vex_modrm_offset(s, OPC_MOVDQU_WxVx | P_VEXL, 1199 arg, 0, arg1, arg2); 1200 break; 1201 default: 1202 g_assert_not_reached(); 1203 } 1204} 1205 1206static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 1207 TCGReg base, intptr_t ofs) 1208{ 1209 int rexw = 0; 1210 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) { 1211 if (val != (int32_t)val) { 1212 return false; 1213 } 1214 rexw = P_REXW; 1215 } else if (type != TCG_TYPE_I32) { 1216 return false; 1217 } 1218 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | rexw, 0, base, ofs); 1219 tcg_out32(s, val); 1220 return true; 1221} 1222 1223static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count) 1224{ 1225 /* Propagate an opcode prefix, such as P_DATA16. */ 1226 int ext = subopc & ~0x7; 1227 subopc &= 0x7; 1228 1229 if (count == 1) { 1230 tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg); 1231 } else { 1232 tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg); 1233 tcg_out8(s, count); 1234 } 1235} 1236 1237static inline void tcg_out_bswap32(TCGContext *s, int reg) 1238{ 1239 tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0); 1240} 1241 1242static inline void tcg_out_rolw_8(TCGContext *s, int reg) 1243{ 1244 tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8); 1245} 1246 1247static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src) 1248{ 1249 /* movzbl */ 1250 tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64); 1251 tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src); 1252} 1253 1254static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) 1255{ 1256 int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; 1257 /* movsbl */ 1258 tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64); 1259 tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src); 1260} 1261 1262static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src) 1263{ 1264 /* movzwl */ 1265 tcg_out_modrm(s, OPC_MOVZWL, dest, src); 1266} 1267 1268static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src) 1269{ 1270 int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; 1271 /* movsw[lq] */ 1272 tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src); 1273} 1274 1275static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src) 1276{ 1277 /* 32-bit mov zero extends. */ 1278 tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src); 1279} 1280 1281static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src) 1282{ 1283 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 1284 tcg_out_modrm(s, OPC_MOVSLQ, dest, src); 1285} 1286 1287static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src) 1288{ 1289 tcg_out_ext32s(s, dest, src); 1290} 1291 1292static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src) 1293{ 1294 if (dest != src) { 1295 tcg_out_ext32u(s, dest, src); 1296 } 1297} 1298 1299static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg dest, TCGReg src) 1300{ 1301 tcg_out_ext32u(s, dest, src); 1302} 1303 1304static inline void tcg_out_bswap64(TCGContext *s, int reg) 1305{ 1306 tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0); 1307} 1308 1309static void tgen_arithi(TCGContext *s, int c, int r0, 1310 tcg_target_long val, int cf) 1311{ 1312 int rexw = 0; 1313 1314 if (TCG_TARGET_REG_BITS == 64) { 1315 rexw = c & -8; 1316 c &= 7; 1317 } 1318 1319 /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce 1320 partial flags update stalls on Pentium4 and are not recommended 1321 by current Intel optimization manuals. */ 1322 if (!cf && (c == ARITH_ADD || c == ARITH_SUB) && (val == 1 || val == -1)) { 1323 int is_inc = (c == ARITH_ADD) ^ (val < 0); 1324 if (TCG_TARGET_REG_BITS == 64) { 1325 /* The single-byte increment encodings are re-tasked as the 1326 REX prefixes. Use the MODRM encoding. */ 1327 tcg_out_modrm(s, OPC_GRP5 + rexw, 1328 (is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0); 1329 } else { 1330 tcg_out8(s, (is_inc ? OPC_INC_r32 : OPC_DEC_r32) + r0); 1331 } 1332 return; 1333 } 1334 1335 if (c == ARITH_AND) { 1336 if (TCG_TARGET_REG_BITS == 64) { 1337 if (val == 0xffffffffu) { 1338 tcg_out_ext32u(s, r0, r0); 1339 return; 1340 } 1341 if (val == (uint32_t)val) { 1342 /* AND with no high bits set can use a 32-bit operation. */ 1343 rexw = 0; 1344 } 1345 } 1346 if (val == 0xffu && (r0 < 4 || TCG_TARGET_REG_BITS == 64)) { 1347 tcg_out_ext8u(s, r0, r0); 1348 return; 1349 } 1350 if (val == 0xffffu) { 1351 tcg_out_ext16u(s, r0, r0); 1352 return; 1353 } 1354 } 1355 1356 if (val == (int8_t)val) { 1357 tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0); 1358 tcg_out8(s, val); 1359 return; 1360 } 1361 if (rexw == 0 || val == (int32_t)val) { 1362 tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0); 1363 tcg_out32(s, val); 1364 return; 1365 } 1366 1367 g_assert_not_reached(); 1368} 1369 1370static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val) 1371{ 1372 if (val != 0) { 1373 tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0); 1374 } 1375} 1376 1377/* Set SMALL to force a short forward branch. */ 1378static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, bool small) 1379{ 1380 int32_t val, val1; 1381 1382 if (l->has_value) { 1383 val = tcg_pcrel_diff(s, l->u.value_ptr); 1384 val1 = val - 2; 1385 if ((int8_t)val1 == val1) { 1386 if (opc == -1) { 1387 tcg_out8(s, OPC_JMP_short); 1388 } else { 1389 tcg_out8(s, OPC_JCC_short + opc); 1390 } 1391 tcg_out8(s, val1); 1392 } else { 1393 tcg_debug_assert(!small); 1394 if (opc == -1) { 1395 tcg_out8(s, OPC_JMP_long); 1396 tcg_out32(s, val - 5); 1397 } else { 1398 tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0); 1399 tcg_out32(s, val - 6); 1400 } 1401 } 1402 } else if (small) { 1403 if (opc == -1) { 1404 tcg_out8(s, OPC_JMP_short); 1405 } else { 1406 tcg_out8(s, OPC_JCC_short + opc); 1407 } 1408 tcg_out_reloc(s, s->code_ptr, R_386_PC8, l, -1); 1409 s->code_ptr += 1; 1410 } else { 1411 if (opc == -1) { 1412 tcg_out8(s, OPC_JMP_long); 1413 } else { 1414 tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0); 1415 } 1416 tcg_out_reloc(s, s->code_ptr, R_386_PC32, l, -4); 1417 s->code_ptr += 4; 1418 } 1419} 1420 1421static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2, 1422 int const_arg2, int rexw) 1423{ 1424 if (const_arg2) { 1425 if (arg2 == 0) { 1426 /* test r, r */ 1427 tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1); 1428 } else { 1429 tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0); 1430 } 1431 } else { 1432 tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2); 1433 } 1434} 1435 1436static void tcg_out_brcond32(TCGContext *s, TCGCond cond, 1437 TCGArg arg1, TCGArg arg2, int const_arg2, 1438 TCGLabel *label, int small) 1439{ 1440 tcg_out_cmp(s, arg1, arg2, const_arg2, 0); 1441 tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small); 1442} 1443 1444#if TCG_TARGET_REG_BITS == 64 1445static void tcg_out_brcond64(TCGContext *s, TCGCond cond, 1446 TCGArg arg1, TCGArg arg2, int const_arg2, 1447 TCGLabel *label, int small) 1448{ 1449 tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW); 1450 tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small); 1451} 1452#else 1453/* XXX: we implement it at the target level to avoid having to 1454 handle cross basic blocks temporaries */ 1455static void tcg_out_brcond2(TCGContext *s, const TCGArg *args, 1456 const int *const_args, int small) 1457{ 1458 TCGLabel *label_next = gen_new_label(); 1459 TCGLabel *label_this = arg_label(args[5]); 1460 1461 switch(args[4]) { 1462 case TCG_COND_EQ: 1463 tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2], 1464 label_next, 1); 1465 tcg_out_brcond32(s, TCG_COND_EQ, args[1], args[3], const_args[3], 1466 label_this, small); 1467 break; 1468 case TCG_COND_NE: 1469 tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2], 1470 label_this, small); 1471 tcg_out_brcond32(s, TCG_COND_NE, args[1], args[3], const_args[3], 1472 label_this, small); 1473 break; 1474 case TCG_COND_LT: 1475 tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3], 1476 label_this, small); 1477 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1478 tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2], 1479 label_this, small); 1480 break; 1481 case TCG_COND_LE: 1482 tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3], 1483 label_this, small); 1484 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1485 tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2], 1486 label_this, small); 1487 break; 1488 case TCG_COND_GT: 1489 tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3], 1490 label_this, small); 1491 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1492 tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2], 1493 label_this, small); 1494 break; 1495 case TCG_COND_GE: 1496 tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3], 1497 label_this, small); 1498 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1499 tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2], 1500 label_this, small); 1501 break; 1502 case TCG_COND_LTU: 1503 tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3], 1504 label_this, small); 1505 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1506 tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2], 1507 label_this, small); 1508 break; 1509 case TCG_COND_LEU: 1510 tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3], 1511 label_this, small); 1512 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1513 tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2], 1514 label_this, small); 1515 break; 1516 case TCG_COND_GTU: 1517 tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3], 1518 label_this, small); 1519 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1520 tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2], 1521 label_this, small); 1522 break; 1523 case TCG_COND_GEU: 1524 tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3], 1525 label_this, small); 1526 tcg_out_jxx(s, JCC_JNE, label_next, 1); 1527 tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2], 1528 label_this, small); 1529 break; 1530 default: 1531 g_assert_not_reached(); 1532 } 1533 tcg_out_label(s, label_next); 1534} 1535#endif 1536 1537static void tcg_out_setcond32(TCGContext *s, TCGCond cond, TCGArg dest, 1538 TCGArg arg1, TCGArg arg2, int const_arg2) 1539{ 1540 tcg_out_cmp(s, arg1, arg2, const_arg2, 0); 1541 tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest); 1542 tcg_out_ext8u(s, dest, dest); 1543} 1544 1545#if TCG_TARGET_REG_BITS == 64 1546static void tcg_out_setcond64(TCGContext *s, TCGCond cond, TCGArg dest, 1547 TCGArg arg1, TCGArg arg2, int const_arg2) 1548{ 1549 tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW); 1550 tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest); 1551 tcg_out_ext8u(s, dest, dest); 1552} 1553#else 1554static void tcg_out_setcond2(TCGContext *s, const TCGArg *args, 1555 const int *const_args) 1556{ 1557 TCGArg new_args[6]; 1558 TCGLabel *label_true, *label_over; 1559 1560 memcpy(new_args, args+1, 5*sizeof(TCGArg)); 1561 1562 if (args[0] == args[1] || args[0] == args[2] 1563 || (!const_args[3] && args[0] == args[3]) 1564 || (!const_args[4] && args[0] == args[4])) { 1565 /* When the destination overlaps with one of the argument 1566 registers, don't do anything tricky. */ 1567 label_true = gen_new_label(); 1568 label_over = gen_new_label(); 1569 1570 new_args[5] = label_arg(label_true); 1571 tcg_out_brcond2(s, new_args, const_args+1, 1); 1572 1573 tcg_out_movi(s, TCG_TYPE_I32, args[0], 0); 1574 tcg_out_jxx(s, JCC_JMP, label_over, 1); 1575 tcg_out_label(s, label_true); 1576 1577 tcg_out_movi(s, TCG_TYPE_I32, args[0], 1); 1578 tcg_out_label(s, label_over); 1579 } else { 1580 /* When the destination does not overlap one of the arguments, 1581 clear the destination first, jump if cond false, and emit an 1582 increment in the true case. This results in smaller code. */ 1583 1584 tcg_out_movi(s, TCG_TYPE_I32, args[0], 0); 1585 1586 label_over = gen_new_label(); 1587 new_args[4] = tcg_invert_cond(new_args[4]); 1588 new_args[5] = label_arg(label_over); 1589 tcg_out_brcond2(s, new_args, const_args+1, 1); 1590 1591 tgen_arithi(s, ARITH_ADD, args[0], 1, 0); 1592 tcg_out_label(s, label_over); 1593 } 1594} 1595#endif 1596 1597static void tcg_out_cmov(TCGContext *s, TCGCond cond, int rexw, 1598 TCGReg dest, TCGReg v1) 1599{ 1600 if (have_cmov) { 1601 tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | rexw, dest, v1); 1602 } else { 1603 TCGLabel *over = gen_new_label(); 1604 tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1); 1605 tcg_out_mov(s, TCG_TYPE_I32, dest, v1); 1606 tcg_out_label(s, over); 1607 } 1608} 1609 1610static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGReg dest, 1611 TCGReg c1, TCGArg c2, int const_c2, 1612 TCGReg v1) 1613{ 1614 tcg_out_cmp(s, c1, c2, const_c2, 0); 1615 tcg_out_cmov(s, cond, 0, dest, v1); 1616} 1617 1618#if TCG_TARGET_REG_BITS == 64 1619static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGReg dest, 1620 TCGReg c1, TCGArg c2, int const_c2, 1621 TCGReg v1) 1622{ 1623 tcg_out_cmp(s, c1, c2, const_c2, P_REXW); 1624 tcg_out_cmov(s, cond, P_REXW, dest, v1); 1625} 1626#endif 1627 1628static void tcg_out_ctz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, 1629 TCGArg arg2, bool const_a2) 1630{ 1631 if (have_bmi1) { 1632 tcg_out_modrm(s, OPC_TZCNT + rexw, dest, arg1); 1633 if (const_a2) { 1634 tcg_debug_assert(arg2 == (rexw ? 64 : 32)); 1635 } else { 1636 tcg_debug_assert(dest != arg2); 1637 tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2); 1638 } 1639 } else { 1640 tcg_debug_assert(dest != arg2); 1641 tcg_out_modrm(s, OPC_BSF + rexw, dest, arg1); 1642 tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2); 1643 } 1644} 1645 1646static void tcg_out_clz(TCGContext *s, int rexw, TCGReg dest, TCGReg arg1, 1647 TCGArg arg2, bool const_a2) 1648{ 1649 if (have_lzcnt) { 1650 tcg_out_modrm(s, OPC_LZCNT + rexw, dest, arg1); 1651 if (const_a2) { 1652 tcg_debug_assert(arg2 == (rexw ? 64 : 32)); 1653 } else { 1654 tcg_debug_assert(dest != arg2); 1655 tcg_out_cmov(s, TCG_COND_LTU, rexw, dest, arg2); 1656 } 1657 } else { 1658 tcg_debug_assert(!const_a2); 1659 tcg_debug_assert(dest != arg1); 1660 tcg_debug_assert(dest != arg2); 1661 1662 /* Recall that the output of BSR is the index not the count. */ 1663 tcg_out_modrm(s, OPC_BSR + rexw, dest, arg1); 1664 tgen_arithi(s, ARITH_XOR + rexw, dest, rexw ? 63 : 31, 0); 1665 1666 /* Since we have destroyed the flags from BSR, we have to re-test. */ 1667 tcg_out_cmp(s, arg1, 0, 1, rexw); 1668 tcg_out_cmov(s, TCG_COND_EQ, rexw, dest, arg2); 1669 } 1670} 1671 1672static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest) 1673{ 1674 intptr_t disp = tcg_pcrel_diff(s, dest) - 5; 1675 1676 if (disp == (int32_t)disp) { 1677 tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0); 1678 tcg_out32(s, disp); 1679 } else { 1680 /* rip-relative addressing into the constant pool. 1681 This is 6 + 8 = 14 bytes, as compared to using an 1682 immediate load 10 + 6 = 16 bytes, plus we may 1683 be able to re-use the pool constant for more calls. */ 1684 tcg_out_opc(s, OPC_GRP5, 0, 0, 0); 1685 tcg_out8(s, (call ? EXT5_CALLN_Ev : EXT5_JMPN_Ev) << 3 | 5); 1686 new_pool_label(s, (uintptr_t)dest, R_386_PC32, s->code_ptr, -4); 1687 tcg_out32(s, 0); 1688 } 1689} 1690 1691static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest, 1692 const TCGHelperInfo *info) 1693{ 1694 tcg_out_branch(s, 1, dest); 1695 1696#ifndef _WIN32 1697 if (TCG_TARGET_REG_BITS == 32 && info->out_kind == TCG_CALL_RET_BY_REF) { 1698 /* 1699 * The sysv i386 abi for struct return places a reference as the 1700 * first argument of the stack, and pops that argument with the 1701 * return statement. Since we want to retain the aligned stack 1702 * pointer for the callee, we do not want to actually push that 1703 * argument before the call but rely on the normal store to the 1704 * stack slot. But we do need to compensate for the pop in order 1705 * to reset our correct stack pointer value. 1706 * Pushing a garbage value back onto the stack is quickest. 1707 */ 1708 tcg_out_push(s, TCG_REG_EAX); 1709 } 1710#endif 1711} 1712 1713static void tcg_out_jmp(TCGContext *s, const tcg_insn_unit *dest) 1714{ 1715 tcg_out_branch(s, 0, dest); 1716} 1717 1718static void tcg_out_nopn(TCGContext *s, int n) 1719{ 1720 int i; 1721 /* Emit 1 or 2 operand size prefixes for the standard one byte nop, 1722 * "xchg %eax,%eax", forming "xchg %ax,%ax". All cores accept the 1723 * duplicate prefix, and all of the interesting recent cores can 1724 * decode and discard the duplicates in a single cycle. 1725 */ 1726 tcg_debug_assert(n >= 1); 1727 for (i = 1; i < n; ++i) { 1728 tcg_out8(s, 0x66); 1729 } 1730 tcg_out8(s, 0x90); 1731} 1732 1733/* Test register R vs immediate bits I, setting Z flag for EQ/NE. */ 1734static void __attribute__((unused)) 1735tcg_out_testi(TCGContext *s, TCGReg r, uint32_t i) 1736{ 1737 /* 1738 * This is used for testing alignment, so we can usually use testb. 1739 * For i686, we have to use testl for %esi/%edi. 1740 */ 1741 if (i <= 0xff && (TCG_TARGET_REG_BITS == 64 || r < 4)) { 1742 tcg_out_modrm(s, OPC_GRP3_Eb | P_REXB_RM, EXT3_TESTi, r); 1743 tcg_out8(s, i); 1744 } else { 1745 tcg_out_modrm(s, OPC_GRP3_Ev, EXT3_TESTi, r); 1746 tcg_out32(s, i); 1747 } 1748} 1749 1750typedef struct { 1751 TCGReg base; 1752 int index; 1753 int ofs; 1754 int seg; 1755 TCGAtomAlign aa; 1756} HostAddress; 1757 1758bool tcg_target_has_memory_bswap(MemOp memop) 1759{ 1760 TCGAtomAlign aa; 1761 1762 if (!have_movbe) { 1763 return false; 1764 } 1765 if ((memop & MO_SIZE) < MO_128) { 1766 return true; 1767 } 1768 1769 /* 1770 * Reject 16-byte memop with 16-byte atomicity, i.e. VMOVDQA, 1771 * but do allow a pair of 64-bit operations, i.e. MOVBEQ. 1772 */ 1773 aa = atom_and_align_for_opc(tcg_ctx, memop, MO_ATOM_IFALIGN, true); 1774 return aa.atom < MO_128; 1775} 1776 1777/* 1778 * Because i686 has no register parameters and because x86_64 has xchg 1779 * to handle addr/data register overlap, we have placed all input arguments 1780 * before we need might need a scratch reg. 1781 * 1782 * Even then, a scratch is only needed for l->raddr. Rather than expose 1783 * a general-purpose scratch when we don't actually know it's available, 1784 * use the ra_gen hook to load into RAX if needed. 1785 */ 1786#if TCG_TARGET_REG_BITS == 64 1787static TCGReg ldst_ra_gen(TCGContext *s, const TCGLabelQemuLdst *l, int arg) 1788{ 1789 if (arg < 0) { 1790 arg = TCG_REG_RAX; 1791 } 1792 tcg_out_movi(s, TCG_TYPE_PTR, arg, (uintptr_t)l->raddr); 1793 return arg; 1794} 1795static const TCGLdstHelperParam ldst_helper_param = { 1796 .ra_gen = ldst_ra_gen 1797}; 1798#else 1799static const TCGLdstHelperParam ldst_helper_param = { }; 1800#endif 1801 1802static void tcg_out_vec_to_pair(TCGContext *s, TCGType type, 1803 TCGReg l, TCGReg h, TCGReg v) 1804{ 1805 int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; 1806 1807 /* vpmov{d,q} %v, %l */ 1808 tcg_out_vex_modrm(s, OPC_MOVD_EyVy + rexw, v, 0, l); 1809 /* vpextr{d,q} $1, %v, %h */ 1810 tcg_out_vex_modrm(s, OPC_PEXTRD + rexw, v, 0, h); 1811 tcg_out8(s, 1); 1812} 1813 1814static void tcg_out_pair_to_vec(TCGContext *s, TCGType type, 1815 TCGReg v, TCGReg l, TCGReg h) 1816{ 1817 int rexw = type == TCG_TYPE_I32 ? 0 : P_REXW; 1818 1819 /* vmov{d,q} %l, %v */ 1820 tcg_out_vex_modrm(s, OPC_MOVD_VyEy + rexw, v, 0, l); 1821 /* vpinsr{d,q} $1, %h, %v, %v */ 1822 tcg_out_vex_modrm(s, OPC_PINSRD + rexw, v, v, h); 1823 tcg_out8(s, 1); 1824} 1825 1826/* 1827 * Generate code for the slow path for a load at the end of block 1828 */ 1829static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 1830{ 1831 MemOp opc = get_memop(l->oi); 1832 tcg_insn_unit **label_ptr = &l->label_ptr[0]; 1833 1834 /* resolve label address */ 1835 tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4); 1836 if (label_ptr[1]) { 1837 tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4); 1838 } 1839 1840 tcg_out_ld_helper_args(s, l, &ldst_helper_param); 1841 tcg_out_branch(s, 1, qemu_ld_helpers[opc & MO_SIZE]); 1842 tcg_out_ld_helper_ret(s, l, false, &ldst_helper_param); 1843 1844 tcg_out_jmp(s, l->raddr); 1845 return true; 1846} 1847 1848/* 1849 * Generate code for the slow path for a store at the end of block 1850 */ 1851static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 1852{ 1853 MemOp opc = get_memop(l->oi); 1854 tcg_insn_unit **label_ptr = &l->label_ptr[0]; 1855 1856 /* resolve label address */ 1857 tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4); 1858 if (label_ptr[1]) { 1859 tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4); 1860 } 1861 1862 tcg_out_st_helper_args(s, l, &ldst_helper_param); 1863 tcg_out_branch(s, 1, qemu_st_helpers[opc & MO_SIZE]); 1864 1865 tcg_out_jmp(s, l->raddr); 1866 return true; 1867} 1868 1869#ifndef CONFIG_SOFTMMU 1870static HostAddress x86_guest_base = { 1871 .index = -1 1872}; 1873 1874#if defined(__x86_64__) && defined(__linux__) 1875# include <asm/prctl.h> 1876# include <sys/prctl.h> 1877int arch_prctl(int code, unsigned long addr); 1878static inline int setup_guest_base_seg(void) 1879{ 1880 if (arch_prctl(ARCH_SET_GS, guest_base) == 0) { 1881 return P_GS; 1882 } 1883 return 0; 1884} 1885#elif defined(__x86_64__) && \ 1886 (defined (__FreeBSD__) || defined (__FreeBSD_kernel__)) 1887# include <machine/sysarch.h> 1888static inline int setup_guest_base_seg(void) 1889{ 1890 if (sysarch(AMD64_SET_GSBASE, &guest_base) == 0) { 1891 return P_GS; 1892 } 1893 return 0; 1894} 1895#else 1896static inline int setup_guest_base_seg(void) 1897{ 1898 return 0; 1899} 1900#endif /* setup_guest_base_seg */ 1901#endif /* !SOFTMMU */ 1902 1903/* 1904 * For softmmu, perform the TLB load and compare. 1905 * For useronly, perform any required alignment tests. 1906 * In both cases, return a TCGLabelQemuLdst structure if the slow path 1907 * is required and fill in @h with the host address for the fast path. 1908 */ 1909static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, HostAddress *h, 1910 TCGReg addrlo, TCGReg addrhi, 1911 MemOpIdx oi, bool is_ld) 1912{ 1913 TCGLabelQemuLdst *ldst = NULL; 1914 MemOp opc = get_memop(oi); 1915 MemOp s_bits = opc & MO_SIZE; 1916 unsigned a_mask; 1917 1918#ifdef CONFIG_SOFTMMU 1919 h->index = TCG_REG_L0; 1920 h->ofs = 0; 1921 h->seg = 0; 1922#else 1923 *h = x86_guest_base; 1924#endif 1925 h->base = addrlo; 1926 h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, s_bits == MO_128); 1927 a_mask = (1 << h->aa.align) - 1; 1928 1929#ifdef CONFIG_SOFTMMU 1930 int cmp_ofs = is_ld ? offsetof(CPUTLBEntry, addr_read) 1931 : offsetof(CPUTLBEntry, addr_write); 1932 TCGType ttype = TCG_TYPE_I32; 1933 TCGType tlbtype = TCG_TYPE_I32; 1934 int trexw = 0, hrexw = 0, tlbrexw = 0; 1935 unsigned mem_index = get_mmuidx(oi); 1936 unsigned s_mask = (1 << s_bits) - 1; 1937 int tlb_mask; 1938 1939 ldst = new_ldst_label(s); 1940 ldst->is_ld = is_ld; 1941 ldst->oi = oi; 1942 ldst->addrlo_reg = addrlo; 1943 ldst->addrhi_reg = addrhi; 1944 1945 if (TCG_TARGET_REG_BITS == 64) { 1946 ttype = s->addr_type; 1947 trexw = (ttype == TCG_TYPE_I32 ? 0 : P_REXW); 1948 if (TCG_TYPE_PTR == TCG_TYPE_I64) { 1949 hrexw = P_REXW; 1950 if (s->page_bits + s->tlb_dyn_max_bits > 32) { 1951 tlbtype = TCG_TYPE_I64; 1952 tlbrexw = P_REXW; 1953 } 1954 } 1955 } 1956 1957 tcg_out_mov(s, tlbtype, TCG_REG_L0, addrlo); 1958 tcg_out_shifti(s, SHIFT_SHR + tlbrexw, TCG_REG_L0, 1959 s->page_bits - CPU_TLB_ENTRY_BITS); 1960 1961 tcg_out_modrm_offset(s, OPC_AND_GvEv + trexw, TCG_REG_L0, TCG_AREG0, 1962 TLB_MASK_TABLE_OFS(mem_index) + 1963 offsetof(CPUTLBDescFast, mask)); 1964 1965 tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, TCG_REG_L0, TCG_AREG0, 1966 TLB_MASK_TABLE_OFS(mem_index) + 1967 offsetof(CPUTLBDescFast, table)); 1968 1969 /* 1970 * If the required alignment is at least as large as the access, simply 1971 * copy the address and mask. For lesser alignments, check that we don't 1972 * cross pages for the complete access. 1973 */ 1974 if (a_mask >= s_mask) { 1975 tcg_out_mov(s, ttype, TCG_REG_L1, addrlo); 1976 } else { 1977 tcg_out_modrm_offset(s, OPC_LEA + trexw, TCG_REG_L1, 1978 addrlo, s_mask - a_mask); 1979 } 1980 tlb_mask = s->page_mask | a_mask; 1981 tgen_arithi(s, ARITH_AND + trexw, TCG_REG_L1, tlb_mask, 0); 1982 1983 /* cmp 0(TCG_REG_L0), TCG_REG_L1 */ 1984 tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, 1985 TCG_REG_L1, TCG_REG_L0, cmp_ofs); 1986 1987 /* jne slow_path */ 1988 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); 1989 ldst->label_ptr[0] = s->code_ptr; 1990 s->code_ptr += 4; 1991 1992 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I64) { 1993 /* cmp 4(TCG_REG_L0), addrhi */ 1994 tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, TCG_REG_L0, cmp_ofs + 4); 1995 1996 /* jne slow_path */ 1997 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); 1998 ldst->label_ptr[1] = s->code_ptr; 1999 s->code_ptr += 4; 2000 } 2001 2002 /* TLB Hit. */ 2003 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_L0, TCG_REG_L0, 2004 offsetof(CPUTLBEntry, addend)); 2005#else 2006 if (a_mask) { 2007 ldst = new_ldst_label(s); 2008 2009 ldst->is_ld = is_ld; 2010 ldst->oi = oi; 2011 ldst->addrlo_reg = addrlo; 2012 ldst->addrhi_reg = addrhi; 2013 2014 tcg_out_testi(s, addrlo, a_mask); 2015 /* jne slow_path */ 2016 tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0); 2017 ldst->label_ptr[0] = s->code_ptr; 2018 s->code_ptr += 4; 2019 } 2020#endif 2021 2022 return ldst; 2023} 2024 2025static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, 2026 HostAddress h, TCGType type, MemOp memop) 2027{ 2028 bool use_movbe = false; 2029 int rexw = (type == TCG_TYPE_I32 ? 0 : P_REXW); 2030 int movop = OPC_MOVL_GvEv; 2031 2032 /* Do big-endian loads with movbe. */ 2033 if (memop & MO_BSWAP) { 2034 tcg_debug_assert(have_movbe); 2035 use_movbe = true; 2036 movop = OPC_MOVBE_GyMy; 2037 } 2038 2039 switch (memop & MO_SSIZE) { 2040 case MO_UB: 2041 tcg_out_modrm_sib_offset(s, OPC_MOVZBL + h.seg, datalo, 2042 h.base, h.index, 0, h.ofs); 2043 break; 2044 case MO_SB: 2045 tcg_out_modrm_sib_offset(s, OPC_MOVSBL + rexw + h.seg, datalo, 2046 h.base, h.index, 0, h.ofs); 2047 break; 2048 case MO_UW: 2049 if (use_movbe) { 2050 /* There is no extending movbe; only low 16-bits are modified. */ 2051 if (datalo != h.base && datalo != h.index) { 2052 /* XOR breaks dependency chains. */ 2053 tgen_arithr(s, ARITH_XOR, datalo, datalo); 2054 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, 2055 datalo, h.base, h.index, 0, h.ofs); 2056 } else { 2057 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, 2058 datalo, h.base, h.index, 0, h.ofs); 2059 tcg_out_ext16u(s, datalo, datalo); 2060 } 2061 } else { 2062 tcg_out_modrm_sib_offset(s, OPC_MOVZWL + h.seg, datalo, 2063 h.base, h.index, 0, h.ofs); 2064 } 2065 break; 2066 case MO_SW: 2067 if (use_movbe) { 2068 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + h.seg, 2069 datalo, h.base, h.index, 0, h.ofs); 2070 tcg_out_ext16s(s, type, datalo, datalo); 2071 } else { 2072 tcg_out_modrm_sib_offset(s, OPC_MOVSWL + rexw + h.seg, 2073 datalo, h.base, h.index, 0, h.ofs); 2074 } 2075 break; 2076 case MO_UL: 2077 tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, 2078 h.base, h.index, 0, h.ofs); 2079 break; 2080#if TCG_TARGET_REG_BITS == 64 2081 case MO_SL: 2082 if (use_movbe) { 2083 tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + h.seg, datalo, 2084 h.base, h.index, 0, h.ofs); 2085 tcg_out_ext32s(s, datalo, datalo); 2086 } else { 2087 tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + h.seg, datalo, 2088 h.base, h.index, 0, h.ofs); 2089 } 2090 break; 2091#endif 2092 case MO_UQ: 2093 if (TCG_TARGET_REG_BITS == 64) { 2094 tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo, 2095 h.base, h.index, 0, h.ofs); 2096 break; 2097 } 2098 if (use_movbe) { 2099 TCGReg t = datalo; 2100 datalo = datahi; 2101 datahi = t; 2102 } 2103 if (h.base == datalo || h.index == datalo) { 2104 tcg_out_modrm_sib_offset(s, OPC_LEA, datahi, 2105 h.base, h.index, 0, h.ofs); 2106 tcg_out_modrm_offset(s, movop + h.seg, datalo, datahi, 0); 2107 tcg_out_modrm_offset(s, movop + h.seg, datahi, datahi, 4); 2108 } else { 2109 tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, 2110 h.base, h.index, 0, h.ofs); 2111 tcg_out_modrm_sib_offset(s, movop + h.seg, datahi, 2112 h.base, h.index, 0, h.ofs + 4); 2113 } 2114 break; 2115 2116 case MO_128: 2117 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 2118 2119 /* 2120 * Without 16-byte atomicity, use integer regs. 2121 * That is where we want the data, and it allows bswaps. 2122 */ 2123 if (h.aa.atom < MO_128) { 2124 if (use_movbe) { 2125 TCGReg t = datalo; 2126 datalo = datahi; 2127 datahi = t; 2128 } 2129 if (h.base == datalo || h.index == datalo) { 2130 tcg_out_modrm_sib_offset(s, OPC_LEA + P_REXW, datahi, 2131 h.base, h.index, 0, h.ofs); 2132 tcg_out_modrm_offset(s, movop + P_REXW + h.seg, 2133 datalo, datahi, 0); 2134 tcg_out_modrm_offset(s, movop + P_REXW + h.seg, 2135 datahi, datahi, 8); 2136 } else { 2137 tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo, 2138 h.base, h.index, 0, h.ofs); 2139 tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datahi, 2140 h.base, h.index, 0, h.ofs + 8); 2141 } 2142 break; 2143 } 2144 2145 /* 2146 * With 16-byte atomicity, a vector load is required. 2147 * If we already have 16-byte alignment, then VMOVDQA always works. 2148 * Else if VMOVDQU has atomicity with dynamic alignment, use that. 2149 * Else use we require a runtime test for alignment for VMOVDQA; 2150 * use VMOVDQU on the unaligned nonatomic path for simplicity. 2151 */ 2152 if (h.aa.align >= MO_128) { 2153 tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_VxWx + h.seg, 2154 TCG_TMP_VEC, 0, 2155 h.base, h.index, 0, h.ofs); 2156 } else if (cpuinfo & CPUINFO_ATOMIC_VMOVDQU) { 2157 tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQU_VxWx + h.seg, 2158 TCG_TMP_VEC, 0, 2159 h.base, h.index, 0, h.ofs); 2160 } else { 2161 TCGLabel *l1 = gen_new_label(); 2162 TCGLabel *l2 = gen_new_label(); 2163 2164 tcg_out_testi(s, h.base, 15); 2165 tcg_out_jxx(s, JCC_JNE, l1, true); 2166 2167 tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_VxWx + h.seg, 2168 TCG_TMP_VEC, 0, 2169 h.base, h.index, 0, h.ofs); 2170 tcg_out_jxx(s, JCC_JMP, l2, true); 2171 2172 tcg_out_label(s, l1); 2173 tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQU_VxWx + h.seg, 2174 TCG_TMP_VEC, 0, 2175 h.base, h.index, 0, h.ofs); 2176 tcg_out_label(s, l2); 2177 } 2178 tcg_out_vec_to_pair(s, TCG_TYPE_I64, datalo, datahi, TCG_TMP_VEC); 2179 break; 2180 2181 default: 2182 g_assert_not_reached(); 2183 } 2184} 2185 2186static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi, 2187 TCGReg addrlo, TCGReg addrhi, 2188 MemOpIdx oi, TCGType data_type) 2189{ 2190 TCGLabelQemuLdst *ldst; 2191 HostAddress h; 2192 2193 ldst = prepare_host_addr(s, &h, addrlo, addrhi, oi, true); 2194 tcg_out_qemu_ld_direct(s, datalo, datahi, h, data_type, get_memop(oi)); 2195 2196 if (ldst) { 2197 ldst->type = data_type; 2198 ldst->datalo_reg = datalo; 2199 ldst->datahi_reg = datahi; 2200 ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); 2201 } 2202} 2203 2204static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi, 2205 HostAddress h, MemOp memop) 2206{ 2207 bool use_movbe = false; 2208 int movop = OPC_MOVL_EvGv; 2209 2210 /* 2211 * Do big-endian stores with movbe or softmmu. 2212 * User-only without movbe will have its swapping done generically. 2213 */ 2214 if (memop & MO_BSWAP) { 2215 tcg_debug_assert(have_movbe); 2216 use_movbe = true; 2217 movop = OPC_MOVBE_MyGy; 2218 } 2219 2220 switch (memop & MO_SIZE) { 2221 case MO_8: 2222 /* This is handled with constraints on INDEX_op_qemu_st8_i32. */ 2223 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || datalo < 4); 2224 tcg_out_modrm_sib_offset(s, OPC_MOVB_EvGv + P_REXB_R + h.seg, 2225 datalo, h.base, h.index, 0, h.ofs); 2226 break; 2227 case MO_16: 2228 tcg_out_modrm_sib_offset(s, movop + P_DATA16 + h.seg, datalo, 2229 h.base, h.index, 0, h.ofs); 2230 break; 2231 case MO_32: 2232 tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, 2233 h.base, h.index, 0, h.ofs); 2234 break; 2235 case MO_64: 2236 if (TCG_TARGET_REG_BITS == 64) { 2237 tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo, 2238 h.base, h.index, 0, h.ofs); 2239 } else { 2240 if (use_movbe) { 2241 TCGReg t = datalo; 2242 datalo = datahi; 2243 datahi = t; 2244 } 2245 tcg_out_modrm_sib_offset(s, movop + h.seg, datalo, 2246 h.base, h.index, 0, h.ofs); 2247 tcg_out_modrm_sib_offset(s, movop + h.seg, datahi, 2248 h.base, h.index, 0, h.ofs + 4); 2249 } 2250 break; 2251 2252 case MO_128: 2253 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 2254 2255 /* 2256 * Without 16-byte atomicity, use integer regs. 2257 * That is where we have the data, and it allows bswaps. 2258 */ 2259 if (h.aa.atom < MO_128) { 2260 if (use_movbe) { 2261 TCGReg t = datalo; 2262 datalo = datahi; 2263 datahi = t; 2264 } 2265 tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datalo, 2266 h.base, h.index, 0, h.ofs); 2267 tcg_out_modrm_sib_offset(s, movop + P_REXW + h.seg, datahi, 2268 h.base, h.index, 0, h.ofs + 8); 2269 break; 2270 } 2271 2272 /* 2273 * With 16-byte atomicity, a vector store is required. 2274 * If we already have 16-byte alignment, then VMOVDQA always works. 2275 * Else if VMOVDQU has atomicity with dynamic alignment, use that. 2276 * Else use we require a runtime test for alignment for VMOVDQA; 2277 * use VMOVDQU on the unaligned nonatomic path for simplicity. 2278 */ 2279 tcg_out_pair_to_vec(s, TCG_TYPE_I64, TCG_TMP_VEC, datalo, datahi); 2280 if (h.aa.align >= MO_128) { 2281 tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_WxVx + h.seg, 2282 TCG_TMP_VEC, 0, 2283 h.base, h.index, 0, h.ofs); 2284 } else if (cpuinfo & CPUINFO_ATOMIC_VMOVDQU) { 2285 tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQU_WxVx + h.seg, 2286 TCG_TMP_VEC, 0, 2287 h.base, h.index, 0, h.ofs); 2288 } else { 2289 TCGLabel *l1 = gen_new_label(); 2290 TCGLabel *l2 = gen_new_label(); 2291 2292 tcg_out_testi(s, h.base, 15); 2293 tcg_out_jxx(s, JCC_JNE, l1, true); 2294 2295 tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQA_WxVx + h.seg, 2296 TCG_TMP_VEC, 0, 2297 h.base, h.index, 0, h.ofs); 2298 tcg_out_jxx(s, JCC_JMP, l2, true); 2299 2300 tcg_out_label(s, l1); 2301 tcg_out_vex_modrm_sib_offset(s, OPC_MOVDQU_WxVx + h.seg, 2302 TCG_TMP_VEC, 0, 2303 h.base, h.index, 0, h.ofs); 2304 tcg_out_label(s, l2); 2305 } 2306 break; 2307 2308 default: 2309 g_assert_not_reached(); 2310 } 2311} 2312 2313static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi, 2314 TCGReg addrlo, TCGReg addrhi, 2315 MemOpIdx oi, TCGType data_type) 2316{ 2317 TCGLabelQemuLdst *ldst; 2318 HostAddress h; 2319 2320 ldst = prepare_host_addr(s, &h, addrlo, addrhi, oi, false); 2321 tcg_out_qemu_st_direct(s, datalo, datahi, h, get_memop(oi)); 2322 2323 if (ldst) { 2324 ldst->type = data_type; 2325 ldst->datalo_reg = datalo; 2326 ldst->datahi_reg = datahi; 2327 ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); 2328 } 2329} 2330 2331static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) 2332{ 2333 /* Reuse the zeroing that exists for goto_ptr. */ 2334 if (a0 == 0) { 2335 tcg_out_jmp(s, tcg_code_gen_epilogue); 2336 } else { 2337 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, a0); 2338 tcg_out_jmp(s, tb_ret_addr); 2339 } 2340} 2341 2342static void tcg_out_goto_tb(TCGContext *s, int which) 2343{ 2344 /* 2345 * Jump displacement must be aligned for atomic patching; 2346 * see if we need to add extra nops before jump 2347 */ 2348 int gap = QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4) - s->code_ptr; 2349 if (gap != 1) { 2350 tcg_out_nopn(s, gap - 1); 2351 } 2352 tcg_out8(s, OPC_JMP_long); /* jmp im */ 2353 set_jmp_insn_offset(s, which); 2354 tcg_out32(s, 0); 2355 set_jmp_reset_offset(s, which); 2356} 2357 2358void tb_target_set_jmp_target(const TranslationBlock *tb, int n, 2359 uintptr_t jmp_rx, uintptr_t jmp_rw) 2360{ 2361 /* patch the branch destination */ 2362 uintptr_t addr = tb->jmp_target_addr[n]; 2363 qatomic_set((int32_t *)jmp_rw, addr - (jmp_rx + 4)); 2364 /* no need to flush icache explicitly */ 2365} 2366 2367static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, 2368 const TCGArg args[TCG_MAX_OP_ARGS], 2369 const int const_args[TCG_MAX_OP_ARGS]) 2370{ 2371 TCGArg a0, a1, a2; 2372 int c, const_a2, vexop, rexw = 0; 2373 2374#if TCG_TARGET_REG_BITS == 64 2375# define OP_32_64(x) \ 2376 case glue(glue(INDEX_op_, x), _i64): \ 2377 rexw = P_REXW; /* FALLTHRU */ \ 2378 case glue(glue(INDEX_op_, x), _i32) 2379#else 2380# define OP_32_64(x) \ 2381 case glue(glue(INDEX_op_, x), _i32) 2382#endif 2383 2384 /* Hoist the loads of the most common arguments. */ 2385 a0 = args[0]; 2386 a1 = args[1]; 2387 a2 = args[2]; 2388 const_a2 = const_args[2]; 2389 2390 switch (opc) { 2391 case INDEX_op_goto_ptr: 2392 /* jmp to the given host address (could be epilogue) */ 2393 tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, a0); 2394 break; 2395 case INDEX_op_br: 2396 tcg_out_jxx(s, JCC_JMP, arg_label(a0), 0); 2397 break; 2398 OP_32_64(ld8u): 2399 /* Note that we can ignore REXW for the zero-extend to 64-bit. */ 2400 tcg_out_modrm_offset(s, OPC_MOVZBL, a0, a1, a2); 2401 break; 2402 OP_32_64(ld8s): 2403 tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, a0, a1, a2); 2404 break; 2405 OP_32_64(ld16u): 2406 /* Note that we can ignore REXW for the zero-extend to 64-bit. */ 2407 tcg_out_modrm_offset(s, OPC_MOVZWL, a0, a1, a2); 2408 break; 2409 OP_32_64(ld16s): 2410 tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, a0, a1, a2); 2411 break; 2412#if TCG_TARGET_REG_BITS == 64 2413 case INDEX_op_ld32u_i64: 2414#endif 2415 case INDEX_op_ld_i32: 2416 tcg_out_ld(s, TCG_TYPE_I32, a0, a1, a2); 2417 break; 2418 2419 OP_32_64(st8): 2420 if (const_args[0]) { 2421 tcg_out_modrm_offset(s, OPC_MOVB_EvIz, 0, a1, a2); 2422 tcg_out8(s, a0); 2423 } else { 2424 tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R, a0, a1, a2); 2425 } 2426 break; 2427 OP_32_64(st16): 2428 if (const_args[0]) { 2429 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16, 0, a1, a2); 2430 tcg_out16(s, a0); 2431 } else { 2432 tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16, a0, a1, a2); 2433 } 2434 break; 2435#if TCG_TARGET_REG_BITS == 64 2436 case INDEX_op_st32_i64: 2437#endif 2438 case INDEX_op_st_i32: 2439 if (const_args[0]) { 2440 tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, a1, a2); 2441 tcg_out32(s, a0); 2442 } else { 2443 tcg_out_st(s, TCG_TYPE_I32, a0, a1, a2); 2444 } 2445 break; 2446 2447 OP_32_64(add): 2448 /* For 3-operand addition, use LEA. */ 2449 if (a0 != a1) { 2450 TCGArg c3 = 0; 2451 if (const_a2) { 2452 c3 = a2, a2 = -1; 2453 } else if (a0 == a2) { 2454 /* Watch out for dest = src + dest, since we've removed 2455 the matching constraint on the add. */ 2456 tgen_arithr(s, ARITH_ADD + rexw, a0, a1); 2457 break; 2458 } 2459 2460 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3); 2461 break; 2462 } 2463 c = ARITH_ADD; 2464 goto gen_arith; 2465 OP_32_64(sub): 2466 c = ARITH_SUB; 2467 goto gen_arith; 2468 OP_32_64(and): 2469 c = ARITH_AND; 2470 goto gen_arith; 2471 OP_32_64(or): 2472 c = ARITH_OR; 2473 goto gen_arith; 2474 OP_32_64(xor): 2475 c = ARITH_XOR; 2476 goto gen_arith; 2477 gen_arith: 2478 if (const_a2) { 2479 tgen_arithi(s, c + rexw, a0, a2, 0); 2480 } else { 2481 tgen_arithr(s, c + rexw, a0, a2); 2482 } 2483 break; 2484 2485 OP_32_64(andc): 2486 if (const_a2) { 2487 tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1); 2488 tgen_arithi(s, ARITH_AND + rexw, a0, ~a2, 0); 2489 } else { 2490 tcg_out_vex_modrm(s, OPC_ANDN + rexw, a0, a2, a1); 2491 } 2492 break; 2493 2494 OP_32_64(mul): 2495 if (const_a2) { 2496 int32_t val; 2497 val = a2; 2498 if (val == (int8_t)val) { 2499 tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, a0, a0); 2500 tcg_out8(s, val); 2501 } else { 2502 tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, a0, a0); 2503 tcg_out32(s, val); 2504 } 2505 } else { 2506 tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, a0, a2); 2507 } 2508 break; 2509 2510 OP_32_64(div2): 2511 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]); 2512 break; 2513 OP_32_64(divu2): 2514 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]); 2515 break; 2516 2517 OP_32_64(shl): 2518 /* For small constant 3-operand shift, use LEA. */ 2519 if (const_a2 && a0 != a1 && (a2 - 1) < 3) { 2520 if (a2 - 1 == 0) { 2521 /* shl $1,a1,a0 -> lea (a1,a1),a0 */ 2522 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a1, 0, 0); 2523 } else { 2524 /* shl $n,a1,a0 -> lea 0(,a1,n),a0 */ 2525 tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, -1, a1, a2, 0); 2526 } 2527 break; 2528 } 2529 c = SHIFT_SHL; 2530 vexop = OPC_SHLX; 2531 goto gen_shift_maybe_vex; 2532 OP_32_64(shr): 2533 c = SHIFT_SHR; 2534 vexop = OPC_SHRX; 2535 goto gen_shift_maybe_vex; 2536 OP_32_64(sar): 2537 c = SHIFT_SAR; 2538 vexop = OPC_SARX; 2539 goto gen_shift_maybe_vex; 2540 OP_32_64(rotl): 2541 c = SHIFT_ROL; 2542 goto gen_shift; 2543 OP_32_64(rotr): 2544 c = SHIFT_ROR; 2545 goto gen_shift; 2546 gen_shift_maybe_vex: 2547 if (have_bmi2) { 2548 if (!const_a2) { 2549 tcg_out_vex_modrm(s, vexop + rexw, a0, a2, a1); 2550 break; 2551 } 2552 tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32, a0, a1); 2553 } 2554 /* FALLTHRU */ 2555 gen_shift: 2556 if (const_a2) { 2557 tcg_out_shifti(s, c + rexw, a0, a2); 2558 } else { 2559 tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, a0); 2560 } 2561 break; 2562 2563 OP_32_64(ctz): 2564 tcg_out_ctz(s, rexw, args[0], args[1], args[2], const_args[2]); 2565 break; 2566 OP_32_64(clz): 2567 tcg_out_clz(s, rexw, args[0], args[1], args[2], const_args[2]); 2568 break; 2569 OP_32_64(ctpop): 2570 tcg_out_modrm(s, OPC_POPCNT + rexw, a0, a1); 2571 break; 2572 2573 case INDEX_op_brcond_i32: 2574 tcg_out_brcond32(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0); 2575 break; 2576 case INDEX_op_setcond_i32: 2577 tcg_out_setcond32(s, args[3], a0, a1, a2, const_a2); 2578 break; 2579 case INDEX_op_movcond_i32: 2580 tcg_out_movcond32(s, args[5], a0, a1, a2, const_a2, args[3]); 2581 break; 2582 2583 OP_32_64(bswap16): 2584 if (a2 & TCG_BSWAP_OS) { 2585 /* Output must be sign-extended. */ 2586 if (rexw) { 2587 tcg_out_bswap64(s, a0); 2588 tcg_out_shifti(s, SHIFT_SAR + rexw, a0, 48); 2589 } else { 2590 tcg_out_bswap32(s, a0); 2591 tcg_out_shifti(s, SHIFT_SAR, a0, 16); 2592 } 2593 } else if ((a2 & (TCG_BSWAP_IZ | TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) { 2594 /* Output must be zero-extended, but input isn't. */ 2595 tcg_out_bswap32(s, a0); 2596 tcg_out_shifti(s, SHIFT_SHR, a0, 16); 2597 } else { 2598 tcg_out_rolw_8(s, a0); 2599 } 2600 break; 2601 OP_32_64(bswap32): 2602 tcg_out_bswap32(s, a0); 2603 if (rexw && (a2 & TCG_BSWAP_OS)) { 2604 tcg_out_ext32s(s, a0, a0); 2605 } 2606 break; 2607 2608 OP_32_64(neg): 2609 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, a0); 2610 break; 2611 OP_32_64(not): 2612 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, a0); 2613 break; 2614 2615 case INDEX_op_qemu_ld_a64_i32: 2616 if (TCG_TARGET_REG_BITS == 32) { 2617 tcg_out_qemu_ld(s, a0, -1, a1, a2, args[3], TCG_TYPE_I32); 2618 break; 2619 } 2620 /* fall through */ 2621 case INDEX_op_qemu_ld_a32_i32: 2622 tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); 2623 break; 2624 case INDEX_op_qemu_ld_a32_i64: 2625 if (TCG_TARGET_REG_BITS == 64) { 2626 tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); 2627 } else { 2628 tcg_out_qemu_ld(s, a0, a1, a2, -1, args[3], TCG_TYPE_I64); 2629 } 2630 break; 2631 case INDEX_op_qemu_ld_a64_i64: 2632 if (TCG_TARGET_REG_BITS == 64) { 2633 tcg_out_qemu_ld(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); 2634 } else { 2635 tcg_out_qemu_ld(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); 2636 } 2637 break; 2638 case INDEX_op_qemu_ld_a32_i128: 2639 case INDEX_op_qemu_ld_a64_i128: 2640 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 2641 tcg_out_qemu_ld(s, a0, a1, a2, -1, args[3], TCG_TYPE_I128); 2642 break; 2643 2644 case INDEX_op_qemu_st_a64_i32: 2645 case INDEX_op_qemu_st8_a64_i32: 2646 if (TCG_TARGET_REG_BITS == 32) { 2647 tcg_out_qemu_st(s, a0, -1, a1, a2, args[3], TCG_TYPE_I32); 2648 break; 2649 } 2650 /* fall through */ 2651 case INDEX_op_qemu_st_a32_i32: 2652 case INDEX_op_qemu_st8_a32_i32: 2653 tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I32); 2654 break; 2655 case INDEX_op_qemu_st_a32_i64: 2656 if (TCG_TARGET_REG_BITS == 64) { 2657 tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); 2658 } else { 2659 tcg_out_qemu_st(s, a0, a1, a2, -1, args[3], TCG_TYPE_I64); 2660 } 2661 break; 2662 case INDEX_op_qemu_st_a64_i64: 2663 if (TCG_TARGET_REG_BITS == 64) { 2664 tcg_out_qemu_st(s, a0, -1, a1, -1, a2, TCG_TYPE_I64); 2665 } else { 2666 tcg_out_qemu_st(s, a0, a1, a2, args[3], args[4], TCG_TYPE_I64); 2667 } 2668 break; 2669 case INDEX_op_qemu_st_a32_i128: 2670 case INDEX_op_qemu_st_a64_i128: 2671 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 2672 tcg_out_qemu_st(s, a0, a1, a2, -1, args[3], TCG_TYPE_I128); 2673 break; 2674 2675 OP_32_64(mulu2): 2676 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]); 2677 break; 2678 OP_32_64(muls2): 2679 tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, args[3]); 2680 break; 2681 OP_32_64(add2): 2682 if (const_args[4]) { 2683 tgen_arithi(s, ARITH_ADD + rexw, a0, args[4], 1); 2684 } else { 2685 tgen_arithr(s, ARITH_ADD + rexw, a0, args[4]); 2686 } 2687 if (const_args[5]) { 2688 tgen_arithi(s, ARITH_ADC + rexw, a1, args[5], 1); 2689 } else { 2690 tgen_arithr(s, ARITH_ADC + rexw, a1, args[5]); 2691 } 2692 break; 2693 OP_32_64(sub2): 2694 if (const_args[4]) { 2695 tgen_arithi(s, ARITH_SUB + rexw, a0, args[4], 1); 2696 } else { 2697 tgen_arithr(s, ARITH_SUB + rexw, a0, args[4]); 2698 } 2699 if (const_args[5]) { 2700 tgen_arithi(s, ARITH_SBB + rexw, a1, args[5], 1); 2701 } else { 2702 tgen_arithr(s, ARITH_SBB + rexw, a1, args[5]); 2703 } 2704 break; 2705 2706#if TCG_TARGET_REG_BITS == 32 2707 case INDEX_op_brcond2_i32: 2708 tcg_out_brcond2(s, args, const_args, 0); 2709 break; 2710 case INDEX_op_setcond2_i32: 2711 tcg_out_setcond2(s, args, const_args); 2712 break; 2713#else /* TCG_TARGET_REG_BITS == 64 */ 2714 case INDEX_op_ld32s_i64: 2715 tcg_out_modrm_offset(s, OPC_MOVSLQ, a0, a1, a2); 2716 break; 2717 case INDEX_op_ld_i64: 2718 tcg_out_ld(s, TCG_TYPE_I64, a0, a1, a2); 2719 break; 2720 case INDEX_op_st_i64: 2721 if (const_args[0]) { 2722 tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW, 0, a1, a2); 2723 tcg_out32(s, a0); 2724 } else { 2725 tcg_out_st(s, TCG_TYPE_I64, a0, a1, a2); 2726 } 2727 break; 2728 2729 case INDEX_op_brcond_i64: 2730 tcg_out_brcond64(s, a2, a0, a1, const_args[1], arg_label(args[3]), 0); 2731 break; 2732 case INDEX_op_setcond_i64: 2733 tcg_out_setcond64(s, args[3], a0, a1, a2, const_a2); 2734 break; 2735 case INDEX_op_movcond_i64: 2736 tcg_out_movcond64(s, args[5], a0, a1, a2, const_a2, args[3]); 2737 break; 2738 2739 case INDEX_op_bswap64_i64: 2740 tcg_out_bswap64(s, a0); 2741 break; 2742 case INDEX_op_extrh_i64_i32: 2743 tcg_out_shifti(s, SHIFT_SHR + P_REXW, a0, 32); 2744 break; 2745#endif 2746 2747 OP_32_64(deposit): 2748 if (args[3] == 0 && args[4] == 8) { 2749 /* load bits 0..7 */ 2750 tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM, a2, a0); 2751 } else if (args[3] == 8 && args[4] == 8) { 2752 /* load bits 8..15 */ 2753 tcg_out_modrm(s, OPC_MOVB_EvGv, a2, a0 + 4); 2754 } else if (args[3] == 0 && args[4] == 16) { 2755 /* load bits 0..15 */ 2756 tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, a2, a0); 2757 } else { 2758 g_assert_not_reached(); 2759 } 2760 break; 2761 2762 case INDEX_op_extract_i64: 2763 if (a2 + args[3] == 32) { 2764 /* This is a 32-bit zero-extending right shift. */ 2765 tcg_out_mov(s, TCG_TYPE_I32, a0, a1); 2766 tcg_out_shifti(s, SHIFT_SHR, a0, a2); 2767 break; 2768 } 2769 /* FALLTHRU */ 2770 case INDEX_op_extract_i32: 2771 /* On the off-chance that we can use the high-byte registers. 2772 Otherwise we emit the same ext16 + shift pattern that we 2773 would have gotten from the normal tcg-op.c expansion. */ 2774 tcg_debug_assert(a2 == 8 && args[3] == 8); 2775 if (a1 < 4 && a0 < 8) { 2776 tcg_out_modrm(s, OPC_MOVZBL, a0, a1 + 4); 2777 } else { 2778 tcg_out_ext16u(s, a0, a1); 2779 tcg_out_shifti(s, SHIFT_SHR, a0, 8); 2780 } 2781 break; 2782 2783 case INDEX_op_sextract_i32: 2784 /* We don't implement sextract_i64, as we cannot sign-extend to 2785 64-bits without using the REX prefix that explicitly excludes 2786 access to the high-byte registers. */ 2787 tcg_debug_assert(a2 == 8 && args[3] == 8); 2788 if (a1 < 4 && a0 < 8) { 2789 tcg_out_modrm(s, OPC_MOVSBL, a0, a1 + 4); 2790 } else { 2791 tcg_out_ext16s(s, TCG_TYPE_I32, a0, a1); 2792 tcg_out_shifti(s, SHIFT_SAR, a0, 8); 2793 } 2794 break; 2795 2796 OP_32_64(extract2): 2797 /* Note that SHRD outputs to the r/m operand. */ 2798 tcg_out_modrm(s, OPC_SHRD_Ib + rexw, a2, a0); 2799 tcg_out8(s, args[3]); 2800 break; 2801 2802 case INDEX_op_mb: 2803 tcg_out_mb(s, a0); 2804 break; 2805 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ 2806 case INDEX_op_mov_i64: 2807 case INDEX_op_call: /* Always emitted via tcg_out_call. */ 2808 case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ 2809 case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ 2810 case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ 2811 case INDEX_op_ext8s_i64: 2812 case INDEX_op_ext8u_i32: 2813 case INDEX_op_ext8u_i64: 2814 case INDEX_op_ext16s_i32: 2815 case INDEX_op_ext16s_i64: 2816 case INDEX_op_ext16u_i32: 2817 case INDEX_op_ext16u_i64: 2818 case INDEX_op_ext32s_i64: 2819 case INDEX_op_ext32u_i64: 2820 case INDEX_op_ext_i32_i64: 2821 case INDEX_op_extu_i32_i64: 2822 case INDEX_op_extrl_i64_i32: 2823 default: 2824 g_assert_not_reached(); 2825 } 2826 2827#undef OP_32_64 2828} 2829 2830static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 2831 unsigned vecl, unsigned vece, 2832 const TCGArg args[TCG_MAX_OP_ARGS], 2833 const int const_args[TCG_MAX_OP_ARGS]) 2834{ 2835 static int const add_insn[4] = { 2836 OPC_PADDB, OPC_PADDW, OPC_PADDD, OPC_PADDQ 2837 }; 2838 static int const ssadd_insn[4] = { 2839 OPC_PADDSB, OPC_PADDSW, OPC_UD2, OPC_UD2 2840 }; 2841 static int const usadd_insn[4] = { 2842 OPC_PADDUB, OPC_PADDUW, OPC_UD2, OPC_UD2 2843 }; 2844 static int const sub_insn[4] = { 2845 OPC_PSUBB, OPC_PSUBW, OPC_PSUBD, OPC_PSUBQ 2846 }; 2847 static int const sssub_insn[4] = { 2848 OPC_PSUBSB, OPC_PSUBSW, OPC_UD2, OPC_UD2 2849 }; 2850 static int const ussub_insn[4] = { 2851 OPC_PSUBUB, OPC_PSUBUW, OPC_UD2, OPC_UD2 2852 }; 2853 static int const mul_insn[4] = { 2854 OPC_UD2, OPC_PMULLW, OPC_PMULLD, OPC_VPMULLQ 2855 }; 2856 static int const shift_imm_insn[4] = { 2857 OPC_UD2, OPC_PSHIFTW_Ib, OPC_PSHIFTD_Ib, OPC_PSHIFTQ_Ib 2858 }; 2859 static int const cmpeq_insn[4] = { 2860 OPC_PCMPEQB, OPC_PCMPEQW, OPC_PCMPEQD, OPC_PCMPEQQ 2861 }; 2862 static int const cmpgt_insn[4] = { 2863 OPC_PCMPGTB, OPC_PCMPGTW, OPC_PCMPGTD, OPC_PCMPGTQ 2864 }; 2865 static int const punpckl_insn[4] = { 2866 OPC_PUNPCKLBW, OPC_PUNPCKLWD, OPC_PUNPCKLDQ, OPC_PUNPCKLQDQ 2867 }; 2868 static int const punpckh_insn[4] = { 2869 OPC_PUNPCKHBW, OPC_PUNPCKHWD, OPC_PUNPCKHDQ, OPC_PUNPCKHQDQ 2870 }; 2871 static int const packss_insn[4] = { 2872 OPC_PACKSSWB, OPC_PACKSSDW, OPC_UD2, OPC_UD2 2873 }; 2874 static int const packus_insn[4] = { 2875 OPC_PACKUSWB, OPC_PACKUSDW, OPC_UD2, OPC_UD2 2876 }; 2877 static int const smin_insn[4] = { 2878 OPC_PMINSB, OPC_PMINSW, OPC_PMINSD, OPC_VPMINSQ 2879 }; 2880 static int const smax_insn[4] = { 2881 OPC_PMAXSB, OPC_PMAXSW, OPC_PMAXSD, OPC_VPMAXSQ 2882 }; 2883 static int const umin_insn[4] = { 2884 OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ 2885 }; 2886 static int const umax_insn[4] = { 2887 OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_VPMAXUQ 2888 }; 2889 static int const rotlv_insn[4] = { 2890 OPC_UD2, OPC_UD2, OPC_VPROLVD, OPC_VPROLVQ 2891 }; 2892 static int const rotrv_insn[4] = { 2893 OPC_UD2, OPC_UD2, OPC_VPRORVD, OPC_VPRORVQ 2894 }; 2895 static int const shlv_insn[4] = { 2896 OPC_UD2, OPC_VPSLLVW, OPC_VPSLLVD, OPC_VPSLLVQ 2897 }; 2898 static int const shrv_insn[4] = { 2899 OPC_UD2, OPC_VPSRLVW, OPC_VPSRLVD, OPC_VPSRLVQ 2900 }; 2901 static int const sarv_insn[4] = { 2902 OPC_UD2, OPC_VPSRAVW, OPC_VPSRAVD, OPC_VPSRAVQ 2903 }; 2904 static int const shls_insn[4] = { 2905 OPC_UD2, OPC_PSLLW, OPC_PSLLD, OPC_PSLLQ 2906 }; 2907 static int const shrs_insn[4] = { 2908 OPC_UD2, OPC_PSRLW, OPC_PSRLD, OPC_PSRLQ 2909 }; 2910 static int const sars_insn[4] = { 2911 OPC_UD2, OPC_PSRAW, OPC_PSRAD, OPC_VPSRAQ 2912 }; 2913 static int const vpshldi_insn[4] = { 2914 OPC_UD2, OPC_VPSHLDW, OPC_VPSHLDD, OPC_VPSHLDQ 2915 }; 2916 static int const vpshldv_insn[4] = { 2917 OPC_UD2, OPC_VPSHLDVW, OPC_VPSHLDVD, OPC_VPSHLDVQ 2918 }; 2919 static int const vpshrdv_insn[4] = { 2920 OPC_UD2, OPC_VPSHRDVW, OPC_VPSHRDVD, OPC_VPSHRDVQ 2921 }; 2922 static int const abs_insn[4] = { 2923 OPC_PABSB, OPC_PABSW, OPC_PABSD, OPC_VPABSQ 2924 }; 2925 2926 TCGType type = vecl + TCG_TYPE_V64; 2927 int insn, sub; 2928 TCGArg a0, a1, a2, a3; 2929 2930 a0 = args[0]; 2931 a1 = args[1]; 2932 a2 = args[2]; 2933 2934 switch (opc) { 2935 case INDEX_op_add_vec: 2936 insn = add_insn[vece]; 2937 goto gen_simd; 2938 case INDEX_op_ssadd_vec: 2939 insn = ssadd_insn[vece]; 2940 goto gen_simd; 2941 case INDEX_op_usadd_vec: 2942 insn = usadd_insn[vece]; 2943 goto gen_simd; 2944 case INDEX_op_sub_vec: 2945 insn = sub_insn[vece]; 2946 goto gen_simd; 2947 case INDEX_op_sssub_vec: 2948 insn = sssub_insn[vece]; 2949 goto gen_simd; 2950 case INDEX_op_ussub_vec: 2951 insn = ussub_insn[vece]; 2952 goto gen_simd; 2953 case INDEX_op_mul_vec: 2954 insn = mul_insn[vece]; 2955 goto gen_simd; 2956 case INDEX_op_and_vec: 2957 insn = OPC_PAND; 2958 goto gen_simd; 2959 case INDEX_op_or_vec: 2960 insn = OPC_POR; 2961 goto gen_simd; 2962 case INDEX_op_xor_vec: 2963 insn = OPC_PXOR; 2964 goto gen_simd; 2965 case INDEX_op_smin_vec: 2966 insn = smin_insn[vece]; 2967 goto gen_simd; 2968 case INDEX_op_umin_vec: 2969 insn = umin_insn[vece]; 2970 goto gen_simd; 2971 case INDEX_op_smax_vec: 2972 insn = smax_insn[vece]; 2973 goto gen_simd; 2974 case INDEX_op_umax_vec: 2975 insn = umax_insn[vece]; 2976 goto gen_simd; 2977 case INDEX_op_shlv_vec: 2978 insn = shlv_insn[vece]; 2979 goto gen_simd; 2980 case INDEX_op_shrv_vec: 2981 insn = shrv_insn[vece]; 2982 goto gen_simd; 2983 case INDEX_op_sarv_vec: 2984 insn = sarv_insn[vece]; 2985 goto gen_simd; 2986 case INDEX_op_rotlv_vec: 2987 insn = rotlv_insn[vece]; 2988 goto gen_simd; 2989 case INDEX_op_rotrv_vec: 2990 insn = rotrv_insn[vece]; 2991 goto gen_simd; 2992 case INDEX_op_shls_vec: 2993 insn = shls_insn[vece]; 2994 goto gen_simd; 2995 case INDEX_op_shrs_vec: 2996 insn = shrs_insn[vece]; 2997 goto gen_simd; 2998 case INDEX_op_sars_vec: 2999 insn = sars_insn[vece]; 3000 goto gen_simd; 3001 case INDEX_op_x86_punpckl_vec: 3002 insn = punpckl_insn[vece]; 3003 goto gen_simd; 3004 case INDEX_op_x86_punpckh_vec: 3005 insn = punpckh_insn[vece]; 3006 goto gen_simd; 3007 case INDEX_op_x86_packss_vec: 3008 insn = packss_insn[vece]; 3009 goto gen_simd; 3010 case INDEX_op_x86_packus_vec: 3011 insn = packus_insn[vece]; 3012 goto gen_simd; 3013 case INDEX_op_x86_vpshldv_vec: 3014 insn = vpshldv_insn[vece]; 3015 a1 = a2; 3016 a2 = args[3]; 3017 goto gen_simd; 3018 case INDEX_op_x86_vpshrdv_vec: 3019 insn = vpshrdv_insn[vece]; 3020 a1 = a2; 3021 a2 = args[3]; 3022 goto gen_simd; 3023#if TCG_TARGET_REG_BITS == 32 3024 case INDEX_op_dup2_vec: 3025 /* First merge the two 32-bit inputs to a single 64-bit element. */ 3026 tcg_out_vex_modrm(s, OPC_PUNPCKLDQ, a0, a1, a2); 3027 /* Then replicate the 64-bit elements across the rest of the vector. */ 3028 if (type != TCG_TYPE_V64) { 3029 tcg_out_dup_vec(s, type, MO_64, a0, a0); 3030 } 3031 break; 3032#endif 3033 case INDEX_op_abs_vec: 3034 insn = abs_insn[vece]; 3035 a2 = a1; 3036 a1 = 0; 3037 goto gen_simd; 3038 gen_simd: 3039 tcg_debug_assert(insn != OPC_UD2); 3040 if (type == TCG_TYPE_V256) { 3041 insn |= P_VEXL; 3042 } 3043 tcg_out_vex_modrm(s, insn, a0, a1, a2); 3044 break; 3045 3046 case INDEX_op_cmp_vec: 3047 sub = args[3]; 3048 if (sub == TCG_COND_EQ) { 3049 insn = cmpeq_insn[vece]; 3050 } else if (sub == TCG_COND_GT) { 3051 insn = cmpgt_insn[vece]; 3052 } else { 3053 g_assert_not_reached(); 3054 } 3055 goto gen_simd; 3056 3057 case INDEX_op_andc_vec: 3058 insn = OPC_PANDN; 3059 if (type == TCG_TYPE_V256) { 3060 insn |= P_VEXL; 3061 } 3062 tcg_out_vex_modrm(s, insn, a0, a2, a1); 3063 break; 3064 3065 case INDEX_op_shli_vec: 3066 insn = shift_imm_insn[vece]; 3067 sub = 6; 3068 goto gen_shift; 3069 case INDEX_op_shri_vec: 3070 insn = shift_imm_insn[vece]; 3071 sub = 2; 3072 goto gen_shift; 3073 case INDEX_op_sari_vec: 3074 if (vece == MO_64) { 3075 insn = OPC_PSHIFTD_Ib | P_VEXW | P_EVEX; 3076 } else { 3077 insn = shift_imm_insn[vece]; 3078 } 3079 sub = 4; 3080 goto gen_shift; 3081 case INDEX_op_rotli_vec: 3082 insn = OPC_PSHIFTD_Ib | P_EVEX; /* VPROL[DQ] */ 3083 if (vece == MO_64) { 3084 insn |= P_VEXW; 3085 } 3086 sub = 1; 3087 goto gen_shift; 3088 gen_shift: 3089 tcg_debug_assert(vece != MO_8); 3090 if (type == TCG_TYPE_V256) { 3091 insn |= P_VEXL; 3092 } 3093 tcg_out_vex_modrm(s, insn, sub, a0, a1); 3094 tcg_out8(s, a2); 3095 break; 3096 3097 case INDEX_op_ld_vec: 3098 tcg_out_ld(s, type, a0, a1, a2); 3099 break; 3100 case INDEX_op_st_vec: 3101 tcg_out_st(s, type, a0, a1, a2); 3102 break; 3103 case INDEX_op_dupm_vec: 3104 tcg_out_dupm_vec(s, type, vece, a0, a1, a2); 3105 break; 3106 3107 case INDEX_op_x86_shufps_vec: 3108 insn = OPC_SHUFPS; 3109 sub = args[3]; 3110 goto gen_simd_imm8; 3111 case INDEX_op_x86_blend_vec: 3112 if (vece == MO_16) { 3113 insn = OPC_PBLENDW; 3114 } else if (vece == MO_32) { 3115 insn = (have_avx2 ? OPC_VPBLENDD : OPC_BLENDPS); 3116 } else { 3117 g_assert_not_reached(); 3118 } 3119 sub = args[3]; 3120 goto gen_simd_imm8; 3121 case INDEX_op_x86_vperm2i128_vec: 3122 insn = OPC_VPERM2I128; 3123 sub = args[3]; 3124 goto gen_simd_imm8; 3125 case INDEX_op_x86_vpshldi_vec: 3126 insn = vpshldi_insn[vece]; 3127 sub = args[3]; 3128 goto gen_simd_imm8; 3129 3130 case INDEX_op_not_vec: 3131 insn = OPC_VPTERNLOGQ; 3132 a2 = a1; 3133 sub = 0x33; /* !B */ 3134 goto gen_simd_imm8; 3135 case INDEX_op_nor_vec: 3136 insn = OPC_VPTERNLOGQ; 3137 sub = 0x11; /* norCB */ 3138 goto gen_simd_imm8; 3139 case INDEX_op_nand_vec: 3140 insn = OPC_VPTERNLOGQ; 3141 sub = 0x77; /* nandCB */ 3142 goto gen_simd_imm8; 3143 case INDEX_op_eqv_vec: 3144 insn = OPC_VPTERNLOGQ; 3145 sub = 0x99; /* xnorCB */ 3146 goto gen_simd_imm8; 3147 case INDEX_op_orc_vec: 3148 insn = OPC_VPTERNLOGQ; 3149 sub = 0xdd; /* orB!C */ 3150 goto gen_simd_imm8; 3151 3152 case INDEX_op_bitsel_vec: 3153 insn = OPC_VPTERNLOGQ; 3154 a3 = args[3]; 3155 if (a0 == a1) { 3156 a1 = a2; 3157 a2 = a3; 3158 sub = 0xca; /* A?B:C */ 3159 } else if (a0 == a2) { 3160 a2 = a3; 3161 sub = 0xe2; /* B?A:C */ 3162 } else { 3163 tcg_out_mov(s, type, a0, a3); 3164 sub = 0xb8; /* B?C:A */ 3165 } 3166 goto gen_simd_imm8; 3167 3168 gen_simd_imm8: 3169 tcg_debug_assert(insn != OPC_UD2); 3170 if (type == TCG_TYPE_V256) { 3171 insn |= P_VEXL; 3172 } 3173 tcg_out_vex_modrm(s, insn, a0, a1, a2); 3174 tcg_out8(s, sub); 3175 break; 3176 3177 case INDEX_op_x86_vpblendvb_vec: 3178 insn = OPC_VPBLENDVB; 3179 if (type == TCG_TYPE_V256) { 3180 insn |= P_VEXL; 3181 } 3182 tcg_out_vex_modrm(s, insn, a0, a1, a2); 3183 tcg_out8(s, args[3] << 4); 3184 break; 3185 3186 case INDEX_op_x86_psrldq_vec: 3187 tcg_out_vex_modrm(s, OPC_GRP14, 3, a0, a1); 3188 tcg_out8(s, a2); 3189 break; 3190 3191 case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */ 3192 case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */ 3193 default: 3194 g_assert_not_reached(); 3195 } 3196} 3197 3198static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) 3199{ 3200 switch (op) { 3201 case INDEX_op_goto_ptr: 3202 return C_O0_I1(r); 3203 3204 case INDEX_op_ld8u_i32: 3205 case INDEX_op_ld8u_i64: 3206 case INDEX_op_ld8s_i32: 3207 case INDEX_op_ld8s_i64: 3208 case INDEX_op_ld16u_i32: 3209 case INDEX_op_ld16u_i64: 3210 case INDEX_op_ld16s_i32: 3211 case INDEX_op_ld16s_i64: 3212 case INDEX_op_ld_i32: 3213 case INDEX_op_ld32u_i64: 3214 case INDEX_op_ld32s_i64: 3215 case INDEX_op_ld_i64: 3216 return C_O1_I1(r, r); 3217 3218 case INDEX_op_st8_i32: 3219 case INDEX_op_st8_i64: 3220 return C_O0_I2(qi, r); 3221 3222 case INDEX_op_st16_i32: 3223 case INDEX_op_st16_i64: 3224 case INDEX_op_st_i32: 3225 case INDEX_op_st32_i64: 3226 return C_O0_I2(ri, r); 3227 3228 case INDEX_op_st_i64: 3229 return C_O0_I2(re, r); 3230 3231 case INDEX_op_add_i32: 3232 case INDEX_op_add_i64: 3233 return C_O1_I2(r, r, re); 3234 3235 case INDEX_op_sub_i32: 3236 case INDEX_op_sub_i64: 3237 case INDEX_op_mul_i32: 3238 case INDEX_op_mul_i64: 3239 case INDEX_op_or_i32: 3240 case INDEX_op_or_i64: 3241 case INDEX_op_xor_i32: 3242 case INDEX_op_xor_i64: 3243 return C_O1_I2(r, 0, re); 3244 3245 case INDEX_op_and_i32: 3246 case INDEX_op_and_i64: 3247 return C_O1_I2(r, 0, reZ); 3248 3249 case INDEX_op_andc_i32: 3250 case INDEX_op_andc_i64: 3251 return C_O1_I2(r, r, rI); 3252 3253 case INDEX_op_shl_i32: 3254 case INDEX_op_shl_i64: 3255 case INDEX_op_shr_i32: 3256 case INDEX_op_shr_i64: 3257 case INDEX_op_sar_i32: 3258 case INDEX_op_sar_i64: 3259 return have_bmi2 ? C_O1_I2(r, r, ri) : C_O1_I2(r, 0, ci); 3260 3261 case INDEX_op_rotl_i32: 3262 case INDEX_op_rotl_i64: 3263 case INDEX_op_rotr_i32: 3264 case INDEX_op_rotr_i64: 3265 return C_O1_I2(r, 0, ci); 3266 3267 case INDEX_op_brcond_i32: 3268 case INDEX_op_brcond_i64: 3269 return C_O0_I2(r, re); 3270 3271 case INDEX_op_bswap16_i32: 3272 case INDEX_op_bswap16_i64: 3273 case INDEX_op_bswap32_i32: 3274 case INDEX_op_bswap32_i64: 3275 case INDEX_op_bswap64_i64: 3276 case INDEX_op_neg_i32: 3277 case INDEX_op_neg_i64: 3278 case INDEX_op_not_i32: 3279 case INDEX_op_not_i64: 3280 case INDEX_op_extrh_i64_i32: 3281 return C_O1_I1(r, 0); 3282 3283 case INDEX_op_ext8s_i32: 3284 case INDEX_op_ext8s_i64: 3285 case INDEX_op_ext8u_i32: 3286 case INDEX_op_ext8u_i64: 3287 return C_O1_I1(r, q); 3288 3289 case INDEX_op_ext16s_i32: 3290 case INDEX_op_ext16s_i64: 3291 case INDEX_op_ext16u_i32: 3292 case INDEX_op_ext16u_i64: 3293 case INDEX_op_ext32s_i64: 3294 case INDEX_op_ext32u_i64: 3295 case INDEX_op_ext_i32_i64: 3296 case INDEX_op_extu_i32_i64: 3297 case INDEX_op_extrl_i64_i32: 3298 case INDEX_op_extract_i32: 3299 case INDEX_op_extract_i64: 3300 case INDEX_op_sextract_i32: 3301 case INDEX_op_ctpop_i32: 3302 case INDEX_op_ctpop_i64: 3303 return C_O1_I1(r, r); 3304 3305 case INDEX_op_extract2_i32: 3306 case INDEX_op_extract2_i64: 3307 return C_O1_I2(r, 0, r); 3308 3309 case INDEX_op_deposit_i32: 3310 case INDEX_op_deposit_i64: 3311 return C_O1_I2(Q, 0, Q); 3312 3313 case INDEX_op_setcond_i32: 3314 case INDEX_op_setcond_i64: 3315 return C_O1_I2(q, r, re); 3316 3317 case INDEX_op_movcond_i32: 3318 case INDEX_op_movcond_i64: 3319 return C_O1_I4(r, r, re, r, 0); 3320 3321 case INDEX_op_div2_i32: 3322 case INDEX_op_div2_i64: 3323 case INDEX_op_divu2_i32: 3324 case INDEX_op_divu2_i64: 3325 return C_O2_I3(a, d, 0, 1, r); 3326 3327 case INDEX_op_mulu2_i32: 3328 case INDEX_op_mulu2_i64: 3329 case INDEX_op_muls2_i32: 3330 case INDEX_op_muls2_i64: 3331 return C_O2_I2(a, d, a, r); 3332 3333 case INDEX_op_add2_i32: 3334 case INDEX_op_add2_i64: 3335 case INDEX_op_sub2_i32: 3336 case INDEX_op_sub2_i64: 3337 return C_O2_I4(r, r, 0, 1, re, re); 3338 3339 case INDEX_op_ctz_i32: 3340 case INDEX_op_ctz_i64: 3341 return have_bmi1 ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); 3342 3343 case INDEX_op_clz_i32: 3344 case INDEX_op_clz_i64: 3345 return have_lzcnt ? C_N1_I2(r, r, rW) : C_N1_I2(r, r, r); 3346 3347 case INDEX_op_qemu_ld_a32_i32: 3348 return C_O1_I1(r, L); 3349 case INDEX_op_qemu_ld_a64_i32: 3350 return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) : C_O1_I2(r, L, L); 3351 3352 case INDEX_op_qemu_st_a32_i32: 3353 return C_O0_I2(L, L); 3354 case INDEX_op_qemu_st_a64_i32: 3355 return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) : C_O0_I3(L, L, L); 3356 case INDEX_op_qemu_st8_a32_i32: 3357 return C_O0_I2(s, L); 3358 case INDEX_op_qemu_st8_a64_i32: 3359 return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(s, L) : C_O0_I3(s, L, L); 3360 3361 case INDEX_op_qemu_ld_a32_i64: 3362 return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) : C_O2_I1(r, r, L); 3363 case INDEX_op_qemu_ld_a64_i64: 3364 return TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) : C_O2_I2(r, r, L, L); 3365 3366 case INDEX_op_qemu_st_a32_i64: 3367 return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) : C_O0_I3(L, L, L); 3368 case INDEX_op_qemu_st_a64_i64: 3369 return TCG_TARGET_REG_BITS == 64 ? C_O0_I2(L, L) : C_O0_I4(L, L, L, L); 3370 3371 case INDEX_op_qemu_ld_a32_i128: 3372 case INDEX_op_qemu_ld_a64_i128: 3373 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 3374 return C_O2_I1(r, r, L); 3375 case INDEX_op_qemu_st_a32_i128: 3376 case INDEX_op_qemu_st_a64_i128: 3377 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 3378 return C_O0_I3(L, L, L); 3379 3380 case INDEX_op_brcond2_i32: 3381 return C_O0_I4(r, r, ri, ri); 3382 3383 case INDEX_op_setcond2_i32: 3384 return C_O1_I4(r, r, r, ri, ri); 3385 3386 case INDEX_op_ld_vec: 3387 case INDEX_op_dupm_vec: 3388 return C_O1_I1(x, r); 3389 3390 case INDEX_op_st_vec: 3391 return C_O0_I2(x, r); 3392 3393 case INDEX_op_add_vec: 3394 case INDEX_op_sub_vec: 3395 case INDEX_op_mul_vec: 3396 case INDEX_op_and_vec: 3397 case INDEX_op_or_vec: 3398 case INDEX_op_xor_vec: 3399 case INDEX_op_andc_vec: 3400 case INDEX_op_orc_vec: 3401 case INDEX_op_nand_vec: 3402 case INDEX_op_nor_vec: 3403 case INDEX_op_eqv_vec: 3404 case INDEX_op_ssadd_vec: 3405 case INDEX_op_usadd_vec: 3406 case INDEX_op_sssub_vec: 3407 case INDEX_op_ussub_vec: 3408 case INDEX_op_smin_vec: 3409 case INDEX_op_umin_vec: 3410 case INDEX_op_smax_vec: 3411 case INDEX_op_umax_vec: 3412 case INDEX_op_shlv_vec: 3413 case INDEX_op_shrv_vec: 3414 case INDEX_op_sarv_vec: 3415 case INDEX_op_rotlv_vec: 3416 case INDEX_op_rotrv_vec: 3417 case INDEX_op_shls_vec: 3418 case INDEX_op_shrs_vec: 3419 case INDEX_op_sars_vec: 3420 case INDEX_op_cmp_vec: 3421 case INDEX_op_x86_shufps_vec: 3422 case INDEX_op_x86_blend_vec: 3423 case INDEX_op_x86_packss_vec: 3424 case INDEX_op_x86_packus_vec: 3425 case INDEX_op_x86_vperm2i128_vec: 3426 case INDEX_op_x86_punpckl_vec: 3427 case INDEX_op_x86_punpckh_vec: 3428 case INDEX_op_x86_vpshldi_vec: 3429#if TCG_TARGET_REG_BITS == 32 3430 case INDEX_op_dup2_vec: 3431#endif 3432 return C_O1_I2(x, x, x); 3433 3434 case INDEX_op_abs_vec: 3435 case INDEX_op_dup_vec: 3436 case INDEX_op_not_vec: 3437 case INDEX_op_shli_vec: 3438 case INDEX_op_shri_vec: 3439 case INDEX_op_sari_vec: 3440 case INDEX_op_rotli_vec: 3441 case INDEX_op_x86_psrldq_vec: 3442 return C_O1_I1(x, x); 3443 3444 case INDEX_op_x86_vpshldv_vec: 3445 case INDEX_op_x86_vpshrdv_vec: 3446 return C_O1_I3(x, 0, x, x); 3447 3448 case INDEX_op_bitsel_vec: 3449 case INDEX_op_x86_vpblendvb_vec: 3450 return C_O1_I3(x, x, x, x); 3451 3452 default: 3453 g_assert_not_reached(); 3454 } 3455} 3456 3457int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) 3458{ 3459 switch (opc) { 3460 case INDEX_op_add_vec: 3461 case INDEX_op_sub_vec: 3462 case INDEX_op_and_vec: 3463 case INDEX_op_or_vec: 3464 case INDEX_op_xor_vec: 3465 case INDEX_op_andc_vec: 3466 case INDEX_op_orc_vec: 3467 case INDEX_op_nand_vec: 3468 case INDEX_op_nor_vec: 3469 case INDEX_op_eqv_vec: 3470 case INDEX_op_not_vec: 3471 case INDEX_op_bitsel_vec: 3472 return 1; 3473 case INDEX_op_cmp_vec: 3474 case INDEX_op_cmpsel_vec: 3475 return -1; 3476 3477 case INDEX_op_rotli_vec: 3478 return have_avx512vl && vece >= MO_32 ? 1 : -1; 3479 3480 case INDEX_op_shli_vec: 3481 case INDEX_op_shri_vec: 3482 /* We must expand the operation for MO_8. */ 3483 return vece == MO_8 ? -1 : 1; 3484 3485 case INDEX_op_sari_vec: 3486 switch (vece) { 3487 case MO_8: 3488 return -1; 3489 case MO_16: 3490 case MO_32: 3491 return 1; 3492 case MO_64: 3493 if (have_avx512vl) { 3494 return 1; 3495 } 3496 /* 3497 * We can emulate this for MO_64, but it does not pay off 3498 * unless we're producing at least 4 values. 3499 */ 3500 return type >= TCG_TYPE_V256 ? -1 : 0; 3501 } 3502 return 0; 3503 3504 case INDEX_op_shls_vec: 3505 case INDEX_op_shrs_vec: 3506 return vece >= MO_16; 3507 case INDEX_op_sars_vec: 3508 switch (vece) { 3509 case MO_16: 3510 case MO_32: 3511 return 1; 3512 case MO_64: 3513 return have_avx512vl; 3514 } 3515 return 0; 3516 case INDEX_op_rotls_vec: 3517 return vece >= MO_16 ? -1 : 0; 3518 3519 case INDEX_op_shlv_vec: 3520 case INDEX_op_shrv_vec: 3521 switch (vece) { 3522 case MO_16: 3523 return have_avx512bw; 3524 case MO_32: 3525 case MO_64: 3526 return have_avx2; 3527 } 3528 return 0; 3529 case INDEX_op_sarv_vec: 3530 switch (vece) { 3531 case MO_16: 3532 return have_avx512bw; 3533 case MO_32: 3534 return have_avx2; 3535 case MO_64: 3536 return have_avx512vl; 3537 } 3538 return 0; 3539 case INDEX_op_rotlv_vec: 3540 case INDEX_op_rotrv_vec: 3541 switch (vece) { 3542 case MO_16: 3543 return have_avx512vbmi2 ? -1 : 0; 3544 case MO_32: 3545 case MO_64: 3546 return have_avx512vl ? 1 : have_avx2 ? -1 : 0; 3547 } 3548 return 0; 3549 3550 case INDEX_op_mul_vec: 3551 switch (vece) { 3552 case MO_8: 3553 return -1; 3554 case MO_64: 3555 return have_avx512dq; 3556 } 3557 return 1; 3558 3559 case INDEX_op_ssadd_vec: 3560 case INDEX_op_usadd_vec: 3561 case INDEX_op_sssub_vec: 3562 case INDEX_op_ussub_vec: 3563 return vece <= MO_16; 3564 case INDEX_op_smin_vec: 3565 case INDEX_op_smax_vec: 3566 case INDEX_op_umin_vec: 3567 case INDEX_op_umax_vec: 3568 case INDEX_op_abs_vec: 3569 return vece <= MO_32 || have_avx512vl; 3570 3571 default: 3572 return 0; 3573 } 3574} 3575 3576static void expand_vec_shi(TCGType type, unsigned vece, TCGOpcode opc, 3577 TCGv_vec v0, TCGv_vec v1, TCGArg imm) 3578{ 3579 TCGv_vec t1, t2; 3580 3581 tcg_debug_assert(vece == MO_8); 3582 3583 t1 = tcg_temp_new_vec(type); 3584 t2 = tcg_temp_new_vec(type); 3585 3586 /* 3587 * Unpack to W, shift, and repack. Tricky bits: 3588 * (1) Use punpck*bw x,x to produce DDCCBBAA, 3589 * i.e. duplicate in other half of the 16-bit lane. 3590 * (2) For right-shift, add 8 so that the high half of the lane 3591 * becomes zero. For left-shift, and left-rotate, we must 3592 * shift up and down again. 3593 * (3) Step 2 leaves high half zero such that PACKUSWB 3594 * (pack with unsigned saturation) does not modify 3595 * the quantity. 3596 */ 3597 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, 3598 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1)); 3599 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, 3600 tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1)); 3601 3602 if (opc != INDEX_op_rotli_vec) { 3603 imm += 8; 3604 } 3605 if (opc == INDEX_op_shri_vec) { 3606 tcg_gen_shri_vec(MO_16, t1, t1, imm); 3607 tcg_gen_shri_vec(MO_16, t2, t2, imm); 3608 } else { 3609 tcg_gen_shli_vec(MO_16, t1, t1, imm); 3610 tcg_gen_shli_vec(MO_16, t2, t2, imm); 3611 tcg_gen_shri_vec(MO_16, t1, t1, 8); 3612 tcg_gen_shri_vec(MO_16, t2, t2, 8); 3613 } 3614 3615 vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8, 3616 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2)); 3617 tcg_temp_free_vec(t1); 3618 tcg_temp_free_vec(t2); 3619} 3620 3621static void expand_vec_sari(TCGType type, unsigned vece, 3622 TCGv_vec v0, TCGv_vec v1, TCGArg imm) 3623{ 3624 TCGv_vec t1, t2; 3625 3626 switch (vece) { 3627 case MO_8: 3628 /* Unpack to W, shift, and repack, as in expand_vec_shi. */ 3629 t1 = tcg_temp_new_vec(type); 3630 t2 = tcg_temp_new_vec(type); 3631 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, 3632 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(v1)); 3633 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, 3634 tcgv_vec_arg(t2), tcgv_vec_arg(v1), tcgv_vec_arg(v1)); 3635 tcg_gen_sari_vec(MO_16, t1, t1, imm + 8); 3636 tcg_gen_sari_vec(MO_16, t2, t2, imm + 8); 3637 vec_gen_3(INDEX_op_x86_packss_vec, type, MO_8, 3638 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t2)); 3639 tcg_temp_free_vec(t1); 3640 tcg_temp_free_vec(t2); 3641 break; 3642 3643 case MO_64: 3644 t1 = tcg_temp_new_vec(type); 3645 if (imm <= 32) { 3646 /* 3647 * We can emulate a small sign extend by performing an arithmetic 3648 * 32-bit shift and overwriting the high half of a 64-bit logical 3649 * shift. Note that the ISA says shift of 32 is valid, but TCG 3650 * does not, so we have to bound the smaller shift -- we get the 3651 * same result in the high half either way. 3652 */ 3653 tcg_gen_sari_vec(MO_32, t1, v1, MIN(imm, 31)); 3654 tcg_gen_shri_vec(MO_64, v0, v1, imm); 3655 vec_gen_4(INDEX_op_x86_blend_vec, type, MO_32, 3656 tcgv_vec_arg(v0), tcgv_vec_arg(v0), 3657 tcgv_vec_arg(t1), 0xaa); 3658 } else { 3659 /* Otherwise we will need to use a compare vs 0 to produce 3660 * the sign-extend, shift and merge. 3661 */ 3662 tcg_gen_cmp_vec(TCG_COND_GT, MO_64, t1, 3663 tcg_constant_vec(type, MO_64, 0), v1); 3664 tcg_gen_shri_vec(MO_64, v0, v1, imm); 3665 tcg_gen_shli_vec(MO_64, t1, t1, 64 - imm); 3666 tcg_gen_or_vec(MO_64, v0, v0, t1); 3667 } 3668 tcg_temp_free_vec(t1); 3669 break; 3670 3671 default: 3672 g_assert_not_reached(); 3673 } 3674} 3675 3676static void expand_vec_rotli(TCGType type, unsigned vece, 3677 TCGv_vec v0, TCGv_vec v1, TCGArg imm) 3678{ 3679 TCGv_vec t; 3680 3681 if (vece == MO_8) { 3682 expand_vec_shi(type, vece, INDEX_op_rotli_vec, v0, v1, imm); 3683 return; 3684 } 3685 3686 if (have_avx512vbmi2) { 3687 vec_gen_4(INDEX_op_x86_vpshldi_vec, type, vece, 3688 tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v1), imm); 3689 return; 3690 } 3691 3692 t = tcg_temp_new_vec(type); 3693 tcg_gen_shli_vec(vece, t, v1, imm); 3694 tcg_gen_shri_vec(vece, v0, v1, (8 << vece) - imm); 3695 tcg_gen_or_vec(vece, v0, v0, t); 3696 tcg_temp_free_vec(t); 3697} 3698 3699static void expand_vec_rotv(TCGType type, unsigned vece, TCGv_vec v0, 3700 TCGv_vec v1, TCGv_vec sh, bool right) 3701{ 3702 TCGv_vec t; 3703 3704 if (have_avx512vbmi2) { 3705 vec_gen_4(right ? INDEX_op_x86_vpshrdv_vec : INDEX_op_x86_vpshldv_vec, 3706 type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v1), 3707 tcgv_vec_arg(v1), tcgv_vec_arg(sh)); 3708 return; 3709 } 3710 3711 t = tcg_temp_new_vec(type); 3712 tcg_gen_dupi_vec(vece, t, 8 << vece); 3713 tcg_gen_sub_vec(vece, t, t, sh); 3714 if (right) { 3715 tcg_gen_shlv_vec(vece, t, v1, t); 3716 tcg_gen_shrv_vec(vece, v0, v1, sh); 3717 } else { 3718 tcg_gen_shrv_vec(vece, t, v1, t); 3719 tcg_gen_shlv_vec(vece, v0, v1, sh); 3720 } 3721 tcg_gen_or_vec(vece, v0, v0, t); 3722 tcg_temp_free_vec(t); 3723} 3724 3725static void expand_vec_rotls(TCGType type, unsigned vece, 3726 TCGv_vec v0, TCGv_vec v1, TCGv_i32 lsh) 3727{ 3728 TCGv_vec t = tcg_temp_new_vec(type); 3729 3730 tcg_debug_assert(vece != MO_8); 3731 3732 if (vece >= MO_32 ? have_avx512vl : have_avx512vbmi2) { 3733 tcg_gen_dup_i32_vec(vece, t, lsh); 3734 if (vece >= MO_32) { 3735 tcg_gen_rotlv_vec(vece, v0, v1, t); 3736 } else { 3737 expand_vec_rotv(type, vece, v0, v1, t, false); 3738 } 3739 } else { 3740 TCGv_i32 rsh = tcg_temp_new_i32(); 3741 3742 tcg_gen_neg_i32(rsh, lsh); 3743 tcg_gen_andi_i32(rsh, rsh, (8 << vece) - 1); 3744 tcg_gen_shls_vec(vece, t, v1, lsh); 3745 tcg_gen_shrs_vec(vece, v0, v1, rsh); 3746 tcg_gen_or_vec(vece, v0, v0, t); 3747 3748 tcg_temp_free_i32(rsh); 3749 } 3750 3751 tcg_temp_free_vec(t); 3752} 3753 3754static void expand_vec_mul(TCGType type, unsigned vece, 3755 TCGv_vec v0, TCGv_vec v1, TCGv_vec v2) 3756{ 3757 TCGv_vec t1, t2, t3, t4, zero; 3758 3759 tcg_debug_assert(vece == MO_8); 3760 3761 /* 3762 * Unpack v1 bytes to words, 0 | x. 3763 * Unpack v2 bytes to words, y | 0. 3764 * This leaves the 8-bit result, x * y, with 8 bits of right padding. 3765 * Shift logical right by 8 bits to clear the high 8 bytes before 3766 * using an unsigned saturated pack. 3767 * 3768 * The difference between the V64, V128 and V256 cases is merely how 3769 * we distribute the expansion between temporaries. 3770 */ 3771 switch (type) { 3772 case TCG_TYPE_V64: 3773 t1 = tcg_temp_new_vec(TCG_TYPE_V128); 3774 t2 = tcg_temp_new_vec(TCG_TYPE_V128); 3775 zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0); 3776 vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8, 3777 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); 3778 vec_gen_3(INDEX_op_x86_punpckl_vec, TCG_TYPE_V128, MO_8, 3779 tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); 3780 tcg_gen_mul_vec(MO_16, t1, t1, t2); 3781 tcg_gen_shri_vec(MO_16, t1, t1, 8); 3782 vec_gen_3(INDEX_op_x86_packus_vec, TCG_TYPE_V128, MO_8, 3783 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t1)); 3784 tcg_temp_free_vec(t1); 3785 tcg_temp_free_vec(t2); 3786 break; 3787 3788 case TCG_TYPE_V128: 3789 case TCG_TYPE_V256: 3790 t1 = tcg_temp_new_vec(type); 3791 t2 = tcg_temp_new_vec(type); 3792 t3 = tcg_temp_new_vec(type); 3793 t4 = tcg_temp_new_vec(type); 3794 zero = tcg_constant_vec(TCG_TYPE_V128, MO_8, 0); 3795 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, 3796 tcgv_vec_arg(t1), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); 3797 vec_gen_3(INDEX_op_x86_punpckl_vec, type, MO_8, 3798 tcgv_vec_arg(t2), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); 3799 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, 3800 tcgv_vec_arg(t3), tcgv_vec_arg(v1), tcgv_vec_arg(zero)); 3801 vec_gen_3(INDEX_op_x86_punpckh_vec, type, MO_8, 3802 tcgv_vec_arg(t4), tcgv_vec_arg(zero), tcgv_vec_arg(v2)); 3803 tcg_gen_mul_vec(MO_16, t1, t1, t2); 3804 tcg_gen_mul_vec(MO_16, t3, t3, t4); 3805 tcg_gen_shri_vec(MO_16, t1, t1, 8); 3806 tcg_gen_shri_vec(MO_16, t3, t3, 8); 3807 vec_gen_3(INDEX_op_x86_packus_vec, type, MO_8, 3808 tcgv_vec_arg(v0), tcgv_vec_arg(t1), tcgv_vec_arg(t3)); 3809 tcg_temp_free_vec(t1); 3810 tcg_temp_free_vec(t2); 3811 tcg_temp_free_vec(t3); 3812 tcg_temp_free_vec(t4); 3813 break; 3814 3815 default: 3816 g_assert_not_reached(); 3817 } 3818} 3819 3820static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, 3821 TCGv_vec v1, TCGv_vec v2, TCGCond cond) 3822{ 3823 enum { 3824 NEED_INV = 1, 3825 NEED_SWAP = 2, 3826 NEED_BIAS = 4, 3827 NEED_UMIN = 8, 3828 NEED_UMAX = 16, 3829 }; 3830 TCGv_vec t1, t2, t3; 3831 uint8_t fixup; 3832 3833 switch (cond) { 3834 case TCG_COND_EQ: 3835 case TCG_COND_GT: 3836 fixup = 0; 3837 break; 3838 case TCG_COND_NE: 3839 case TCG_COND_LE: 3840 fixup = NEED_INV; 3841 break; 3842 case TCG_COND_LT: 3843 fixup = NEED_SWAP; 3844 break; 3845 case TCG_COND_GE: 3846 fixup = NEED_SWAP | NEED_INV; 3847 break; 3848 case TCG_COND_LEU: 3849 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) { 3850 fixup = NEED_UMIN; 3851 } else { 3852 fixup = NEED_BIAS | NEED_INV; 3853 } 3854 break; 3855 case TCG_COND_GTU: 3856 if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) { 3857 fixup = NEED_UMIN | NEED_INV; 3858 } else { 3859 fixup = NEED_BIAS; 3860 } 3861 break; 3862 case TCG_COND_GEU: 3863 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) { 3864 fixup = NEED_UMAX; 3865 } else { 3866 fixup = NEED_BIAS | NEED_SWAP | NEED_INV; 3867 } 3868 break; 3869 case TCG_COND_LTU: 3870 if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) { 3871 fixup = NEED_UMAX | NEED_INV; 3872 } else { 3873 fixup = NEED_BIAS | NEED_SWAP; 3874 } 3875 break; 3876 default: 3877 g_assert_not_reached(); 3878 } 3879 3880 if (fixup & NEED_INV) { 3881 cond = tcg_invert_cond(cond); 3882 } 3883 if (fixup & NEED_SWAP) { 3884 t1 = v1, v1 = v2, v2 = t1; 3885 cond = tcg_swap_cond(cond); 3886 } 3887 3888 t1 = t2 = NULL; 3889 if (fixup & (NEED_UMIN | NEED_UMAX)) { 3890 t1 = tcg_temp_new_vec(type); 3891 if (fixup & NEED_UMIN) { 3892 tcg_gen_umin_vec(vece, t1, v1, v2); 3893 } else { 3894 tcg_gen_umax_vec(vece, t1, v1, v2); 3895 } 3896 v2 = t1; 3897 cond = TCG_COND_EQ; 3898 } else if (fixup & NEED_BIAS) { 3899 t1 = tcg_temp_new_vec(type); 3900 t2 = tcg_temp_new_vec(type); 3901 t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1)); 3902 tcg_gen_sub_vec(vece, t1, v1, t3); 3903 tcg_gen_sub_vec(vece, t2, v2, t3); 3904 v1 = t1; 3905 v2 = t2; 3906 cond = tcg_signed_cond(cond); 3907 } 3908 3909 tcg_debug_assert(cond == TCG_COND_EQ || cond == TCG_COND_GT); 3910 /* Expand directly; do not recurse. */ 3911 vec_gen_4(INDEX_op_cmp_vec, type, vece, 3912 tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond); 3913 3914 if (t1) { 3915 tcg_temp_free_vec(t1); 3916 if (t2) { 3917 tcg_temp_free_vec(t2); 3918 } 3919 } 3920 return fixup & NEED_INV; 3921} 3922 3923static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, 3924 TCGv_vec v1, TCGv_vec v2, TCGCond cond) 3925{ 3926 if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) { 3927 tcg_gen_not_vec(vece, v0, v0); 3928 } 3929} 3930 3931static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0, 3932 TCGv_vec c1, TCGv_vec c2, 3933 TCGv_vec v3, TCGv_vec v4, TCGCond cond) 3934{ 3935 TCGv_vec t = tcg_temp_new_vec(type); 3936 3937 if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) { 3938 /* Invert the sense of the compare by swapping arguments. */ 3939 TCGv_vec x; 3940 x = v3, v3 = v4, v4 = x; 3941 } 3942 vec_gen_4(INDEX_op_x86_vpblendvb_vec, type, vece, 3943 tcgv_vec_arg(v0), tcgv_vec_arg(v4), 3944 tcgv_vec_arg(v3), tcgv_vec_arg(t)); 3945 tcg_temp_free_vec(t); 3946} 3947 3948void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, 3949 TCGArg a0, ...) 3950{ 3951 va_list va; 3952 TCGArg a2; 3953 TCGv_vec v0, v1, v2, v3, v4; 3954 3955 va_start(va, a0); 3956 v0 = temp_tcgv_vec(arg_temp(a0)); 3957 v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); 3958 a2 = va_arg(va, TCGArg); 3959 3960 switch (opc) { 3961 case INDEX_op_shli_vec: 3962 case INDEX_op_shri_vec: 3963 expand_vec_shi(type, vece, opc, v0, v1, a2); 3964 break; 3965 3966 case INDEX_op_sari_vec: 3967 expand_vec_sari(type, vece, v0, v1, a2); 3968 break; 3969 3970 case INDEX_op_rotli_vec: 3971 expand_vec_rotli(type, vece, v0, v1, a2); 3972 break; 3973 3974 case INDEX_op_rotls_vec: 3975 expand_vec_rotls(type, vece, v0, v1, temp_tcgv_i32(arg_temp(a2))); 3976 break; 3977 3978 case INDEX_op_rotlv_vec: 3979 v2 = temp_tcgv_vec(arg_temp(a2)); 3980 expand_vec_rotv(type, vece, v0, v1, v2, false); 3981 break; 3982 case INDEX_op_rotrv_vec: 3983 v2 = temp_tcgv_vec(arg_temp(a2)); 3984 expand_vec_rotv(type, vece, v0, v1, v2, true); 3985 break; 3986 3987 case INDEX_op_mul_vec: 3988 v2 = temp_tcgv_vec(arg_temp(a2)); 3989 expand_vec_mul(type, vece, v0, v1, v2); 3990 break; 3991 3992 case INDEX_op_cmp_vec: 3993 v2 = temp_tcgv_vec(arg_temp(a2)); 3994 expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); 3995 break; 3996 3997 case INDEX_op_cmpsel_vec: 3998 v2 = temp_tcgv_vec(arg_temp(a2)); 3999 v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); 4000 v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); 4001 expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg)); 4002 break; 4003 4004 default: 4005 break; 4006 } 4007 4008 va_end(va); 4009} 4010 4011static const int tcg_target_callee_save_regs[] = { 4012#if TCG_TARGET_REG_BITS == 64 4013 TCG_REG_RBP, 4014 TCG_REG_RBX, 4015#if defined(_WIN64) 4016 TCG_REG_RDI, 4017 TCG_REG_RSI, 4018#endif 4019 TCG_REG_R12, 4020 TCG_REG_R13, 4021 TCG_REG_R14, /* Currently used for the global env. */ 4022 TCG_REG_R15, 4023#else 4024 TCG_REG_EBP, /* Currently used for the global env. */ 4025 TCG_REG_EBX, 4026 TCG_REG_ESI, 4027 TCG_REG_EDI, 4028#endif 4029}; 4030 4031/* Compute frame size via macros, to share between tcg_target_qemu_prologue 4032 and tcg_register_jit. */ 4033 4034#define PUSH_SIZE \ 4035 ((1 + ARRAY_SIZE(tcg_target_callee_save_regs)) \ 4036 * (TCG_TARGET_REG_BITS / 8)) 4037 4038#define FRAME_SIZE \ 4039 ((PUSH_SIZE \ 4040 + TCG_STATIC_CALL_ARGS_SIZE \ 4041 + CPU_TEMP_BUF_NLONGS * sizeof(long) \ 4042 + TCG_TARGET_STACK_ALIGN - 1) \ 4043 & ~(TCG_TARGET_STACK_ALIGN - 1)) 4044 4045/* Generate global QEMU prologue and epilogue code */ 4046static void tcg_target_qemu_prologue(TCGContext *s) 4047{ 4048 int i, stack_addend; 4049 4050 /* TB prologue */ 4051 4052 /* Reserve some stack space, also for TCG temps. */ 4053 stack_addend = FRAME_SIZE - PUSH_SIZE; 4054 tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE, 4055 CPU_TEMP_BUF_NLONGS * sizeof(long)); 4056 4057 /* Save all callee saved registers. */ 4058 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { 4059 tcg_out_push(s, tcg_target_callee_save_regs[i]); 4060 } 4061 4062#if TCG_TARGET_REG_BITS == 32 4063 tcg_out_ld(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, 4064 (ARRAY_SIZE(tcg_target_callee_save_regs) + 1) * 4); 4065 tcg_out_addi(s, TCG_REG_ESP, -stack_addend); 4066 /* jmp *tb. */ 4067 tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, TCG_REG_ESP, 4068 (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4 4069 + stack_addend); 4070#else 4071# if !defined(CONFIG_SOFTMMU) 4072 if (guest_base) { 4073 int seg = setup_guest_base_seg(); 4074 if (seg != 0) { 4075 x86_guest_base.seg = seg; 4076 } else if (guest_base == (int32_t)guest_base) { 4077 x86_guest_base.ofs = guest_base; 4078 } else { 4079 /* Choose R12 because, as a base, it requires a SIB byte. */ 4080 x86_guest_base.index = TCG_REG_R12; 4081 tcg_out_movi(s, TCG_TYPE_PTR, x86_guest_base.index, guest_base); 4082 tcg_regset_set_reg(s->reserved_regs, x86_guest_base.index); 4083 } 4084 } 4085# endif 4086 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); 4087 tcg_out_addi(s, TCG_REG_ESP, -stack_addend); 4088 /* jmp *tb. */ 4089 tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]); 4090#endif 4091 4092 /* 4093 * Return path for goto_ptr. Set return value to 0, a-la exit_tb, 4094 * and fall through to the rest of the epilogue. 4095 */ 4096 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); 4097 tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_EAX, 0); 4098 4099 /* TB epilogue */ 4100 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); 4101 4102 tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend); 4103 4104 if (have_avx2) { 4105 tcg_out_vex_opc(s, OPC_VZEROUPPER, 0, 0, 0, 0); 4106 } 4107 for (i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) { 4108 tcg_out_pop(s, tcg_target_callee_save_regs[i]); 4109 } 4110 tcg_out_opc(s, OPC_RET, 0, 0, 0); 4111} 4112 4113static void tcg_out_nop_fill(tcg_insn_unit *p, int count) 4114{ 4115 memset(p, 0x90, count); 4116} 4117 4118static void tcg_target_init(TCGContext *s) 4119{ 4120 tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS; 4121 if (TCG_TARGET_REG_BITS == 64) { 4122 tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS; 4123 } 4124 if (have_avx1) { 4125 tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS; 4126 tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS; 4127 } 4128 if (have_avx2) { 4129 tcg_target_available_regs[TCG_TYPE_V256] = ALL_VECTOR_REGS; 4130 } 4131 4132 tcg_target_call_clobber_regs = ALL_VECTOR_REGS; 4133 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EAX); 4134 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EDX); 4135 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_ECX); 4136 if (TCG_TARGET_REG_BITS == 64) { 4137#if !defined(_WIN64) 4138 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RDI); 4139 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RSI); 4140#endif 4141 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8); 4142 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9); 4143 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10); 4144 tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11); 4145 } 4146 4147 s->reserved_regs = 0; 4148 tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK); 4149 tcg_regset_set_reg(s->reserved_regs, TCG_TMP_VEC); 4150#ifdef _WIN64 4151 /* These are call saved, and we don't save them, so don't use them. */ 4152 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM6); 4153 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM7); 4154 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM8); 4155 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM9); 4156 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM10); 4157 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM11); 4158 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM12); 4159 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM13); 4160 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM14); 4161 tcg_regset_set_reg(s->reserved_regs, TCG_REG_XMM15); 4162#endif 4163} 4164 4165typedef struct { 4166 DebugFrameHeader h; 4167 uint8_t fde_def_cfa[4]; 4168 uint8_t fde_reg_ofs[14]; 4169} DebugFrame; 4170 4171/* We're expecting a 2 byte uleb128 encoded value. */ 4172QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14)); 4173 4174#if !defined(__ELF__) 4175 /* Host machine without ELF. */ 4176#elif TCG_TARGET_REG_BITS == 64 4177#define ELF_HOST_MACHINE EM_X86_64 4178static const DebugFrame debug_frame = { 4179 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */ 4180 .h.cie.id = -1, 4181 .h.cie.version = 1, 4182 .h.cie.code_align = 1, 4183 .h.cie.data_align = 0x78, /* sleb128 -8 */ 4184 .h.cie.return_column = 16, 4185 4186 /* Total FDE size does not include the "len" member. */ 4187 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), 4188 4189 .fde_def_cfa = { 4190 12, 7, /* DW_CFA_def_cfa %rsp, ... */ 4191 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ 4192 (FRAME_SIZE >> 7) 4193 }, 4194 .fde_reg_ofs = { 4195 0x90, 1, /* DW_CFA_offset, %rip, -8 */ 4196 /* The following ordering must match tcg_target_callee_save_regs. */ 4197 0x86, 2, /* DW_CFA_offset, %rbp, -16 */ 4198 0x83, 3, /* DW_CFA_offset, %rbx, -24 */ 4199 0x8c, 4, /* DW_CFA_offset, %r12, -32 */ 4200 0x8d, 5, /* DW_CFA_offset, %r13, -40 */ 4201 0x8e, 6, /* DW_CFA_offset, %r14, -48 */ 4202 0x8f, 7, /* DW_CFA_offset, %r15, -56 */ 4203 } 4204}; 4205#else 4206#define ELF_HOST_MACHINE EM_386 4207static const DebugFrame debug_frame = { 4208 .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */ 4209 .h.cie.id = -1, 4210 .h.cie.version = 1, 4211 .h.cie.code_align = 1, 4212 .h.cie.data_align = 0x7c, /* sleb128 -4 */ 4213 .h.cie.return_column = 8, 4214 4215 /* Total FDE size does not include the "len" member. */ 4216 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), 4217 4218 .fde_def_cfa = { 4219 12, 4, /* DW_CFA_def_cfa %esp, ... */ 4220 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ 4221 (FRAME_SIZE >> 7) 4222 }, 4223 .fde_reg_ofs = { 4224 0x88, 1, /* DW_CFA_offset, %eip, -4 */ 4225 /* The following ordering must match tcg_target_callee_save_regs. */ 4226 0x85, 2, /* DW_CFA_offset, %ebp, -8 */ 4227 0x83, 3, /* DW_CFA_offset, %ebx, -12 */ 4228 0x86, 4, /* DW_CFA_offset, %esi, -16 */ 4229 0x87, 5, /* DW_CFA_offset, %edi, -20 */ 4230 } 4231}; 4232#endif 4233 4234#if defined(ELF_HOST_MACHINE) 4235void tcg_register_jit(const void *buf, size_t buf_size) 4236{ 4237 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); 4238} 4239#endif 4240