1/* 2 * New-style TCG opcode generator for i386 instructions 3 * 4 * Copyright (c) 2022 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This library is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU Lesser General Public 10 * License as published by the Free Software Foundation; either 11 * version 2.1 of the License, or (at your option) any later version. 12 * 13 * This library is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * Lesser General Public License for more details. 17 * 18 * You should have received a copy of the GNU Lesser General Public 19 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22#define ZMM_OFFSET(reg) offsetof(CPUX86State, xmm_regs[reg]) 23 24typedef void (*SSEFunc_i_ep)(TCGv_i32 val, TCGv_ptr env, TCGv_ptr reg); 25typedef void (*SSEFunc_l_ep)(TCGv_i64 val, TCGv_ptr env, TCGv_ptr reg); 26typedef void (*SSEFunc_0_epp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b); 27typedef void (*SSEFunc_0_eppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 28 TCGv_ptr reg_c); 29typedef void (*SSEFunc_0_epppp)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 30 TCGv_ptr reg_c, TCGv_ptr reg_d); 31typedef void (*SSEFunc_0_eppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 32 TCGv_i32 val); 33typedef void (*SSEFunc_0_epppi)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 34 TCGv_ptr reg_c, TCGv_i32 val); 35typedef void (*SSEFunc_0_ppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_i32 val); 36typedef void (*SSEFunc_0_pppi)(TCGv_ptr reg_a, TCGv_ptr reg_b, TCGv_ptr reg_c, 37 TCGv_i32 val); 38typedef void (*SSEFunc_0_eppt)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 39 TCGv val); 40typedef void (*SSEFunc_0_epppti)(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b, 41 TCGv_ptr reg_c, TCGv a0, TCGv_i32 scale); 42 43static inline TCGv_i32 tcg_constant8u_i32(uint8_t val) 44{ 45 return tcg_constant_i32(val); 46} 47 48static void gen_NM_exception(DisasContext *s) 49{ 50 gen_exception(s, EXCP07_PREX); 51} 52 53static void gen_illegal(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 54{ 55 gen_illegal_opcode(s); 56} 57 58static void gen_load_ea(DisasContext *s, AddressParts *mem, bool is_vsib) 59{ 60 TCGv ea = gen_lea_modrm_1(s, *mem, is_vsib); 61 gen_lea_v_seg(s, s->aflag, ea, mem->def_seg, s->override); 62} 63 64static inline int mmx_offset(MemOp ot) 65{ 66 switch (ot) { 67 case MO_8: 68 return offsetof(MMXReg, MMX_B(0)); 69 case MO_16: 70 return offsetof(MMXReg, MMX_W(0)); 71 case MO_32: 72 return offsetof(MMXReg, MMX_L(0)); 73 case MO_64: 74 return offsetof(MMXReg, MMX_Q(0)); 75 default: 76 g_assert_not_reached(); 77 } 78} 79 80static inline int xmm_offset(MemOp ot) 81{ 82 switch (ot) { 83 case MO_8: 84 return offsetof(ZMMReg, ZMM_B(0)); 85 case MO_16: 86 return offsetof(ZMMReg, ZMM_W(0)); 87 case MO_32: 88 return offsetof(ZMMReg, ZMM_L(0)); 89 case MO_64: 90 return offsetof(ZMMReg, ZMM_Q(0)); 91 case MO_128: 92 return offsetof(ZMMReg, ZMM_X(0)); 93 case MO_256: 94 return offsetof(ZMMReg, ZMM_Y(0)); 95 default: 96 g_assert_not_reached(); 97 } 98} 99 100static int vector_reg_offset(X86DecodedOp *op) 101{ 102 assert(op->unit == X86_OP_MMX || op->unit == X86_OP_SSE); 103 104 if (op->unit == X86_OP_MMX) { 105 return op->offset - mmx_offset(op->ot); 106 } else { 107 return op->offset - xmm_offset(op->ot); 108 } 109} 110 111static int vector_elem_offset(X86DecodedOp *op, MemOp ot, int n) 112{ 113 int base_ofs = vector_reg_offset(op); 114 switch(ot) { 115 case MO_8: 116 if (op->unit == X86_OP_MMX) { 117 return base_ofs + offsetof(MMXReg, MMX_B(n)); 118 } else { 119 return base_ofs + offsetof(ZMMReg, ZMM_B(n)); 120 } 121 case MO_16: 122 if (op->unit == X86_OP_MMX) { 123 return base_ofs + offsetof(MMXReg, MMX_W(n)); 124 } else { 125 return base_ofs + offsetof(ZMMReg, ZMM_W(n)); 126 } 127 case MO_32: 128 if (op->unit == X86_OP_MMX) { 129 return base_ofs + offsetof(MMXReg, MMX_L(n)); 130 } else { 131 return base_ofs + offsetof(ZMMReg, ZMM_L(n)); 132 } 133 case MO_64: 134 if (op->unit == X86_OP_MMX) { 135 return base_ofs; 136 } else { 137 return base_ofs + offsetof(ZMMReg, ZMM_Q(n)); 138 } 139 case MO_128: 140 assert(op->unit == X86_OP_SSE); 141 return base_ofs + offsetof(ZMMReg, ZMM_X(n)); 142 case MO_256: 143 assert(op->unit == X86_OP_SSE); 144 return base_ofs + offsetof(ZMMReg, ZMM_Y(n)); 145 default: 146 g_assert_not_reached(); 147 } 148} 149 150static void compute_mmx_offset(X86DecodedOp *op) 151{ 152 if (!op->has_ea) { 153 op->offset = offsetof(CPUX86State, fpregs[op->n].mmx) + mmx_offset(op->ot); 154 } else { 155 op->offset = offsetof(CPUX86State, mmx_t0) + mmx_offset(op->ot); 156 } 157} 158 159static void compute_xmm_offset(X86DecodedOp *op) 160{ 161 if (!op->has_ea) { 162 op->offset = ZMM_OFFSET(op->n) + xmm_offset(op->ot); 163 } else { 164 op->offset = offsetof(CPUX86State, xmm_t0) + xmm_offset(op->ot); 165 } 166} 167 168static void gen_load_sse(DisasContext *s, TCGv temp, MemOp ot, int dest_ofs, bool aligned) 169{ 170 switch(ot) { 171 case MO_8: 172 gen_op_ld_v(s, MO_8, temp, s->A0); 173 tcg_gen_st8_tl(temp, cpu_env, dest_ofs); 174 break; 175 case MO_16: 176 gen_op_ld_v(s, MO_16, temp, s->A0); 177 tcg_gen_st16_tl(temp, cpu_env, dest_ofs); 178 break; 179 case MO_32: 180 gen_op_ld_v(s, MO_32, temp, s->A0); 181 tcg_gen_st32_tl(temp, cpu_env, dest_ofs); 182 break; 183 case MO_64: 184 gen_ldq_env_A0(s, dest_ofs); 185 break; 186 case MO_128: 187 gen_ldo_env_A0(s, dest_ofs, aligned); 188 break; 189 case MO_256: 190 gen_ldy_env_A0(s, dest_ofs, aligned); 191 break; 192 default: 193 g_assert_not_reached(); 194 } 195} 196 197static bool sse_needs_alignment(DisasContext *s, X86DecodedInsn *decode, MemOp ot) 198{ 199 switch (decode->e.vex_class) { 200 case 2: 201 case 4: 202 if ((s->prefix & PREFIX_VEX) || 203 decode->e.vex_special == X86_VEX_SSEUnaligned) { 204 /* MOST legacy SSE instructions require aligned memory operands, but not all. */ 205 return false; 206 } 207 /* fall through */ 208 case 1: 209 return ot >= MO_128; 210 211 default: 212 return false; 213 } 214} 215 216static void gen_load(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 217{ 218 X86DecodedOp *op = &decode->op[opn]; 219 220 switch (op->unit) { 221 case X86_OP_SKIP: 222 return; 223 case X86_OP_SEG: 224 tcg_gen_ld32u_tl(v, cpu_env, 225 offsetof(CPUX86State,segs[op->n].selector)); 226 break; 227 case X86_OP_CR: 228 tcg_gen_ld_tl(v, cpu_env, offsetof(CPUX86State, cr[op->n])); 229 break; 230 case X86_OP_DR: 231 tcg_gen_ld_tl(v, cpu_env, offsetof(CPUX86State, dr[op->n])); 232 break; 233 case X86_OP_INT: 234 if (op->has_ea) { 235 gen_op_ld_v(s, op->ot, v, s->A0); 236 } else { 237 gen_op_mov_v_reg(s, op->ot, v, op->n); 238 } 239 break; 240 case X86_OP_IMM: 241 tcg_gen_movi_tl(v, decode->immediate); 242 break; 243 244 case X86_OP_MMX: 245 compute_mmx_offset(op); 246 goto load_vector; 247 248 case X86_OP_SSE: 249 compute_xmm_offset(op); 250 load_vector: 251 if (op->has_ea) { 252 bool aligned = sse_needs_alignment(s, decode, op->ot); 253 gen_load_sse(s, v, op->ot, op->offset, aligned); 254 } 255 break; 256 257 default: 258 g_assert_not_reached(); 259 } 260} 261 262static TCGv_ptr op_ptr(X86DecodedInsn *decode, int opn) 263{ 264 X86DecodedOp *op = &decode->op[opn]; 265 if (op->v_ptr) { 266 return op->v_ptr; 267 } 268 op->v_ptr = tcg_temp_new_ptr(); 269 270 /* The temporary points to the MMXReg or ZMMReg. */ 271 tcg_gen_addi_ptr(op->v_ptr, cpu_env, vector_reg_offset(op)); 272 return op->v_ptr; 273} 274 275#define OP_PTR0 op_ptr(decode, 0) 276#define OP_PTR1 op_ptr(decode, 1) 277#define OP_PTR2 op_ptr(decode, 2) 278 279static void gen_writeback(DisasContext *s, X86DecodedInsn *decode, int opn, TCGv v) 280{ 281 X86DecodedOp *op = &decode->op[opn]; 282 switch (op->unit) { 283 case X86_OP_SKIP: 284 break; 285 case X86_OP_SEG: 286 /* Note that gen_movl_seg_T0 takes care of interrupt shadow and TF. */ 287 gen_movl_seg_T0(s, op->n); 288 break; 289 case X86_OP_INT: 290 if (op->has_ea) { 291 gen_op_st_v(s, op->ot, v, s->A0); 292 } else { 293 gen_op_mov_reg_v(s, op->ot, op->n, v); 294 } 295 break; 296 case X86_OP_MMX: 297 break; 298 case X86_OP_SSE: 299 if ((s->prefix & PREFIX_VEX) && op->ot == MO_128) { 300 tcg_gen_gvec_dup_imm(MO_64, 301 offsetof(CPUX86State, xmm_regs[op->n].ZMM_X(1)), 302 16, 16, 0); 303 } 304 break; 305 case X86_OP_CR: 306 case X86_OP_DR: 307 default: 308 g_assert_not_reached(); 309 } 310} 311 312static inline int vector_len(DisasContext *s, X86DecodedInsn *decode) 313{ 314 if (decode->e.special == X86_SPECIAL_MMX && 315 !(s->prefix & (PREFIX_DATA | PREFIX_REPZ | PREFIX_REPNZ))) { 316 return 8; 317 } 318 return s->vex_l ? 32 : 16; 319} 320 321static void gen_store_sse(DisasContext *s, X86DecodedInsn *decode, int src_ofs) 322{ 323 MemOp ot = decode->op[0].ot; 324 int vec_len = vector_len(s, decode); 325 bool aligned = sse_needs_alignment(s, decode, ot); 326 327 if (!decode->op[0].has_ea) { 328 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, vec_len, vec_len); 329 return; 330 } 331 332 switch (ot) { 333 case MO_64: 334 gen_stq_env_A0(s, src_ofs); 335 break; 336 case MO_128: 337 gen_sto_env_A0(s, src_ofs, aligned); 338 break; 339 case MO_256: 340 gen_sty_env_A0(s, src_ofs, aligned); 341 break; 342 default: 343 g_assert_not_reached(); 344 } 345} 346 347static void gen_helper_pavgusb(TCGv_ptr env, TCGv_ptr reg_a, TCGv_ptr reg_b) 348{ 349 gen_helper_pavgb_mmx(env, reg_a, reg_a, reg_b); 350} 351 352#define FN_3DNOW_MOVE ((SSEFunc_0_epp) (uintptr_t) 1) 353static const SSEFunc_0_epp fns_3dnow[] = { 354 [0x0c] = gen_helper_pi2fw, 355 [0x0d] = gen_helper_pi2fd, 356 [0x1c] = gen_helper_pf2iw, 357 [0x1d] = gen_helper_pf2id, 358 [0x8a] = gen_helper_pfnacc, 359 [0x8e] = gen_helper_pfpnacc, 360 [0x90] = gen_helper_pfcmpge, 361 [0x94] = gen_helper_pfmin, 362 [0x96] = gen_helper_pfrcp, 363 [0x97] = gen_helper_pfrsqrt, 364 [0x9a] = gen_helper_pfsub, 365 [0x9e] = gen_helper_pfadd, 366 [0xa0] = gen_helper_pfcmpgt, 367 [0xa4] = gen_helper_pfmax, 368 [0xa6] = FN_3DNOW_MOVE, /* PFRCPIT1; no need to actually increase precision */ 369 [0xa7] = FN_3DNOW_MOVE, /* PFRSQIT1 */ 370 [0xb6] = FN_3DNOW_MOVE, /* PFRCPIT2 */ 371 [0xaa] = gen_helper_pfsubr, 372 [0xae] = gen_helper_pfacc, 373 [0xb0] = gen_helper_pfcmpeq, 374 [0xb4] = gen_helper_pfmul, 375 [0xb7] = gen_helper_pmulhrw_mmx, 376 [0xbb] = gen_helper_pswapd, 377 [0xbf] = gen_helper_pavgusb, 378}; 379 380static void gen_3dnow(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 381{ 382 uint8_t b = decode->immediate; 383 SSEFunc_0_epp fn = b < ARRAY_SIZE(fns_3dnow) ? fns_3dnow[b] : NULL; 384 385 if (!fn) { 386 gen_illegal_opcode(s); 387 return; 388 } 389 if (s->flags & HF_TS_MASK) { 390 gen_NM_exception(s); 391 return; 392 } 393 if (s->flags & HF_EM_MASK) { 394 gen_illegal_opcode(s); 395 return; 396 } 397 398 gen_helper_enter_mmx(cpu_env); 399 if (fn == FN_3DNOW_MOVE) { 400 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset); 401 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset); 402 } else { 403 fn(cpu_env, OP_PTR0, OP_PTR1); 404 } 405} 406 407/* 408 * 00 = v*ps Vps, Hps, Wpd 409 * 66 = v*pd Vpd, Hpd, Wps 410 * f3 = v*ss Vss, Hss, Wps 411 * f2 = v*sd Vsd, Hsd, Wps 412 */ 413static inline void gen_unary_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 414 SSEFunc_0_epp pd_xmm, SSEFunc_0_epp ps_xmm, 415 SSEFunc_0_epp pd_ymm, SSEFunc_0_epp ps_ymm, 416 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 417{ 418 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 419 SSEFunc_0_eppp fn = s->prefix & PREFIX_REPZ ? ss : sd; 420 if (!fn) { 421 gen_illegal_opcode(s); 422 return; 423 } 424 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 425 } else { 426 SSEFunc_0_epp ps, pd, fn; 427 ps = s->vex_l ? ps_ymm : ps_xmm; 428 pd = s->vex_l ? pd_ymm : pd_xmm; 429 fn = s->prefix & PREFIX_DATA ? pd : ps; 430 if (!fn) { 431 gen_illegal_opcode(s); 432 return; 433 } 434 fn(cpu_env, OP_PTR0, OP_PTR2); 435 } 436} 437#define UNARY_FP_SSE(uname, lname) \ 438static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 439{ \ 440 gen_unary_fp_sse(s, env, decode, \ 441 gen_helper_##lname##pd_xmm, \ 442 gen_helper_##lname##ps_xmm, \ 443 gen_helper_##lname##pd_ymm, \ 444 gen_helper_##lname##ps_ymm, \ 445 gen_helper_##lname##sd, \ 446 gen_helper_##lname##ss); \ 447} 448UNARY_FP_SSE(VSQRT, sqrt) 449 450/* 451 * 00 = v*ps Vps, Hps, Wpd 452 * 66 = v*pd Vpd, Hpd, Wps 453 * f3 = v*ss Vss, Hss, Wps 454 * f2 = v*sd Vsd, Hsd, Wps 455 */ 456static inline void gen_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 457 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 458 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm, 459 SSEFunc_0_eppp sd, SSEFunc_0_eppp ss) 460{ 461 SSEFunc_0_eppp ps, pd, fn; 462 if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ)) != 0) { 463 fn = s->prefix & PREFIX_REPZ ? ss : sd; 464 } else { 465 ps = s->vex_l ? ps_ymm : ps_xmm; 466 pd = s->vex_l ? pd_ymm : pd_xmm; 467 fn = s->prefix & PREFIX_DATA ? pd : ps; 468 } 469 if (fn) { 470 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 471 } else { 472 gen_illegal_opcode(s); 473 } 474} 475 476#define FP_SSE(uname, lname) \ 477static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 478{ \ 479 gen_fp_sse(s, env, decode, \ 480 gen_helper_##lname##pd_xmm, \ 481 gen_helper_##lname##ps_xmm, \ 482 gen_helper_##lname##pd_ymm, \ 483 gen_helper_##lname##ps_ymm, \ 484 gen_helper_##lname##sd, \ 485 gen_helper_##lname##ss); \ 486} 487FP_SSE(VADD, add) 488FP_SSE(VMUL, mul) 489FP_SSE(VSUB, sub) 490FP_SSE(VMIN, min) 491FP_SSE(VDIV, div) 492FP_SSE(VMAX, max) 493 494#define FP_UNPACK_SSE(uname, lname) \ 495static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 496{ \ 497 /* PS maps to the DQ integer instruction, PD maps to QDQ. */ \ 498 gen_fp_sse(s, env, decode, \ 499 gen_helper_##lname##qdq_xmm, \ 500 gen_helper_##lname##dq_xmm, \ 501 gen_helper_##lname##qdq_ymm, \ 502 gen_helper_##lname##dq_ymm, \ 503 NULL, NULL); \ 504} 505FP_UNPACK_SSE(VUNPCKLPx, punpckl) 506FP_UNPACK_SSE(VUNPCKHPx, punpckh) 507 508/* 509 * 00 = v*ps Vps, Wpd 510 * f3 = v*ss Vss, Wps 511 */ 512static inline void gen_unary_fp32_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 513 SSEFunc_0_epp ps_xmm, 514 SSEFunc_0_epp ps_ymm, 515 SSEFunc_0_eppp ss) 516{ 517 if ((s->prefix & (PREFIX_DATA | PREFIX_REPNZ)) != 0) { 518 goto illegal_op; 519 } else if (s->prefix & PREFIX_REPZ) { 520 if (!ss) { 521 goto illegal_op; 522 } 523 ss(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 524 } else { 525 SSEFunc_0_epp fn = s->vex_l ? ps_ymm : ps_xmm; 526 if (!fn) { 527 goto illegal_op; 528 } 529 fn(cpu_env, OP_PTR0, OP_PTR2); 530 } 531 return; 532 533illegal_op: 534 gen_illegal_opcode(s); 535} 536#define UNARY_FP32_SSE(uname, lname) \ 537static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 538{ \ 539 gen_unary_fp32_sse(s, env, decode, \ 540 gen_helper_##lname##ps_xmm, \ 541 gen_helper_##lname##ps_ymm, \ 542 gen_helper_##lname##ss); \ 543} 544UNARY_FP32_SSE(VRSQRT, rsqrt) 545UNARY_FP32_SSE(VRCP, rcp) 546 547/* 548 * 66 = v*pd Vpd, Hpd, Wpd 549 * f2 = v*ps Vps, Hps, Wps 550 */ 551static inline void gen_horizontal_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 552 SSEFunc_0_eppp pd_xmm, SSEFunc_0_eppp ps_xmm, 553 SSEFunc_0_eppp pd_ymm, SSEFunc_0_eppp ps_ymm) 554{ 555 SSEFunc_0_eppp ps, pd, fn; 556 ps = s->vex_l ? ps_ymm : ps_xmm; 557 pd = s->vex_l ? pd_ymm : pd_xmm; 558 fn = s->prefix & PREFIX_DATA ? pd : ps; 559 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 560} 561#define HORIZONTAL_FP_SSE(uname, lname) \ 562static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 563{ \ 564 gen_horizontal_fp_sse(s, env, decode, \ 565 gen_helper_##lname##pd_xmm, gen_helper_##lname##ps_xmm, \ 566 gen_helper_##lname##pd_ymm, gen_helper_##lname##ps_ymm); \ 567} 568HORIZONTAL_FP_SSE(VHADD, hadd) 569HORIZONTAL_FP_SSE(VHSUB, hsub) 570HORIZONTAL_FP_SSE(VADDSUB, addsub) 571 572static inline void gen_ternary_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 573 int op3, SSEFunc_0_epppp xmm, SSEFunc_0_epppp ymm) 574{ 575 SSEFunc_0_epppp fn = s->vex_l ? ymm : xmm; 576 TCGv_ptr ptr3 = tcg_temp_new_ptr(); 577 578 /* The format of the fourth input is Lx */ 579 tcg_gen_addi_ptr(ptr3, cpu_env, ZMM_OFFSET(op3)); 580 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, ptr3); 581 tcg_temp_free_ptr(ptr3); 582} 583#define TERNARY_SSE(uname, uvname, lname) \ 584static void gen_##uvname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 585{ \ 586 gen_ternary_sse(s, env, decode, (uint8_t)decode->immediate >> 4, \ 587 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 588} \ 589static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 590{ \ 591 gen_ternary_sse(s, env, decode, 0, \ 592 gen_helper_##lname##_xmm, gen_helper_##lname##_ymm); \ 593} 594TERNARY_SSE(BLENDVPS, VBLENDVPS, blendvps) 595TERNARY_SSE(BLENDVPD, VBLENDVPD, blendvpd) 596TERNARY_SSE(PBLENDVB, VPBLENDVB, pblendvb) 597 598static inline void gen_binary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 599 SSEFunc_0_epppi xmm, SSEFunc_0_epppi ymm) 600{ 601 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 602 if (!s->vex_l) { 603 xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 604 } else { 605 ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 606 } 607} 608 609#define BINARY_IMM_SSE(uname, lname) \ 610static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 611{ \ 612 gen_binary_imm_sse(s, env, decode, \ 613 gen_helper_##lname##_xmm, \ 614 gen_helper_##lname##_ymm); \ 615} 616 617BINARY_IMM_SSE(VBLENDPD, blendpd) 618BINARY_IMM_SSE(VBLENDPS, blendps) 619BINARY_IMM_SSE(VPBLENDW, pblendw) 620BINARY_IMM_SSE(VDDPS, dpps) 621#define gen_helper_dppd_ymm NULL 622BINARY_IMM_SSE(VDDPD, dppd) 623BINARY_IMM_SSE(VMPSADBW, mpsadbw) 624BINARY_IMM_SSE(PCLMULQDQ, pclmulqdq) 625 626 627#define UNARY_INT_GVEC(uname, func, ...) \ 628static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 629{ \ 630 int vec_len = vector_len(s, decode); \ 631 \ 632 func(__VA_ARGS__, decode->op[0].offset, \ 633 decode->op[2].offset, vec_len, vec_len); \ 634} 635UNARY_INT_GVEC(PABSB, tcg_gen_gvec_abs, MO_8) 636UNARY_INT_GVEC(PABSW, tcg_gen_gvec_abs, MO_16) 637UNARY_INT_GVEC(PABSD, tcg_gen_gvec_abs, MO_32) 638UNARY_INT_GVEC(VBROADCASTx128, tcg_gen_gvec_dup_mem, MO_128) 639UNARY_INT_GVEC(VPBROADCASTB, tcg_gen_gvec_dup_mem, MO_8) 640UNARY_INT_GVEC(VPBROADCASTW, tcg_gen_gvec_dup_mem, MO_16) 641UNARY_INT_GVEC(VPBROADCASTD, tcg_gen_gvec_dup_mem, MO_32) 642UNARY_INT_GVEC(VPBROADCASTQ, tcg_gen_gvec_dup_mem, MO_64) 643 644 645#define BINARY_INT_GVEC(uname, func, ...) \ 646static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 647{ \ 648 int vec_len = vector_len(s, decode); \ 649 \ 650 func(__VA_ARGS__, \ 651 decode->op[0].offset, decode->op[1].offset, \ 652 decode->op[2].offset, vec_len, vec_len); \ 653} 654 655BINARY_INT_GVEC(PADDB, tcg_gen_gvec_add, MO_8) 656BINARY_INT_GVEC(PADDW, tcg_gen_gvec_add, MO_16) 657BINARY_INT_GVEC(PADDD, tcg_gen_gvec_add, MO_32) 658BINARY_INT_GVEC(PADDQ, tcg_gen_gvec_add, MO_64) 659BINARY_INT_GVEC(PADDSB, tcg_gen_gvec_ssadd, MO_8) 660BINARY_INT_GVEC(PADDSW, tcg_gen_gvec_ssadd, MO_16) 661BINARY_INT_GVEC(PADDUSB, tcg_gen_gvec_usadd, MO_8) 662BINARY_INT_GVEC(PADDUSW, tcg_gen_gvec_usadd, MO_16) 663BINARY_INT_GVEC(PAND, tcg_gen_gvec_and, MO_64) 664BINARY_INT_GVEC(PCMPEQB, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_8) 665BINARY_INT_GVEC(PCMPEQD, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_32) 666BINARY_INT_GVEC(PCMPEQW, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_16) 667BINARY_INT_GVEC(PCMPEQQ, tcg_gen_gvec_cmp, TCG_COND_EQ, MO_64) 668BINARY_INT_GVEC(PCMPGTB, tcg_gen_gvec_cmp, TCG_COND_GT, MO_8) 669BINARY_INT_GVEC(PCMPGTW, tcg_gen_gvec_cmp, TCG_COND_GT, MO_16) 670BINARY_INT_GVEC(PCMPGTD, tcg_gen_gvec_cmp, TCG_COND_GT, MO_32) 671BINARY_INT_GVEC(PCMPGTQ, tcg_gen_gvec_cmp, TCG_COND_GT, MO_64) 672BINARY_INT_GVEC(PMAXSB, tcg_gen_gvec_smax, MO_8) 673BINARY_INT_GVEC(PMAXSW, tcg_gen_gvec_smax, MO_16) 674BINARY_INT_GVEC(PMAXSD, tcg_gen_gvec_smax, MO_32) 675BINARY_INT_GVEC(PMAXUB, tcg_gen_gvec_umax, MO_8) 676BINARY_INT_GVEC(PMAXUW, tcg_gen_gvec_umax, MO_16) 677BINARY_INT_GVEC(PMAXUD, tcg_gen_gvec_umax, MO_32) 678BINARY_INT_GVEC(PMINSB, tcg_gen_gvec_smin, MO_8) 679BINARY_INT_GVEC(PMINSW, tcg_gen_gvec_smin, MO_16) 680BINARY_INT_GVEC(PMINSD, tcg_gen_gvec_smin, MO_32) 681BINARY_INT_GVEC(PMINUB, tcg_gen_gvec_umin, MO_8) 682BINARY_INT_GVEC(PMINUW, tcg_gen_gvec_umin, MO_16) 683BINARY_INT_GVEC(PMINUD, tcg_gen_gvec_umin, MO_32) 684BINARY_INT_GVEC(PMULLW, tcg_gen_gvec_mul, MO_16) 685BINARY_INT_GVEC(PMULLD, tcg_gen_gvec_mul, MO_32) 686BINARY_INT_GVEC(POR, tcg_gen_gvec_or, MO_64) 687BINARY_INT_GVEC(PSUBB, tcg_gen_gvec_sub, MO_8) 688BINARY_INT_GVEC(PSUBW, tcg_gen_gvec_sub, MO_16) 689BINARY_INT_GVEC(PSUBD, tcg_gen_gvec_sub, MO_32) 690BINARY_INT_GVEC(PSUBQ, tcg_gen_gvec_sub, MO_64) 691BINARY_INT_GVEC(PSUBSB, tcg_gen_gvec_sssub, MO_8) 692BINARY_INT_GVEC(PSUBSW, tcg_gen_gvec_sssub, MO_16) 693BINARY_INT_GVEC(PSUBUSB, tcg_gen_gvec_ussub, MO_8) 694BINARY_INT_GVEC(PSUBUSW, tcg_gen_gvec_ussub, MO_16) 695BINARY_INT_GVEC(PXOR, tcg_gen_gvec_xor, MO_64) 696 697 698/* 699 * 00 = p* Pq, Qq (if mmx not NULL; no VEX) 700 * 66 = vp* Vx, Hx, Wx 701 * 702 * These are really the same encoding, because 1) V is the same as P when VEX.V 703 * is not present 2) P and Q are the same as H and W apart from MM/XMM 704 */ 705static inline void gen_binary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 706 SSEFunc_0_eppp mmx, SSEFunc_0_eppp xmm, SSEFunc_0_eppp ymm) 707{ 708 assert(!!mmx == !!(decode->e.special == X86_SPECIAL_MMX)); 709 710 if (mmx && (s->prefix & PREFIX_VEX) && !(s->prefix & PREFIX_DATA)) { 711 /* VEX encoding is not applicable to MMX instructions. */ 712 gen_illegal_opcode(s); 713 return; 714 } 715 if (!(s->prefix & PREFIX_DATA)) { 716 mmx(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 717 } else if (!s->vex_l) { 718 xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 719 } else { 720 ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 721 } 722} 723 724 725#define BINARY_INT_MMX(uname, lname) \ 726static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 727{ \ 728 gen_binary_int_sse(s, env, decode, \ 729 gen_helper_##lname##_mmx, \ 730 gen_helper_##lname##_xmm, \ 731 gen_helper_##lname##_ymm); \ 732} 733BINARY_INT_MMX(PUNPCKLBW, punpcklbw) 734BINARY_INT_MMX(PUNPCKLWD, punpcklwd) 735BINARY_INT_MMX(PUNPCKLDQ, punpckldq) 736BINARY_INT_MMX(PACKSSWB, packsswb) 737BINARY_INT_MMX(PACKUSWB, packuswb) 738BINARY_INT_MMX(PUNPCKHBW, punpckhbw) 739BINARY_INT_MMX(PUNPCKHWD, punpckhwd) 740BINARY_INT_MMX(PUNPCKHDQ, punpckhdq) 741BINARY_INT_MMX(PACKSSDW, packssdw) 742 743BINARY_INT_MMX(PAVGB, pavgb) 744BINARY_INT_MMX(PAVGW, pavgw) 745BINARY_INT_MMX(PMADDWD, pmaddwd) 746BINARY_INT_MMX(PMULHUW, pmulhuw) 747BINARY_INT_MMX(PMULHW, pmulhw) 748BINARY_INT_MMX(PMULUDQ, pmuludq) 749BINARY_INT_MMX(PSADBW, psadbw) 750 751BINARY_INT_MMX(PSLLW_r, psllw) 752BINARY_INT_MMX(PSLLD_r, pslld) 753BINARY_INT_MMX(PSLLQ_r, psllq) 754BINARY_INT_MMX(PSRLW_r, psrlw) 755BINARY_INT_MMX(PSRLD_r, psrld) 756BINARY_INT_MMX(PSRLQ_r, psrlq) 757BINARY_INT_MMX(PSRAW_r, psraw) 758BINARY_INT_MMX(PSRAD_r, psrad) 759 760BINARY_INT_MMX(PHADDW, phaddw) 761BINARY_INT_MMX(PHADDSW, phaddsw) 762BINARY_INT_MMX(PHADDD, phaddd) 763BINARY_INT_MMX(PHSUBW, phsubw) 764BINARY_INT_MMX(PHSUBSW, phsubsw) 765BINARY_INT_MMX(PHSUBD, phsubd) 766BINARY_INT_MMX(PMADDUBSW, pmaddubsw) 767BINARY_INT_MMX(PSHUFB, pshufb) 768BINARY_INT_MMX(PSIGNB, psignb) 769BINARY_INT_MMX(PSIGNW, psignw) 770BINARY_INT_MMX(PSIGND, psignd) 771BINARY_INT_MMX(PMULHRSW, pmulhrsw) 772 773/* Instructions with no MMX equivalent. */ 774#define BINARY_INT_SSE(uname, lname) \ 775static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 776{ \ 777 gen_binary_int_sse(s, env, decode, \ 778 NULL, \ 779 gen_helper_##lname##_xmm, \ 780 gen_helper_##lname##_ymm); \ 781} 782 783/* Instructions with no MMX equivalent. */ 784BINARY_INT_SSE(PUNPCKLQDQ, punpcklqdq) 785BINARY_INT_SSE(PUNPCKHQDQ, punpckhqdq) 786BINARY_INT_SSE(VPACKUSDW, packusdw) 787BINARY_INT_SSE(VPERMILPS, vpermilps) 788BINARY_INT_SSE(VPERMILPD, vpermilpd) 789BINARY_INT_SSE(VMASKMOVPS, vpmaskmovd) 790BINARY_INT_SSE(VMASKMOVPD, vpmaskmovq) 791 792BINARY_INT_SSE(PMULDQ, pmuldq) 793 794BINARY_INT_SSE(VAESDEC, aesdec) 795BINARY_INT_SSE(VAESDECLAST, aesdeclast) 796BINARY_INT_SSE(VAESENC, aesenc) 797BINARY_INT_SSE(VAESENCLAST, aesenclast) 798 799#define UNARY_CMP_SSE(uname, lname) \ 800static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 801{ \ 802 if (!s->vex_l) { \ 803 gen_helper_##lname##_xmm(cpu_env, OP_PTR1, OP_PTR2); \ 804 } else { \ 805 gen_helper_##lname##_ymm(cpu_env, OP_PTR1, OP_PTR2); \ 806 } \ 807 set_cc_op(s, CC_OP_EFLAGS); \ 808} 809UNARY_CMP_SSE(VPTEST, ptest) 810UNARY_CMP_SSE(VTESTPS, vtestps) 811UNARY_CMP_SSE(VTESTPD, vtestpd) 812 813static inline void gen_unary_int_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 814 SSEFunc_0_epp xmm, SSEFunc_0_epp ymm) 815{ 816 if (!s->vex_l) { 817 xmm(cpu_env, OP_PTR0, OP_PTR2); 818 } else { 819 ymm(cpu_env, OP_PTR0, OP_PTR2); 820 } 821} 822 823#define UNARY_INT_SSE(uname, lname) \ 824static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 825{ \ 826 gen_unary_int_sse(s, env, decode, \ 827 gen_helper_##lname##_xmm, \ 828 gen_helper_##lname##_ymm); \ 829} 830 831UNARY_INT_SSE(VPMOVSXBW, pmovsxbw) 832UNARY_INT_SSE(VPMOVSXBD, pmovsxbd) 833UNARY_INT_SSE(VPMOVSXBQ, pmovsxbq) 834UNARY_INT_SSE(VPMOVSXWD, pmovsxwd) 835UNARY_INT_SSE(VPMOVSXWQ, pmovsxwq) 836UNARY_INT_SSE(VPMOVSXDQ, pmovsxdq) 837 838UNARY_INT_SSE(VPMOVZXBW, pmovzxbw) 839UNARY_INT_SSE(VPMOVZXBD, pmovzxbd) 840UNARY_INT_SSE(VPMOVZXBQ, pmovzxbq) 841UNARY_INT_SSE(VPMOVZXWD, pmovzxwd) 842UNARY_INT_SSE(VPMOVZXWQ, pmovzxwq) 843UNARY_INT_SSE(VPMOVZXDQ, pmovzxdq) 844 845UNARY_INT_SSE(VMOVSLDUP, pmovsldup) 846UNARY_INT_SSE(VMOVSHDUP, pmovshdup) 847UNARY_INT_SSE(VMOVDDUP, pmovdldup) 848 849UNARY_INT_SSE(VCVTDQ2PD, cvtdq2pd) 850UNARY_INT_SSE(VCVTPD2DQ, cvtpd2dq) 851UNARY_INT_SSE(VCVTTPD2DQ, cvttpd2dq) 852UNARY_INT_SSE(VCVTDQ2PS, cvtdq2ps) 853UNARY_INT_SSE(VCVTPS2DQ, cvtps2dq) 854UNARY_INT_SSE(VCVTTPS2DQ, cvttps2dq) 855 856 857static inline void gen_unary_imm_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 858 SSEFunc_0_ppi xmm, SSEFunc_0_ppi ymm) 859{ 860 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 861 if (!s->vex_l) { 862 xmm(OP_PTR0, OP_PTR1, imm); 863 } else { 864 ymm(OP_PTR0, OP_PTR1, imm); 865 } 866} 867 868#define UNARY_IMM_SSE(uname, lname) \ 869static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 870{ \ 871 gen_unary_imm_sse(s, env, decode, \ 872 gen_helper_##lname##_xmm, \ 873 gen_helper_##lname##_ymm); \ 874} 875 876UNARY_IMM_SSE(PSHUFD, pshufd) 877UNARY_IMM_SSE(PSHUFHW, pshufhw) 878UNARY_IMM_SSE(PSHUFLW, pshuflw) 879#define gen_helper_vpermq_xmm NULL 880UNARY_IMM_SSE(VPERMQ, vpermq) 881UNARY_IMM_SSE(VPERMILPS_i, vpermilps_imm) 882UNARY_IMM_SSE(VPERMILPD_i, vpermilpd_imm) 883 884static inline void gen_unary_imm_fp_sse(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 885 SSEFunc_0_eppi xmm, SSEFunc_0_eppi ymm) 886{ 887 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 888 if (!s->vex_l) { 889 xmm(cpu_env, OP_PTR0, OP_PTR1, imm); 890 } else { 891 ymm(cpu_env, OP_PTR0, OP_PTR1, imm); 892 } 893} 894 895#define UNARY_IMM_FP_SSE(uname, lname) \ 896static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 897{ \ 898 gen_unary_imm_fp_sse(s, env, decode, \ 899 gen_helper_##lname##_xmm, \ 900 gen_helper_##lname##_ymm); \ 901} 902 903UNARY_IMM_FP_SSE(VROUNDPS, roundps) 904UNARY_IMM_FP_SSE(VROUNDPD, roundpd) 905 906static inline void gen_vexw_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 907 SSEFunc_0_eppp d_xmm, SSEFunc_0_eppp q_xmm, 908 SSEFunc_0_eppp d_ymm, SSEFunc_0_eppp q_ymm) 909{ 910 SSEFunc_0_eppp d = s->vex_l ? d_ymm : d_xmm; 911 SSEFunc_0_eppp q = s->vex_l ? q_ymm : q_xmm; 912 SSEFunc_0_eppp fn = s->vex_w ? q : d; 913 fn(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 914} 915 916/* VEX.W affects whether to operate on 32- or 64-bit elements. */ 917#define VEXW_AVX(uname, lname) \ 918static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 919{ \ 920 gen_vexw_avx(s, env, decode, \ 921 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 922 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 923} 924VEXW_AVX(VPSLLV, vpsllv) 925VEXW_AVX(VPSRLV, vpsrlv) 926VEXW_AVX(VPSRAV, vpsrav) 927VEXW_AVX(VPMASKMOV, vpmaskmov) 928 929/* Same as above, but with extra arguments to the helper. */ 930static inline void gen_vsib_avx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 931 SSEFunc_0_epppti d_xmm, SSEFunc_0_epppti q_xmm, 932 SSEFunc_0_epppti d_ymm, SSEFunc_0_epppti q_ymm) 933{ 934 SSEFunc_0_epppti d = s->vex_l ? d_ymm : d_xmm; 935 SSEFunc_0_epppti q = s->vex_l ? q_ymm : q_xmm; 936 SSEFunc_0_epppti fn = s->vex_w ? q : d; 937 TCGv_i32 scale = tcg_constant_i32(decode->mem.scale); 938 TCGv_ptr index = tcg_temp_new_ptr(); 939 940 /* Pass third input as (index, base, scale) */ 941 tcg_gen_addi_ptr(index, cpu_env, ZMM_OFFSET(decode->mem.index)); 942 fn(cpu_env, OP_PTR0, OP_PTR1, index, s->A0, scale); 943 944 /* 945 * There are two output operands, so zero OP1's high 128 bits 946 * in the VEX.128 case. 947 */ 948 if (!s->vex_l) { 949 int ymmh_ofs = vector_elem_offset(&decode->op[1], MO_128, 1); 950 tcg_gen_gvec_dup_imm(MO_64, ymmh_ofs, 16, 16, 0); 951 } 952 tcg_temp_free_ptr(index); 953} 954#define VSIB_AVX(uname, lname) \ 955static void gen_##uname(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) \ 956{ \ 957 gen_vsib_avx(s, env, decode, \ 958 gen_helper_##lname##d_xmm, gen_helper_##lname##q_xmm, \ 959 gen_helper_##lname##d_ymm, gen_helper_##lname##q_ymm); \ 960} 961VSIB_AVX(VPGATHERD, vpgatherd) 962VSIB_AVX(VPGATHERQ, vpgatherq) 963 964static void gen_ADCOX(DisasContext *s, CPUX86State *env, MemOp ot, int cc_op) 965{ 966 TCGv carry_in = NULL; 967 TCGv carry_out = (cc_op == CC_OP_ADCX ? cpu_cc_dst : cpu_cc_src2); 968 TCGv zero; 969 970 if (cc_op == s->cc_op || s->cc_op == CC_OP_ADCOX) { 971 /* Re-use the carry-out from a previous round. */ 972 carry_in = carry_out; 973 cc_op = s->cc_op; 974 } else if (s->cc_op == CC_OP_ADCX || s->cc_op == CC_OP_ADOX) { 975 /* Merge with the carry-out from the opposite instruction. */ 976 cc_op = CC_OP_ADCOX; 977 } 978 979 /* If we don't have a carry-in, get it out of EFLAGS. */ 980 if (!carry_in) { 981 if (s->cc_op != CC_OP_ADCX && s->cc_op != CC_OP_ADOX) { 982 gen_compute_eflags(s); 983 } 984 carry_in = s->tmp0; 985 tcg_gen_extract_tl(carry_in, cpu_cc_src, 986 ctz32(cc_op == CC_OP_ADCX ? CC_C : CC_O), 1); 987 } 988 989 switch (ot) { 990#ifdef TARGET_X86_64 991 case MO_32: 992 /* If TL is 64-bit just do everything in 64-bit arithmetic. */ 993 tcg_gen_add_i64(s->T0, s->T0, s->T1); 994 tcg_gen_add_i64(s->T0, s->T0, carry_in); 995 tcg_gen_shri_i64(carry_out, s->T0, 32); 996 break; 997#endif 998 default: 999 zero = tcg_constant_tl(0); 1000 tcg_gen_add2_tl(s->T0, carry_out, s->T0, zero, carry_in, zero); 1001 tcg_gen_add2_tl(s->T0, carry_out, s->T0, carry_out, s->T1, zero); 1002 break; 1003 } 1004 set_cc_op(s, cc_op); 1005} 1006 1007static void gen_ADCX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1008{ 1009 gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADCX); 1010} 1011 1012static void gen_ADOX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1013{ 1014 gen_ADCOX(s, env, decode->op[0].ot, CC_OP_ADOX); 1015} 1016 1017static void gen_ANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1018{ 1019 MemOp ot = decode->op[0].ot; 1020 1021 tcg_gen_andc_tl(s->T0, s->T1, s->T0); 1022 gen_op_update1_cc(s); 1023 set_cc_op(s, CC_OP_LOGICB + ot); 1024} 1025 1026static void gen_BEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1027{ 1028 MemOp ot = decode->op[0].ot; 1029 TCGv bound, zero; 1030 1031 /* 1032 * Extract START, and shift the operand. 1033 * Shifts larger than operand size get zeros. 1034 */ 1035 tcg_gen_ext8u_tl(s->A0, s->T1); 1036 tcg_gen_shr_tl(s->T0, s->T0, s->A0); 1037 1038 bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1039 zero = tcg_constant_tl(0); 1040 tcg_gen_movcond_tl(TCG_COND_LEU, s->T0, s->A0, bound, s->T0, zero); 1041 1042 /* 1043 * Extract the LEN into a mask. Lengths larger than 1044 * operand size get all ones. 1045 */ 1046 tcg_gen_extract_tl(s->A0, s->T1, 8, 8); 1047 tcg_gen_movcond_tl(TCG_COND_LEU, s->A0, s->A0, bound, s->A0, bound); 1048 1049 tcg_gen_movi_tl(s->T1, 1); 1050 tcg_gen_shl_tl(s->T1, s->T1, s->A0); 1051 tcg_gen_subi_tl(s->T1, s->T1, 1); 1052 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1053 1054 gen_op_update1_cc(s); 1055 set_cc_op(s, CC_OP_LOGICB + ot); 1056} 1057 1058static void gen_BLSI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1059{ 1060 MemOp ot = decode->op[0].ot; 1061 1062 tcg_gen_neg_tl(s->T1, s->T0); 1063 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1064 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1065 set_cc_op(s, CC_OP_BMILGB + ot); 1066} 1067 1068static void gen_BLSMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1069{ 1070 MemOp ot = decode->op[0].ot; 1071 1072 tcg_gen_subi_tl(s->T1, s->T0, 1); 1073 tcg_gen_xor_tl(s->T0, s->T0, s->T1); 1074 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1075 set_cc_op(s, CC_OP_BMILGB + ot); 1076} 1077 1078static void gen_BLSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1079{ 1080 MemOp ot = decode->op[0].ot; 1081 1082 tcg_gen_subi_tl(s->T1, s->T0, 1); 1083 tcg_gen_and_tl(s->T0, s->T0, s->T1); 1084 tcg_gen_mov_tl(cpu_cc_dst, s->T0); 1085 set_cc_op(s, CC_OP_BMILGB + ot); 1086} 1087 1088static void gen_BZHI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1089{ 1090 MemOp ot = decode->op[0].ot; 1091 TCGv bound; 1092 1093 tcg_gen_ext8u_tl(s->T1, cpu_regs[s->vex_v]); 1094 bound = tcg_constant_tl(ot == MO_64 ? 63 : 31); 1095 1096 /* 1097 * Note that since we're using BMILG (in order to get O 1098 * cleared) we need to store the inverse into C. 1099 */ 1100 tcg_gen_setcond_tl(TCG_COND_LT, cpu_cc_src, s->T1, bound); 1101 tcg_gen_movcond_tl(TCG_COND_GT, s->T1, s->T1, bound, bound, s->T1); 1102 1103 tcg_gen_movi_tl(s->A0, -1); 1104 tcg_gen_shl_tl(s->A0, s->A0, s->T1); 1105 tcg_gen_andc_tl(s->T0, s->T0, s->A0); 1106 1107 gen_op_update1_cc(s); 1108 set_cc_op(s, CC_OP_BMILGB + ot); 1109} 1110 1111static void gen_CRC32(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1112{ 1113 MemOp ot = decode->op[2].ot; 1114 1115 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1116 gen_helper_crc32(s->T0, s->tmp2_i32, s->T1, tcg_constant_i32(8 << ot)); 1117} 1118 1119static void gen_CVTPI2Px(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1120{ 1121 gen_helper_enter_mmx(cpu_env); 1122 if (s->prefix & PREFIX_DATA) { 1123 gen_helper_cvtpi2pd(cpu_env, OP_PTR0, OP_PTR2); 1124 } else { 1125 gen_helper_cvtpi2ps(cpu_env, OP_PTR0, OP_PTR2); 1126 } 1127} 1128 1129static void gen_CVTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1130{ 1131 gen_helper_enter_mmx(cpu_env); 1132 if (s->prefix & PREFIX_DATA) { 1133 gen_helper_cvtpd2pi(cpu_env, OP_PTR0, OP_PTR2); 1134 } else { 1135 gen_helper_cvtps2pi(cpu_env, OP_PTR0, OP_PTR2); 1136 } 1137} 1138 1139static void gen_CVTTPx2PI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1140{ 1141 gen_helper_enter_mmx(cpu_env); 1142 if (s->prefix & PREFIX_DATA) { 1143 gen_helper_cvttpd2pi(cpu_env, OP_PTR0, OP_PTR2); 1144 } else { 1145 gen_helper_cvttps2pi(cpu_env, OP_PTR0, OP_PTR2); 1146 } 1147} 1148 1149static void gen_EMMS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1150{ 1151 gen_helper_emms(cpu_env); 1152} 1153 1154static void gen_EXTRQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1155{ 1156 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1157 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1158 1159 gen_helper_extrq_i(cpu_env, OP_PTR0, index, length); 1160} 1161 1162static void gen_EXTRQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1163{ 1164 gen_helper_extrq_r(cpu_env, OP_PTR0, OP_PTR2); 1165} 1166 1167static void gen_INSERTQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1168{ 1169 TCGv_i32 length = tcg_constant_i32(decode->immediate & 63); 1170 TCGv_i32 index = tcg_constant_i32((decode->immediate >> 8) & 63); 1171 1172 gen_helper_insertq_i(cpu_env, OP_PTR0, OP_PTR1, index, length); 1173} 1174 1175static void gen_INSERTQ_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1176{ 1177 gen_helper_insertq_r(cpu_env, OP_PTR0, OP_PTR2); 1178} 1179 1180static void gen_LDMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1181{ 1182 if (s->vex_l) { 1183 gen_illegal_opcode(s); 1184 return; 1185 } 1186 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T1); 1187 gen_helper_ldmxcsr(cpu_env, s->tmp2_i32); 1188} 1189 1190static void gen_MASKMOV(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1191{ 1192 tcg_gen_mov_tl(s->A0, cpu_regs[R_EDI]); 1193 gen_extu(s->aflag, s->A0); 1194 gen_add_A0_ds_seg(s); 1195 1196 if (s->prefix & PREFIX_DATA) { 1197 gen_helper_maskmov_xmm(cpu_env, OP_PTR1, OP_PTR2, s->A0); 1198 } else { 1199 gen_helper_maskmov_mmx(cpu_env, OP_PTR1, OP_PTR2, s->A0); 1200 } 1201} 1202 1203static void gen_MOVBE(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1204{ 1205 MemOp ot = decode->op[0].ot; 1206 1207 /* M operand type does not load/store */ 1208 if (decode->e.op0 == X86_TYPE_M) { 1209 tcg_gen_qemu_st_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 1210 } else { 1211 tcg_gen_qemu_ld_tl(s->T0, s->A0, s->mem_index, ot | MO_BE); 1212 } 1213} 1214 1215static void gen_MOVD_from(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1216{ 1217 MemOp ot = decode->op[2].ot; 1218 1219 switch (ot) { 1220 case MO_32: 1221#ifdef TARGET_X86_64 1222 tcg_gen_ld32u_tl(s->T0, cpu_env, decode->op[2].offset); 1223 break; 1224 case MO_64: 1225#endif 1226 tcg_gen_ld_tl(s->T0, cpu_env, decode->op[2].offset); 1227 break; 1228 default: 1229 abort(); 1230 } 1231} 1232 1233static void gen_MOVD_to(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1234{ 1235 MemOp ot = decode->op[2].ot; 1236 int vec_len = vector_len(s, decode); 1237 int lo_ofs = vector_elem_offset(&decode->op[0], ot, 0); 1238 1239 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1240 1241 switch (ot) { 1242 case MO_32: 1243#ifdef TARGET_X86_64 1244 tcg_gen_st32_tl(s->T1, cpu_env, lo_ofs); 1245 break; 1246 case MO_64: 1247#endif 1248 tcg_gen_st_tl(s->T1, cpu_env, lo_ofs); 1249 break; 1250 default: 1251 g_assert_not_reached(); 1252 } 1253} 1254 1255static void gen_MOVDQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1256{ 1257 gen_store_sse(s, decode, decode->op[2].offset); 1258} 1259 1260static void gen_MOVMSK(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1261{ 1262 typeof(gen_helper_movmskps_ymm) *ps, *pd, *fn; 1263 ps = s->vex_l ? gen_helper_movmskps_ymm : gen_helper_movmskps_xmm; 1264 pd = s->vex_l ? gen_helper_movmskpd_ymm : gen_helper_movmskpd_xmm; 1265 fn = s->prefix & PREFIX_DATA ? pd : ps; 1266 fn(s->tmp2_i32, cpu_env, OP_PTR2); 1267 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 1268} 1269 1270static void gen_MOVQ(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1271{ 1272 int vec_len = vector_len(s, decode); 1273 int lo_ofs = vector_elem_offset(&decode->op[0], MO_64, 0); 1274 1275 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset); 1276 if (decode->op[0].has_ea) { 1277 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 1278 } else { 1279 /* 1280 * tcg_gen_gvec_dup_i64(MO_64, op0.offset, 8, vec_len, s->tmp1_64) would 1281 * seem to work, but it does not on big-endian platforms; the cleared parts 1282 * are always at higher addresses, but cross-endian emulation inverts the 1283 * byte order so that the cleared parts need to be at *lower* addresses. 1284 * Because oprsz is 8, we see this here even for SSE; but more in general, 1285 * it disqualifies using oprsz < maxsz to emulate VEX128. 1286 */ 1287 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1288 tcg_gen_st_i64(s->tmp1_i64, cpu_env, lo_ofs); 1289 } 1290} 1291 1292static void gen_MOVq_dq(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1293{ 1294 gen_helper_enter_mmx(cpu_env); 1295 /* Otherwise the same as any other movq. */ 1296 return gen_MOVQ(s, env, decode); 1297} 1298 1299static void gen_MULX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1300{ 1301 MemOp ot = decode->op[0].ot; 1302 1303 /* low part of result in VEX.vvvv, high in MODRM */ 1304 switch (ot) { 1305 default: 1306 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1307 tcg_gen_trunc_tl_i32(s->tmp3_i32, s->T1); 1308 tcg_gen_mulu2_i32(s->tmp2_i32, s->tmp3_i32, 1309 s->tmp2_i32, s->tmp3_i32); 1310 tcg_gen_extu_i32_tl(cpu_regs[s->vex_v], s->tmp2_i32); 1311 tcg_gen_extu_i32_tl(s->T0, s->tmp3_i32); 1312 break; 1313#ifdef TARGET_X86_64 1314 case MO_64: 1315 tcg_gen_mulu2_i64(cpu_regs[s->vex_v], s->T0, s->T0, s->T1); 1316 break; 1317#endif 1318 } 1319 1320} 1321 1322static void gen_PALIGNR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1323{ 1324 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1325 if (!(s->prefix & PREFIX_DATA)) { 1326 gen_helper_palignr_mmx(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 1327 } else if (!s->vex_l) { 1328 gen_helper_palignr_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 1329 } else { 1330 gen_helper_palignr_ymm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 1331 } 1332} 1333 1334static void gen_PANDN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1335{ 1336 int vec_len = vector_len(s, decode); 1337 1338 /* Careful, operand order is reversed! */ 1339 tcg_gen_gvec_andc(MO_64, 1340 decode->op[0].offset, decode->op[2].offset, 1341 decode->op[1].offset, vec_len, vec_len); 1342} 1343 1344static void gen_PCMPESTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1345{ 1346 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1347 gen_helper_pcmpestri_xmm(cpu_env, OP_PTR1, OP_PTR2, imm); 1348 set_cc_op(s, CC_OP_EFLAGS); 1349} 1350 1351static void gen_PCMPESTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1352{ 1353 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1354 gen_helper_pcmpestrm_xmm(cpu_env, OP_PTR1, OP_PTR2, imm); 1355 set_cc_op(s, CC_OP_EFLAGS); 1356 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 1357 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 1358 16, 16, 0); 1359 } 1360} 1361 1362static void gen_PCMPISTRI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1363{ 1364 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1365 gen_helper_pcmpistri_xmm(cpu_env, OP_PTR1, OP_PTR2, imm); 1366 set_cc_op(s, CC_OP_EFLAGS); 1367} 1368 1369static void gen_PCMPISTRM(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1370{ 1371 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1372 gen_helper_pcmpistrm_xmm(cpu_env, OP_PTR1, OP_PTR2, imm); 1373 set_cc_op(s, CC_OP_EFLAGS); 1374 if ((s->prefix & PREFIX_VEX) && !s->vex_l) { 1375 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_regs[0].ZMM_X(1)), 1376 16, 16, 0); 1377 } 1378} 1379 1380static void gen_PDEP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1381{ 1382 MemOp ot = decode->op[1].ot; 1383 if (ot < MO_64) { 1384 tcg_gen_ext32u_tl(s->T0, s->T0); 1385 } 1386 gen_helper_pdep(s->T0, s->T0, s->T1); 1387} 1388 1389static void gen_PEXT(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1390{ 1391 MemOp ot = decode->op[1].ot; 1392 if (ot < MO_64) { 1393 tcg_gen_ext32u_tl(s->T0, s->T0); 1394 } 1395 gen_helper_pext(s->T0, s->T0, s->T1); 1396} 1397 1398static inline void gen_pextr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot) 1399{ 1400 int vec_len = vector_len(s, decode); 1401 int mask = (vec_len >> ot) - 1; 1402 int val = decode->immediate & mask; 1403 1404 switch (ot) { 1405 case MO_8: 1406 tcg_gen_ld8u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val)); 1407 break; 1408 case MO_16: 1409 tcg_gen_ld16u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val)); 1410 break; 1411 case MO_32: 1412#ifdef TARGET_X86_64 1413 tcg_gen_ld32u_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val)); 1414 break; 1415 case MO_64: 1416#endif 1417 tcg_gen_ld_tl(s->T0, cpu_env, vector_elem_offset(&decode->op[1], ot, val)); 1418 break; 1419 default: 1420 abort(); 1421 } 1422} 1423 1424static void gen_PEXTRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1425{ 1426 gen_pextr(s, env, decode, MO_8); 1427} 1428 1429static void gen_PEXTRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1430{ 1431 gen_pextr(s, env, decode, MO_16); 1432} 1433 1434static void gen_PEXTR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1435{ 1436 MemOp ot = decode->op[0].ot; 1437 gen_pextr(s, env, decode, ot); 1438} 1439 1440static inline void gen_pinsr(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, MemOp ot) 1441{ 1442 int vec_len = vector_len(s, decode); 1443 int mask = (vec_len >> ot) - 1; 1444 int val = decode->immediate & mask; 1445 1446 if (decode->op[1].offset != decode->op[0].offset) { 1447 assert(vec_len == 16); 1448 gen_store_sse(s, decode, decode->op[1].offset); 1449 } 1450 1451 switch (ot) { 1452 case MO_8: 1453 tcg_gen_st8_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val)); 1454 break; 1455 case MO_16: 1456 tcg_gen_st16_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val)); 1457 break; 1458 case MO_32: 1459#ifdef TARGET_X86_64 1460 tcg_gen_st32_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val)); 1461 break; 1462 case MO_64: 1463#endif 1464 tcg_gen_st_tl(s->T1, cpu_env, vector_elem_offset(&decode->op[0], ot, val)); 1465 break; 1466 default: 1467 abort(); 1468 } 1469} 1470 1471static void gen_PINSRB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1472{ 1473 gen_pinsr(s, env, decode, MO_8); 1474} 1475 1476static void gen_PINSRW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1477{ 1478 gen_pinsr(s, env, decode, MO_16); 1479} 1480 1481static void gen_PINSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1482{ 1483 gen_pinsr(s, env, decode, decode->op[2].ot); 1484} 1485 1486static void gen_pmovmskb_i64(TCGv_i64 d, TCGv_i64 s) 1487{ 1488 TCGv_i64 t = tcg_temp_new_i64(); 1489 1490 tcg_gen_andi_i64(d, s, 0x8080808080808080ull); 1491 1492 /* 1493 * After each shift+or pair: 1494 * 0: a.......b.......c.......d.......e.......f.......g.......h....... 1495 * 7: ab......bc......cd......de......ef......fg......gh......h....... 1496 * 14: abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... 1497 * 28: abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... 1498 * The result is left in the high bits of the word. 1499 */ 1500 tcg_gen_shli_i64(t, d, 7); 1501 tcg_gen_or_i64(d, d, t); 1502 tcg_gen_shli_i64(t, d, 14); 1503 tcg_gen_or_i64(d, d, t); 1504 tcg_gen_shli_i64(t, d, 28); 1505 tcg_gen_or_i64(d, d, t); 1506} 1507 1508static void gen_pmovmskb_vec(unsigned vece, TCGv_vec d, TCGv_vec s) 1509{ 1510 TCGv_vec t = tcg_temp_new_vec_matching(d); 1511 TCGv_vec m = tcg_constant_vec_matching(d, MO_8, 0x80); 1512 1513 /* See above */ 1514 tcg_gen_and_vec(vece, d, s, m); 1515 tcg_gen_shli_vec(vece, t, d, 7); 1516 tcg_gen_or_vec(vece, d, d, t); 1517 tcg_gen_shli_vec(vece, t, d, 14); 1518 tcg_gen_or_vec(vece, d, d, t); 1519 tcg_gen_shli_vec(vece, t, d, 28); 1520 tcg_gen_or_vec(vece, d, d, t); 1521} 1522 1523#ifdef TARGET_X86_64 1524#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i64 1525#else 1526#define TCG_TARGET_HAS_extract2_tl TCG_TARGET_HAS_extract2_i32 1527#endif 1528 1529static void gen_PMOVMSKB(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1530{ 1531 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 }; 1532 static const GVecGen2 g = { 1533 .fni8 = gen_pmovmskb_i64, 1534 .fniv = gen_pmovmskb_vec, 1535 .opt_opc = vecop_list, 1536 .vece = MO_64, 1537 .prefer_i64 = TCG_TARGET_REG_BITS == 64 1538 }; 1539 MemOp ot = decode->op[2].ot; 1540 int vec_len = vector_len(s, decode); 1541 TCGv t = tcg_temp_new(); 1542 1543 tcg_gen_gvec_2(offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), decode->op[2].offset, 1544 vec_len, vec_len, &g); 1545 tcg_gen_ld8u_tl(s->T0, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 1546 while (vec_len > 8) { 1547 vec_len -= 8; 1548 if (TCG_TARGET_HAS_extract2_tl) { 1549 /* 1550 * Load the next byte of the result into the high byte of T. 1551 * TCG does a similar expansion of deposit to shl+extract2; by 1552 * loading the whole word, the shift left is avoided. 1553 */ 1554#ifdef TARGET_X86_64 1555 tcg_gen_ld_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_Q((vec_len - 1) / 8))); 1556#else 1557 tcg_gen_ld_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_L((vec_len - 1) / 4))); 1558#endif 1559 1560 tcg_gen_extract2_tl(s->T0, t, s->T0, TARGET_LONG_BITS - 8); 1561 } else { 1562 /* 1563 * The _previous_ value is deposited into bits 8 and higher of t. Because 1564 * those bits are known to be zero after ld8u, this becomes a shift+or 1565 * if deposit is not available. 1566 */ 1567 tcg_gen_ld8u_tl(t, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_B(vec_len - 1))); 1568 tcg_gen_deposit_tl(s->T0, t, s->T0, 8, TARGET_LONG_BITS - 8); 1569 } 1570 } 1571 tcg_temp_free(t); 1572} 1573 1574static void gen_PSHUFW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1575{ 1576 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1577 gen_helper_pshufw_mmx(OP_PTR0, OP_PTR1, imm); 1578} 1579 1580static void gen_PSRLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1581{ 1582 int vec_len = vector_len(s, decode); 1583 1584 if (decode->immediate >= 16) { 1585 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1586 } else { 1587 tcg_gen_gvec_shri(MO_16, 1588 decode->op[0].offset, decode->op[1].offset, 1589 decode->immediate, vec_len, vec_len); 1590 } 1591} 1592 1593static void gen_PSLLW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1594{ 1595 int vec_len = vector_len(s, decode); 1596 1597 if (decode->immediate >= 16) { 1598 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1599 } else { 1600 tcg_gen_gvec_shli(MO_16, 1601 decode->op[0].offset, decode->op[1].offset, 1602 decode->immediate, vec_len, vec_len); 1603 } 1604} 1605 1606static void gen_PSRAW_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1607{ 1608 int vec_len = vector_len(s, decode); 1609 1610 if (decode->immediate >= 16) { 1611 decode->immediate = 15; 1612 } 1613 tcg_gen_gvec_sari(MO_16, 1614 decode->op[0].offset, decode->op[1].offset, 1615 decode->immediate, vec_len, vec_len); 1616} 1617 1618static void gen_PSRLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1619{ 1620 int vec_len = vector_len(s, decode); 1621 1622 if (decode->immediate >= 32) { 1623 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1624 } else { 1625 tcg_gen_gvec_shri(MO_32, 1626 decode->op[0].offset, decode->op[1].offset, 1627 decode->immediate, vec_len, vec_len); 1628 } 1629} 1630 1631static void gen_PSLLD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1632{ 1633 int vec_len = vector_len(s, decode); 1634 1635 if (decode->immediate >= 32) { 1636 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1637 } else { 1638 tcg_gen_gvec_shli(MO_32, 1639 decode->op[0].offset, decode->op[1].offset, 1640 decode->immediate, vec_len, vec_len); 1641 } 1642} 1643 1644static void gen_PSRAD_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1645{ 1646 int vec_len = vector_len(s, decode); 1647 1648 if (decode->immediate >= 32) { 1649 decode->immediate = 31; 1650 } 1651 tcg_gen_gvec_sari(MO_32, 1652 decode->op[0].offset, decode->op[1].offset, 1653 decode->immediate, vec_len, vec_len); 1654} 1655 1656static void gen_PSRLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1657{ 1658 int vec_len = vector_len(s, decode); 1659 1660 if (decode->immediate >= 64) { 1661 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1662 } else { 1663 tcg_gen_gvec_shri(MO_64, 1664 decode->op[0].offset, decode->op[1].offset, 1665 decode->immediate, vec_len, vec_len); 1666 } 1667} 1668 1669static void gen_PSLLQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1670{ 1671 int vec_len = vector_len(s, decode); 1672 1673 if (decode->immediate >= 64) { 1674 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1675 } else { 1676 tcg_gen_gvec_shli(MO_64, 1677 decode->op[0].offset, decode->op[1].offset, 1678 decode->immediate, vec_len, vec_len); 1679 } 1680} 1681 1682static TCGv_ptr make_imm8u_xmm_vec(uint8_t imm, int vec_len) 1683{ 1684 MemOp ot = vec_len == 16 ? MO_128 : MO_256; 1685 TCGv_i32 imm_v = tcg_constant8u_i32(imm); 1686 TCGv_ptr ptr = tcg_temp_new_ptr(); 1687 1688 tcg_gen_gvec_dup_imm(MO_64, offsetof(CPUX86State, xmm_t0) + xmm_offset(ot), 1689 vec_len, vec_len, 0); 1690 1691 tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_t0)); 1692 tcg_gen_st_i32(imm_v, cpu_env, offsetof(CPUX86State, xmm_t0.ZMM_L(0))); 1693 return ptr; 1694} 1695 1696static void gen_PSRLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1697{ 1698 int vec_len = vector_len(s, decode); 1699 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 1700 1701 if (s->vex_l) { 1702 gen_helper_psrldq_ymm(cpu_env, OP_PTR0, OP_PTR1, imm_vec); 1703 } else { 1704 gen_helper_psrldq_xmm(cpu_env, OP_PTR0, OP_PTR1, imm_vec); 1705 } 1706 tcg_temp_free_ptr(imm_vec); 1707} 1708 1709static void gen_PSLLDQ_i(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1710{ 1711 int vec_len = vector_len(s, decode); 1712 TCGv_ptr imm_vec = make_imm8u_xmm_vec(decode->immediate, vec_len); 1713 1714 if (s->vex_l) { 1715 gen_helper_pslldq_ymm(cpu_env, OP_PTR0, OP_PTR1, imm_vec); 1716 } else { 1717 gen_helper_pslldq_xmm(cpu_env, OP_PTR0, OP_PTR1, imm_vec); 1718 } 1719 tcg_temp_free_ptr(imm_vec); 1720} 1721 1722static void gen_RORX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1723{ 1724 MemOp ot = decode->op[0].ot; 1725 int b = decode->immediate; 1726 1727 if (ot == MO_64) { 1728 tcg_gen_rotri_tl(s->T0, s->T0, b & 63); 1729 } else { 1730 tcg_gen_trunc_tl_i32(s->tmp2_i32, s->T0); 1731 tcg_gen_rotri_i32(s->tmp2_i32, s->tmp2_i32, b & 31); 1732 tcg_gen_extu_i32_tl(s->T0, s->tmp2_i32); 1733 } 1734} 1735 1736static void gen_SARX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1737{ 1738 MemOp ot = decode->op[0].ot; 1739 int mask; 1740 1741 mask = ot == MO_64 ? 63 : 31; 1742 tcg_gen_andi_tl(s->T1, s->T1, mask); 1743 if (ot != MO_64) { 1744 tcg_gen_ext32s_tl(s->T0, s->T0); 1745 } 1746 tcg_gen_sar_tl(s->T0, s->T0, s->T1); 1747} 1748 1749static void gen_SHLX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1750{ 1751 MemOp ot = decode->op[0].ot; 1752 int mask; 1753 1754 mask = ot == MO_64 ? 63 : 31; 1755 tcg_gen_andi_tl(s->T1, s->T1, mask); 1756 tcg_gen_shl_tl(s->T0, s->T0, s->T1); 1757} 1758 1759static void gen_SHRX(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1760{ 1761 MemOp ot = decode->op[0].ot; 1762 int mask; 1763 1764 mask = ot == MO_64 ? 63 : 31; 1765 tcg_gen_andi_tl(s->T1, s->T1, mask); 1766 if (ot != MO_64) { 1767 tcg_gen_ext32u_tl(s->T0, s->T0); 1768 } 1769 tcg_gen_shr_tl(s->T0, s->T0, s->T1); 1770} 1771 1772static void gen_VAESKEYGEN(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1773{ 1774 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 1775 assert(!s->vex_l); 1776 gen_helper_aeskeygenassist_xmm(cpu_env, OP_PTR0, OP_PTR1, imm); 1777} 1778 1779static void gen_STMXCSR(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1780{ 1781 if (s->vex_l) { 1782 gen_illegal_opcode(s); 1783 return; 1784 } 1785 gen_helper_update_mxcsr(cpu_env); 1786 tcg_gen_ld32u_tl(s->T0, cpu_env, offsetof(CPUX86State, mxcsr)); 1787} 1788 1789static void gen_VAESIMC(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1790{ 1791 assert(!s->vex_l); 1792 gen_helper_aesimc_xmm(cpu_env, OP_PTR0, OP_PTR2); 1793} 1794 1795/* 1796 * 00 = v*ps Vps, Hps, Wpd 1797 * 66 = v*pd Vpd, Hpd, Wps 1798 * f3 = v*ss Vss, Hss, Wps 1799 * f2 = v*sd Vsd, Hsd, Wps 1800 */ 1801#define SSE_CMP(x) { \ 1802 gen_helper_ ## x ## ps ## _xmm, gen_helper_ ## x ## pd ## _xmm, \ 1803 gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, \ 1804 gen_helper_ ## x ## ps ## _ymm, gen_helper_ ## x ## pd ## _ymm} 1805static const SSEFunc_0_eppp gen_helper_cmp_funcs[32][6] = { 1806 SSE_CMP(cmpeq), 1807 SSE_CMP(cmplt), 1808 SSE_CMP(cmple), 1809 SSE_CMP(cmpunord), 1810 SSE_CMP(cmpneq), 1811 SSE_CMP(cmpnlt), 1812 SSE_CMP(cmpnle), 1813 SSE_CMP(cmpord), 1814 1815 SSE_CMP(cmpequ), 1816 SSE_CMP(cmpnge), 1817 SSE_CMP(cmpngt), 1818 SSE_CMP(cmpfalse), 1819 SSE_CMP(cmpnequ), 1820 SSE_CMP(cmpge), 1821 SSE_CMP(cmpgt), 1822 SSE_CMP(cmptrue), 1823 1824 SSE_CMP(cmpeqs), 1825 SSE_CMP(cmpltq), 1826 SSE_CMP(cmpleq), 1827 SSE_CMP(cmpunords), 1828 SSE_CMP(cmpneqq), 1829 SSE_CMP(cmpnltq), 1830 SSE_CMP(cmpnleq), 1831 SSE_CMP(cmpords), 1832 1833 SSE_CMP(cmpequs), 1834 SSE_CMP(cmpngeq), 1835 SSE_CMP(cmpngtq), 1836 SSE_CMP(cmpfalses), 1837 SSE_CMP(cmpnequs), 1838 SSE_CMP(cmpgeq), 1839 SSE_CMP(cmpgtq), 1840 SSE_CMP(cmptrues), 1841}; 1842#undef SSE_CMP 1843 1844static void gen_VCMP(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1845{ 1846 int index = decode->immediate & (s->prefix & PREFIX_VEX ? 31 : 7); 1847 int b = 1848 s->prefix & PREFIX_REPZ ? 2 /* ss */ : 1849 s->prefix & PREFIX_REPNZ ? 3 /* sd */ : 1850 !!(s->prefix & PREFIX_DATA) /* pd */ + (s->vex_l << 2); 1851 1852 gen_helper_cmp_funcs[index][b](cpu_env, OP_PTR0, OP_PTR1, OP_PTR2); 1853} 1854 1855static void gen_VCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1856{ 1857 SSEFunc_0_epp fn; 1858 fn = s->prefix & PREFIX_DATA ? gen_helper_comisd : gen_helper_comiss; 1859 fn(cpu_env, OP_PTR1, OP_PTR2); 1860 set_cc_op(s, CC_OP_EFLAGS); 1861} 1862 1863static void gen_VCVTfp2fp(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1864{ 1865 gen_unary_fp_sse(s, env, decode, 1866 gen_helper_cvtpd2ps_xmm, gen_helper_cvtps2pd_xmm, 1867 gen_helper_cvtpd2ps_ymm, gen_helper_cvtps2pd_ymm, 1868 gen_helper_cvtsd2ss, gen_helper_cvtss2sd); 1869} 1870 1871static void gen_VCVTSI2Sx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1872{ 1873 int vec_len = vector_len(s, decode); 1874 TCGv_i32 in; 1875 1876 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 1877 1878#ifdef TARGET_X86_64 1879 MemOp ot = decode->op[2].ot; 1880 if (ot == MO_64) { 1881 if (s->prefix & PREFIX_REPNZ) { 1882 gen_helper_cvtsq2sd(cpu_env, OP_PTR0, s->T1); 1883 } else { 1884 gen_helper_cvtsq2ss(cpu_env, OP_PTR0, s->T1); 1885 } 1886 return; 1887 } 1888 in = s->tmp2_i32; 1889 tcg_gen_trunc_tl_i32(in, s->T1); 1890#else 1891 in = s->T1; 1892#endif 1893 1894 if (s->prefix & PREFIX_REPNZ) { 1895 gen_helper_cvtsi2sd(cpu_env, OP_PTR0, in); 1896 } else { 1897 gen_helper_cvtsi2ss(cpu_env, OP_PTR0, in); 1898 } 1899} 1900 1901static inline void gen_VCVTtSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 1902 SSEFunc_i_ep ss2si, SSEFunc_l_ep ss2sq, 1903 SSEFunc_i_ep sd2si, SSEFunc_l_ep sd2sq) 1904{ 1905 TCGv_i32 out; 1906 1907#ifdef TARGET_X86_64 1908 MemOp ot = decode->op[0].ot; 1909 if (ot == MO_64) { 1910 if (s->prefix & PREFIX_REPNZ) { 1911 sd2sq(s->T0, cpu_env, OP_PTR2); 1912 } else { 1913 ss2sq(s->T0, cpu_env, OP_PTR2); 1914 } 1915 return; 1916 } 1917 1918 out = s->tmp2_i32; 1919#else 1920 out = s->T0; 1921#endif 1922 if (s->prefix & PREFIX_REPNZ) { 1923 sd2si(out, cpu_env, OP_PTR2); 1924 } else { 1925 ss2si(out, cpu_env, OP_PTR2); 1926 } 1927#ifdef TARGET_X86_64 1928 tcg_gen_extu_i32_tl(s->T0, out); 1929#endif 1930} 1931 1932#ifndef TARGET_X86_64 1933#define gen_helper_cvtss2sq NULL 1934#define gen_helper_cvtsd2sq NULL 1935#define gen_helper_cvttss2sq NULL 1936#define gen_helper_cvttsd2sq NULL 1937#endif 1938 1939static void gen_VCVTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1940{ 1941 gen_VCVTtSx2SI(s, env, decode, 1942 gen_helper_cvtss2si, gen_helper_cvtss2sq, 1943 gen_helper_cvtsd2si, gen_helper_cvtsd2sq); 1944} 1945 1946static void gen_VCVTTSx2SI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1947{ 1948 gen_VCVTtSx2SI(s, env, decode, 1949 gen_helper_cvttss2si, gen_helper_cvttss2sq, 1950 gen_helper_cvttsd2si, gen_helper_cvttsd2sq); 1951} 1952 1953static void gen_VEXTRACTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1954{ 1955 int mask = decode->immediate & 1; 1956 int src_ofs = vector_elem_offset(&decode->op[1], MO_128, mask); 1957 if (decode->op[0].has_ea) { 1958 /* VEX-only instruction, no alignment requirements. */ 1959 gen_sto_env_A0(s, src_ofs, false); 1960 } else { 1961 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, src_ofs, 16, 16); 1962 } 1963} 1964 1965static void gen_VEXTRACTPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1966{ 1967 gen_pextr(s, env, decode, MO_32); 1968} 1969 1970static void gen_vinsertps(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 1971{ 1972 int val = decode->immediate; 1973 int dest_word = (val >> 4) & 3; 1974 int new_mask = (val & 15) | (1 << dest_word); 1975 int vec_len = 16; 1976 1977 assert(!s->vex_l); 1978 1979 if (new_mask == 15) { 1980 /* All zeroes except possibly for the inserted element */ 1981 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 1982 } else if (decode->op[1].offset != decode->op[0].offset) { 1983 gen_store_sse(s, decode, decode->op[1].offset); 1984 } 1985 1986 if (new_mask != (val & 15)) { 1987 tcg_gen_st_i32(s->tmp2_i32, cpu_env, 1988 vector_elem_offset(&decode->op[0], MO_32, dest_word)); 1989 } 1990 1991 if (new_mask != 15) { 1992 TCGv_i32 zero = tcg_constant_i32(0); /* float32_zero */ 1993 int i; 1994 for (i = 0; i < 4; i++) { 1995 if ((val >> i) & 1) { 1996 tcg_gen_st_i32(zero, cpu_env, 1997 vector_elem_offset(&decode->op[0], MO_32, i)); 1998 } 1999 } 2000 } 2001} 2002 2003static void gen_VINSERTPS_r(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2004{ 2005 int val = decode->immediate; 2006 tcg_gen_ld_i32(s->tmp2_i32, cpu_env, 2007 vector_elem_offset(&decode->op[2], MO_32, (val >> 6) & 3)); 2008 gen_vinsertps(s, env, decode); 2009} 2010 2011static void gen_VINSERTPS_m(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2012{ 2013 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 2014 gen_vinsertps(s, env, decode); 2015} 2016 2017static void gen_VINSERTx128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2018{ 2019 int mask = decode->immediate & 1; 2020 tcg_gen_gvec_mov(MO_64, 2021 decode->op[0].offset + offsetof(YMMReg, YMM_X(mask)), 2022 decode->op[2].offset + offsetof(YMMReg, YMM_X(0)), 16, 16); 2023 tcg_gen_gvec_mov(MO_64, 2024 decode->op[0].offset + offsetof(YMMReg, YMM_X(!mask)), 2025 decode->op[1].offset + offsetof(YMMReg, YMM_X(!mask)), 16, 16); 2026} 2027 2028static inline void gen_maskmov(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode, 2029 SSEFunc_0_eppt xmm, SSEFunc_0_eppt ymm) 2030{ 2031 if (!s->vex_l) { 2032 xmm(cpu_env, OP_PTR2, OP_PTR1, s->A0); 2033 } else { 2034 ymm(cpu_env, OP_PTR2, OP_PTR1, s->A0); 2035 } 2036} 2037 2038static void gen_VMASKMOVPD_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2039{ 2040 gen_maskmov(s, env, decode, gen_helper_vpmaskmovq_st_xmm, gen_helper_vpmaskmovq_st_ymm); 2041} 2042 2043static void gen_VMASKMOVPS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2044{ 2045 gen_maskmov(s, env, decode, gen_helper_vpmaskmovd_st_xmm, gen_helper_vpmaskmovd_st_ymm); 2046} 2047 2048static void gen_VMOVHPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2049{ 2050 gen_ldq_env_A0(s, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 2051 if (decode->op[0].offset != decode->op[1].offset) { 2052 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 2053 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2054 } 2055} 2056 2057static void gen_VMOVHPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2058{ 2059 gen_stq_env_A0(s, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 2060} 2061 2062static void gen_VMOVHPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2063{ 2064 if (decode->op[0].offset != decode->op[2].offset) { 2065 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 2066 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 2067 } 2068 if (decode->op[0].offset != decode->op[1].offset) { 2069 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 2070 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2071 } 2072} 2073 2074static void gen_VMOVHLPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2075{ 2076 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(1))); 2077 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2078 if (decode->op[0].offset != decode->op[1].offset) { 2079 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(1))); 2080 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 2081 } 2082} 2083 2084static void gen_VMOVLHPS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2085{ 2086 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset); 2087 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(1))); 2088 if (decode->op[0].offset != decode->op[1].offset) { 2089 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[1].offset + offsetof(XMMReg, XMM_Q(0))); 2090 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2091 } 2092} 2093 2094/* 2095 * Note that MOVLPx supports 256-bit operation unlike MOVHLPx, MOVLHPx, MOXHPx. 2096 * Use a gvec move to move everything above the bottom 64 bits. 2097 */ 2098 2099static void gen_VMOVLPx(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2100{ 2101 int vec_len = vector_len(s, decode); 2102 2103 tcg_gen_ld_i64(s->tmp1_i64, cpu_env, decode->op[2].offset + offsetof(XMMReg, XMM_Q(0))); 2104 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 2105 tcg_gen_st_i64(s->tmp1_i64, cpu_env, decode->op[0].offset + offsetof(XMMReg, XMM_Q(0))); 2106} 2107 2108static void gen_VMOVLPx_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2109{ 2110 int vec_len = vector_len(s, decode); 2111 2112 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2113 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 2114 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 2115} 2116 2117static void gen_VMOVLPx_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2118{ 2119 tcg_gen_ld_i64(s->tmp1_i64, OP_PTR2, offsetof(ZMMReg, ZMM_Q(0))); 2120 tcg_gen_qemu_st_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2121} 2122 2123static void gen_VMOVSD_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2124{ 2125 TCGv_i64 zero = tcg_constant_i64(0); 2126 2127 tcg_gen_qemu_ld_i64(s->tmp1_i64, s->A0, s->mem_index, MO_LEUQ); 2128 tcg_gen_st_i64(zero, OP_PTR0, offsetof(ZMMReg, ZMM_Q(1))); 2129 tcg_gen_st_i64(s->tmp1_i64, OP_PTR0, offsetof(ZMMReg, ZMM_Q(0))); 2130} 2131 2132static void gen_VMOVSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2133{ 2134 int vec_len = vector_len(s, decode); 2135 2136 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 2137 tcg_gen_gvec_mov(MO_64, decode->op[0].offset, decode->op[1].offset, vec_len, vec_len); 2138 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 2139} 2140 2141static void gen_VMOVSS_ld(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2142{ 2143 int vec_len = vector_len(s, decode); 2144 2145 tcg_gen_qemu_ld_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 2146 tcg_gen_gvec_dup_imm(MO_64, decode->op[0].offset, vec_len, vec_len, 0); 2147 tcg_gen_st_i32(s->tmp2_i32, OP_PTR0, offsetof(ZMMReg, ZMM_L(0))); 2148} 2149 2150static void gen_VMOVSS_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2151{ 2152 tcg_gen_ld_i32(s->tmp2_i32, OP_PTR2, offsetof(ZMMReg, ZMM_L(0))); 2153 tcg_gen_qemu_st_i32(s->tmp2_i32, s->A0, s->mem_index, MO_LEUL); 2154} 2155 2156static void gen_VPMASKMOV_st(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2157{ 2158 if (s->vex_w) { 2159 gen_VMASKMOVPD_st(s, env, decode); 2160 } else { 2161 gen_VMASKMOVPS_st(s, env, decode); 2162 } 2163} 2164 2165static void gen_VPERMD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2166{ 2167 assert(s->vex_l); 2168 gen_helper_vpermd_ymm(OP_PTR0, OP_PTR1, OP_PTR2); 2169} 2170 2171static void gen_VPERM2x128(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2172{ 2173 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2174 assert(s->vex_l); 2175 gen_helper_vpermdq_ymm(OP_PTR0, OP_PTR1, OP_PTR2, imm); 2176} 2177 2178static void gen_VPHMINPOSUW(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2179{ 2180 assert(!s->vex_l); 2181 gen_helper_phminposuw_xmm(cpu_env, OP_PTR0, OP_PTR2); 2182} 2183 2184static void gen_VROUNDSD(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2185{ 2186 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2187 assert(!s->vex_l); 2188 gen_helper_roundsd_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2189} 2190 2191static void gen_VROUNDSS(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2192{ 2193 TCGv_i32 imm = tcg_constant8u_i32(decode->immediate); 2194 assert(!s->vex_l); 2195 gen_helper_roundss_xmm(cpu_env, OP_PTR0, OP_PTR1, OP_PTR2, imm); 2196} 2197 2198static void gen_VSHUF(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2199{ 2200 TCGv_i32 imm = tcg_constant_i32(decode->immediate); 2201 SSEFunc_0_pppi ps, pd, fn; 2202 ps = s->vex_l ? gen_helper_shufps_ymm : gen_helper_shufps_xmm; 2203 pd = s->vex_l ? gen_helper_shufpd_ymm : gen_helper_shufpd_xmm; 2204 fn = s->prefix & PREFIX_DATA ? pd : ps; 2205 fn(OP_PTR0, OP_PTR1, OP_PTR2, imm); 2206} 2207 2208static void gen_VUCOMI(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2209{ 2210 SSEFunc_0_epp fn; 2211 fn = s->prefix & PREFIX_DATA ? gen_helper_ucomisd : gen_helper_ucomiss; 2212 fn(cpu_env, OP_PTR1, OP_PTR2); 2213 set_cc_op(s, CC_OP_EFLAGS); 2214} 2215 2216static void gen_VZEROALL(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2217{ 2218 TCGv_ptr ptr = tcg_temp_new_ptr(); 2219 2220 tcg_gen_addi_ptr(ptr, cpu_env, offsetof(CPUX86State, xmm_t0)); 2221 gen_helper_memset(ptr, ptr, tcg_constant_i32(0), 2222 tcg_constant_ptr(CPU_NB_REGS * sizeof(ZMMReg))); 2223 tcg_temp_free_ptr(ptr); 2224} 2225 2226static void gen_VZEROUPPER(DisasContext *s, CPUX86State *env, X86DecodedInsn *decode) 2227{ 2228 int i; 2229 2230 for (i = 0; i < CPU_NB_REGS; i++) { 2231 int offset = offsetof(CPUX86State, xmm_regs[i].ZMM_X(1)); 2232 tcg_gen_gvec_dup_imm(MO_64, offset, 16, 16, 0); 2233 } 2234} 2235