1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 #include "qemu-common.h" 27 #include "cpu.h" 28 #include "exec/exec-all.h" 29 #include "tcg.h" 30 #include "tcg-op.h" 31 #include "trace-tcg.h" 32 #include "trace/mem.h" 33 34 /* Reduce the number of ifdefs below. This assumes that all uses of 35 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that 36 the compiler can eliminate. */ 37 #if TCG_TARGET_REG_BITS == 64 38 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); 39 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); 40 #define TCGV_LOW TCGV_LOW_link_error 41 #define TCGV_HIGH TCGV_HIGH_link_error 42 #endif 43 44 /* Note that this is optimized for sequential allocation during translate. 45 Up to and including filling in the forward link immediately. We'll do 46 proper termination of the end of the list after we finish translation. */ 47 48 static void tcg_emit_op(TCGContext *ctx, TCGOpcode opc, int args) 49 { 50 int oi = ctx->gen_next_op_idx; 51 int ni = oi + 1; 52 int pi = oi - 1; 53 54 tcg_debug_assert(oi < OPC_BUF_SIZE); 55 ctx->gen_op_buf[0].prev = oi; 56 ctx->gen_next_op_idx = ni; 57 58 ctx->gen_op_buf[oi] = (TCGOp){ 59 .opc = opc, 60 .args = args, 61 .prev = pi, 62 .next = ni 63 }; 64 } 65 66 void tcg_gen_op1(TCGContext *ctx, TCGOpcode opc, TCGArg a1) 67 { 68 int pi = ctx->gen_next_parm_idx; 69 70 tcg_debug_assert(pi + 1 <= OPPARAM_BUF_SIZE); 71 ctx->gen_next_parm_idx = pi + 1; 72 ctx->gen_opparam_buf[pi] = a1; 73 74 tcg_emit_op(ctx, opc, pi); 75 } 76 77 void tcg_gen_op2(TCGContext *ctx, TCGOpcode opc, TCGArg a1, TCGArg a2) 78 { 79 int pi = ctx->gen_next_parm_idx; 80 81 tcg_debug_assert(pi + 2 <= OPPARAM_BUF_SIZE); 82 ctx->gen_next_parm_idx = pi + 2; 83 ctx->gen_opparam_buf[pi + 0] = a1; 84 ctx->gen_opparam_buf[pi + 1] = a2; 85 86 tcg_emit_op(ctx, opc, pi); 87 } 88 89 void tcg_gen_op3(TCGContext *ctx, TCGOpcode opc, TCGArg a1, 90 TCGArg a2, TCGArg a3) 91 { 92 int pi = ctx->gen_next_parm_idx; 93 94 tcg_debug_assert(pi + 3 <= OPPARAM_BUF_SIZE); 95 ctx->gen_next_parm_idx = pi + 3; 96 ctx->gen_opparam_buf[pi + 0] = a1; 97 ctx->gen_opparam_buf[pi + 1] = a2; 98 ctx->gen_opparam_buf[pi + 2] = a3; 99 100 tcg_emit_op(ctx, opc, pi); 101 } 102 103 void tcg_gen_op4(TCGContext *ctx, TCGOpcode opc, TCGArg a1, 104 TCGArg a2, TCGArg a3, TCGArg a4) 105 { 106 int pi = ctx->gen_next_parm_idx; 107 108 tcg_debug_assert(pi + 4 <= OPPARAM_BUF_SIZE); 109 ctx->gen_next_parm_idx = pi + 4; 110 ctx->gen_opparam_buf[pi + 0] = a1; 111 ctx->gen_opparam_buf[pi + 1] = a2; 112 ctx->gen_opparam_buf[pi + 2] = a3; 113 ctx->gen_opparam_buf[pi + 3] = a4; 114 115 tcg_emit_op(ctx, opc, pi); 116 } 117 118 void tcg_gen_op5(TCGContext *ctx, TCGOpcode opc, TCGArg a1, 119 TCGArg a2, TCGArg a3, TCGArg a4, TCGArg a5) 120 { 121 int pi = ctx->gen_next_parm_idx; 122 123 tcg_debug_assert(pi + 5 <= OPPARAM_BUF_SIZE); 124 ctx->gen_next_parm_idx = pi + 5; 125 ctx->gen_opparam_buf[pi + 0] = a1; 126 ctx->gen_opparam_buf[pi + 1] = a2; 127 ctx->gen_opparam_buf[pi + 2] = a3; 128 ctx->gen_opparam_buf[pi + 3] = a4; 129 ctx->gen_opparam_buf[pi + 4] = a5; 130 131 tcg_emit_op(ctx, opc, pi); 132 } 133 134 void tcg_gen_op6(TCGContext *ctx, TCGOpcode opc, TCGArg a1, TCGArg a2, 135 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6) 136 { 137 int pi = ctx->gen_next_parm_idx; 138 139 tcg_debug_assert(pi + 6 <= OPPARAM_BUF_SIZE); 140 ctx->gen_next_parm_idx = pi + 6; 141 ctx->gen_opparam_buf[pi + 0] = a1; 142 ctx->gen_opparam_buf[pi + 1] = a2; 143 ctx->gen_opparam_buf[pi + 2] = a3; 144 ctx->gen_opparam_buf[pi + 3] = a4; 145 ctx->gen_opparam_buf[pi + 4] = a5; 146 ctx->gen_opparam_buf[pi + 5] = a6; 147 148 tcg_emit_op(ctx, opc, pi); 149 } 150 151 void tcg_gen_mb(TCGBar mb_type) 152 { 153 if (parallel_cpus) { 154 tcg_gen_op1(&tcg_ctx, INDEX_op_mb, mb_type); 155 } 156 } 157 158 /* 32 bit ops */ 159 160 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 161 { 162 /* some cases can be optimized here */ 163 if (arg2 == 0) { 164 tcg_gen_mov_i32(ret, arg1); 165 } else { 166 TCGv_i32 t0 = tcg_const_i32(arg2); 167 tcg_gen_add_i32(ret, arg1, t0); 168 tcg_temp_free_i32(t0); 169 } 170 } 171 172 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2) 173 { 174 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) { 175 /* Don't recurse with tcg_gen_neg_i32. */ 176 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2); 177 } else { 178 TCGv_i32 t0 = tcg_const_i32(arg1); 179 tcg_gen_sub_i32(ret, t0, arg2); 180 tcg_temp_free_i32(t0); 181 } 182 } 183 184 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 185 { 186 /* some cases can be optimized here */ 187 if (arg2 == 0) { 188 tcg_gen_mov_i32(ret, arg1); 189 } else { 190 TCGv_i32 t0 = tcg_const_i32(arg2); 191 tcg_gen_sub_i32(ret, arg1, t0); 192 tcg_temp_free_i32(t0); 193 } 194 } 195 196 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2) 197 { 198 TCGv_i32 t0; 199 /* Some cases can be optimized here. */ 200 switch (arg2) { 201 case 0: 202 tcg_gen_movi_i32(ret, 0); 203 return; 204 case 0xffffffffu: 205 tcg_gen_mov_i32(ret, arg1); 206 return; 207 case 0xffu: 208 /* Don't recurse with tcg_gen_ext8u_i32. */ 209 if (TCG_TARGET_HAS_ext8u_i32) { 210 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1); 211 return; 212 } 213 break; 214 case 0xffffu: 215 if (TCG_TARGET_HAS_ext16u_i32) { 216 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1); 217 return; 218 } 219 break; 220 } 221 t0 = tcg_const_i32(arg2); 222 tcg_gen_and_i32(ret, arg1, t0); 223 tcg_temp_free_i32(t0); 224 } 225 226 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 227 { 228 /* Some cases can be optimized here. */ 229 if (arg2 == -1) { 230 tcg_gen_movi_i32(ret, -1); 231 } else if (arg2 == 0) { 232 tcg_gen_mov_i32(ret, arg1); 233 } else { 234 TCGv_i32 t0 = tcg_const_i32(arg2); 235 tcg_gen_or_i32(ret, arg1, t0); 236 tcg_temp_free_i32(t0); 237 } 238 } 239 240 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 241 { 242 /* Some cases can be optimized here. */ 243 if (arg2 == 0) { 244 tcg_gen_mov_i32(ret, arg1); 245 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) { 246 /* Don't recurse with tcg_gen_not_i32. */ 247 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1); 248 } else { 249 TCGv_i32 t0 = tcg_const_i32(arg2); 250 tcg_gen_xor_i32(ret, arg1, t0); 251 tcg_temp_free_i32(t0); 252 } 253 } 254 255 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 256 { 257 tcg_debug_assert(arg2 < 32); 258 if (arg2 == 0) { 259 tcg_gen_mov_i32(ret, arg1); 260 } else { 261 TCGv_i32 t0 = tcg_const_i32(arg2); 262 tcg_gen_shl_i32(ret, arg1, t0); 263 tcg_temp_free_i32(t0); 264 } 265 } 266 267 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 268 { 269 tcg_debug_assert(arg2 < 32); 270 if (arg2 == 0) { 271 tcg_gen_mov_i32(ret, arg1); 272 } else { 273 TCGv_i32 t0 = tcg_const_i32(arg2); 274 tcg_gen_shr_i32(ret, arg1, t0); 275 tcg_temp_free_i32(t0); 276 } 277 } 278 279 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 280 { 281 tcg_debug_assert(arg2 < 32); 282 if (arg2 == 0) { 283 tcg_gen_mov_i32(ret, arg1); 284 } else { 285 TCGv_i32 t0 = tcg_const_i32(arg2); 286 tcg_gen_sar_i32(ret, arg1, t0); 287 tcg_temp_free_i32(t0); 288 } 289 } 290 291 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) 292 { 293 if (cond == TCG_COND_ALWAYS) { 294 tcg_gen_br(l); 295 } else if (cond != TCG_COND_NEVER) { 296 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); 297 } 298 } 299 300 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l) 301 { 302 if (cond == TCG_COND_ALWAYS) { 303 tcg_gen_br(l); 304 } else if (cond != TCG_COND_NEVER) { 305 TCGv_i32 t0 = tcg_const_i32(arg2); 306 tcg_gen_brcond_i32(cond, arg1, t0, l); 307 tcg_temp_free_i32(t0); 308 } 309 } 310 311 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret, 312 TCGv_i32 arg1, TCGv_i32 arg2) 313 { 314 if (cond == TCG_COND_ALWAYS) { 315 tcg_gen_movi_i32(ret, 1); 316 } else if (cond == TCG_COND_NEVER) { 317 tcg_gen_movi_i32(ret, 0); 318 } else { 319 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond); 320 } 321 } 322 323 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret, 324 TCGv_i32 arg1, int32_t arg2) 325 { 326 TCGv_i32 t0 = tcg_const_i32(arg2); 327 tcg_gen_setcond_i32(cond, ret, arg1, t0); 328 tcg_temp_free_i32(t0); 329 } 330 331 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2) 332 { 333 TCGv_i32 t0 = tcg_const_i32(arg2); 334 tcg_gen_mul_i32(ret, arg1, t0); 335 tcg_temp_free_i32(t0); 336 } 337 338 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 339 { 340 if (TCG_TARGET_HAS_div_i32) { 341 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2); 342 } else if (TCG_TARGET_HAS_div2_i32) { 343 TCGv_i32 t0 = tcg_temp_new_i32(); 344 tcg_gen_sari_i32(t0, arg1, 31); 345 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2); 346 tcg_temp_free_i32(t0); 347 } else { 348 gen_helper_div_i32(ret, arg1, arg2); 349 } 350 } 351 352 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 353 { 354 if (TCG_TARGET_HAS_rem_i32) { 355 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2); 356 } else if (TCG_TARGET_HAS_div_i32) { 357 TCGv_i32 t0 = tcg_temp_new_i32(); 358 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2); 359 tcg_gen_mul_i32(t0, t0, arg2); 360 tcg_gen_sub_i32(ret, arg1, t0); 361 tcg_temp_free_i32(t0); 362 } else if (TCG_TARGET_HAS_div2_i32) { 363 TCGv_i32 t0 = tcg_temp_new_i32(); 364 tcg_gen_sari_i32(t0, arg1, 31); 365 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2); 366 tcg_temp_free_i32(t0); 367 } else { 368 gen_helper_rem_i32(ret, arg1, arg2); 369 } 370 } 371 372 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 373 { 374 if (TCG_TARGET_HAS_div_i32) { 375 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2); 376 } else if (TCG_TARGET_HAS_div2_i32) { 377 TCGv_i32 t0 = tcg_temp_new_i32(); 378 tcg_gen_movi_i32(t0, 0); 379 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2); 380 tcg_temp_free_i32(t0); 381 } else { 382 gen_helper_divu_i32(ret, arg1, arg2); 383 } 384 } 385 386 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 387 { 388 if (TCG_TARGET_HAS_rem_i32) { 389 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2); 390 } else if (TCG_TARGET_HAS_div_i32) { 391 TCGv_i32 t0 = tcg_temp_new_i32(); 392 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2); 393 tcg_gen_mul_i32(t0, t0, arg2); 394 tcg_gen_sub_i32(ret, arg1, t0); 395 tcg_temp_free_i32(t0); 396 } else if (TCG_TARGET_HAS_div2_i32) { 397 TCGv_i32 t0 = tcg_temp_new_i32(); 398 tcg_gen_movi_i32(t0, 0); 399 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2); 400 tcg_temp_free_i32(t0); 401 } else { 402 gen_helper_remu_i32(ret, arg1, arg2); 403 } 404 } 405 406 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 407 { 408 if (TCG_TARGET_HAS_andc_i32) { 409 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2); 410 } else { 411 TCGv_i32 t0 = tcg_temp_new_i32(); 412 tcg_gen_not_i32(t0, arg2); 413 tcg_gen_and_i32(ret, arg1, t0); 414 tcg_temp_free_i32(t0); 415 } 416 } 417 418 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 419 { 420 if (TCG_TARGET_HAS_eqv_i32) { 421 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2); 422 } else { 423 tcg_gen_xor_i32(ret, arg1, arg2); 424 tcg_gen_not_i32(ret, ret); 425 } 426 } 427 428 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 429 { 430 if (TCG_TARGET_HAS_nand_i32) { 431 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2); 432 } else { 433 tcg_gen_and_i32(ret, arg1, arg2); 434 tcg_gen_not_i32(ret, ret); 435 } 436 } 437 438 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 439 { 440 if (TCG_TARGET_HAS_nor_i32) { 441 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2); 442 } else { 443 tcg_gen_or_i32(ret, arg1, arg2); 444 tcg_gen_not_i32(ret, ret); 445 } 446 } 447 448 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 449 { 450 if (TCG_TARGET_HAS_orc_i32) { 451 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2); 452 } else { 453 TCGv_i32 t0 = tcg_temp_new_i32(); 454 tcg_gen_not_i32(t0, arg2); 455 tcg_gen_or_i32(ret, arg1, t0); 456 tcg_temp_free_i32(t0); 457 } 458 } 459 460 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 461 { 462 if (TCG_TARGET_HAS_rot_i32) { 463 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2); 464 } else { 465 TCGv_i32 t0, t1; 466 467 t0 = tcg_temp_new_i32(); 468 t1 = tcg_temp_new_i32(); 469 tcg_gen_shl_i32(t0, arg1, arg2); 470 tcg_gen_subfi_i32(t1, 32, arg2); 471 tcg_gen_shr_i32(t1, arg1, t1); 472 tcg_gen_or_i32(ret, t0, t1); 473 tcg_temp_free_i32(t0); 474 tcg_temp_free_i32(t1); 475 } 476 } 477 478 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 479 { 480 tcg_debug_assert(arg2 < 32); 481 /* some cases can be optimized here */ 482 if (arg2 == 0) { 483 tcg_gen_mov_i32(ret, arg1); 484 } else if (TCG_TARGET_HAS_rot_i32) { 485 TCGv_i32 t0 = tcg_const_i32(arg2); 486 tcg_gen_rotl_i32(ret, arg1, t0); 487 tcg_temp_free_i32(t0); 488 } else { 489 TCGv_i32 t0, t1; 490 t0 = tcg_temp_new_i32(); 491 t1 = tcg_temp_new_i32(); 492 tcg_gen_shli_i32(t0, arg1, arg2); 493 tcg_gen_shri_i32(t1, arg1, 32 - arg2); 494 tcg_gen_or_i32(ret, t0, t1); 495 tcg_temp_free_i32(t0); 496 tcg_temp_free_i32(t1); 497 } 498 } 499 500 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2) 501 { 502 if (TCG_TARGET_HAS_rot_i32) { 503 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2); 504 } else { 505 TCGv_i32 t0, t1; 506 507 t0 = tcg_temp_new_i32(); 508 t1 = tcg_temp_new_i32(); 509 tcg_gen_shr_i32(t0, arg1, arg2); 510 tcg_gen_subfi_i32(t1, 32, arg2); 511 tcg_gen_shl_i32(t1, arg1, t1); 512 tcg_gen_or_i32(ret, t0, t1); 513 tcg_temp_free_i32(t0); 514 tcg_temp_free_i32(t1); 515 } 516 } 517 518 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2) 519 { 520 tcg_debug_assert(arg2 < 32); 521 /* some cases can be optimized here */ 522 if (arg2 == 0) { 523 tcg_gen_mov_i32(ret, arg1); 524 } else { 525 tcg_gen_rotli_i32(ret, arg1, 32 - arg2); 526 } 527 } 528 529 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2, 530 unsigned int ofs, unsigned int len) 531 { 532 uint32_t mask; 533 TCGv_i32 t1; 534 535 tcg_debug_assert(ofs < 32); 536 tcg_debug_assert(len <= 32); 537 tcg_debug_assert(ofs + len <= 32); 538 539 if (ofs == 0 && len == 32) { 540 tcg_gen_mov_i32(ret, arg2); 541 return; 542 } 543 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) { 544 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len); 545 return; 546 } 547 548 mask = (1u << len) - 1; 549 t1 = tcg_temp_new_i32(); 550 551 if (ofs + len < 32) { 552 tcg_gen_andi_i32(t1, arg2, mask); 553 tcg_gen_shli_i32(t1, t1, ofs); 554 } else { 555 tcg_gen_shli_i32(t1, arg2, ofs); 556 } 557 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs)); 558 tcg_gen_or_i32(ret, ret, t1); 559 560 tcg_temp_free_i32(t1); 561 } 562 563 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1, 564 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2) 565 { 566 if (cond == TCG_COND_ALWAYS) { 567 tcg_gen_mov_i32(ret, v1); 568 } else if (cond == TCG_COND_NEVER) { 569 tcg_gen_mov_i32(ret, v2); 570 } else if (TCG_TARGET_HAS_movcond_i32) { 571 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond); 572 } else { 573 TCGv_i32 t0 = tcg_temp_new_i32(); 574 TCGv_i32 t1 = tcg_temp_new_i32(); 575 tcg_gen_setcond_i32(cond, t0, c1, c2); 576 tcg_gen_neg_i32(t0, t0); 577 tcg_gen_and_i32(t1, v1, t0); 578 tcg_gen_andc_i32(ret, v2, t0); 579 tcg_gen_or_i32(ret, ret, t1); 580 tcg_temp_free_i32(t0); 581 tcg_temp_free_i32(t1); 582 } 583 } 584 585 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 586 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 587 { 588 if (TCG_TARGET_HAS_add2_i32) { 589 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh); 590 } else { 591 TCGv_i64 t0 = tcg_temp_new_i64(); 592 TCGv_i64 t1 = tcg_temp_new_i64(); 593 tcg_gen_concat_i32_i64(t0, al, ah); 594 tcg_gen_concat_i32_i64(t1, bl, bh); 595 tcg_gen_add_i64(t0, t0, t1); 596 tcg_gen_extr_i64_i32(rl, rh, t0); 597 tcg_temp_free_i64(t0); 598 tcg_temp_free_i64(t1); 599 } 600 } 601 602 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al, 603 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh) 604 { 605 if (TCG_TARGET_HAS_sub2_i32) { 606 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh); 607 } else { 608 TCGv_i64 t0 = tcg_temp_new_i64(); 609 TCGv_i64 t1 = tcg_temp_new_i64(); 610 tcg_gen_concat_i32_i64(t0, al, ah); 611 tcg_gen_concat_i32_i64(t1, bl, bh); 612 tcg_gen_sub_i64(t0, t0, t1); 613 tcg_gen_extr_i64_i32(rl, rh, t0); 614 tcg_temp_free_i64(t0); 615 tcg_temp_free_i64(t1); 616 } 617 } 618 619 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 620 { 621 if (TCG_TARGET_HAS_mulu2_i32) { 622 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2); 623 } else if (TCG_TARGET_HAS_muluh_i32) { 624 TCGv_i32 t = tcg_temp_new_i32(); 625 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 626 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2); 627 tcg_gen_mov_i32(rl, t); 628 tcg_temp_free_i32(t); 629 } else { 630 TCGv_i64 t0 = tcg_temp_new_i64(); 631 TCGv_i64 t1 = tcg_temp_new_i64(); 632 tcg_gen_extu_i32_i64(t0, arg1); 633 tcg_gen_extu_i32_i64(t1, arg2); 634 tcg_gen_mul_i64(t0, t0, t1); 635 tcg_gen_extr_i64_i32(rl, rh, t0); 636 tcg_temp_free_i64(t0); 637 tcg_temp_free_i64(t1); 638 } 639 } 640 641 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2) 642 { 643 if (TCG_TARGET_HAS_muls2_i32) { 644 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2); 645 } else if (TCG_TARGET_HAS_mulsh_i32) { 646 TCGv_i32 t = tcg_temp_new_i32(); 647 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2); 648 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2); 649 tcg_gen_mov_i32(rl, t); 650 tcg_temp_free_i32(t); 651 } else if (TCG_TARGET_REG_BITS == 32) { 652 TCGv_i32 t0 = tcg_temp_new_i32(); 653 TCGv_i32 t1 = tcg_temp_new_i32(); 654 TCGv_i32 t2 = tcg_temp_new_i32(); 655 TCGv_i32 t3 = tcg_temp_new_i32(); 656 tcg_gen_mulu2_i32(t0, t1, arg1, arg2); 657 /* Adjust for negative inputs. */ 658 tcg_gen_sari_i32(t2, arg1, 31); 659 tcg_gen_sari_i32(t3, arg2, 31); 660 tcg_gen_and_i32(t2, t2, arg2); 661 tcg_gen_and_i32(t3, t3, arg1); 662 tcg_gen_sub_i32(rh, t1, t2); 663 tcg_gen_sub_i32(rh, rh, t3); 664 tcg_gen_mov_i32(rl, t0); 665 tcg_temp_free_i32(t0); 666 tcg_temp_free_i32(t1); 667 tcg_temp_free_i32(t2); 668 tcg_temp_free_i32(t3); 669 } else { 670 TCGv_i64 t0 = tcg_temp_new_i64(); 671 TCGv_i64 t1 = tcg_temp_new_i64(); 672 tcg_gen_ext_i32_i64(t0, arg1); 673 tcg_gen_ext_i32_i64(t1, arg2); 674 tcg_gen_mul_i64(t0, t0, t1); 675 tcg_gen_extr_i64_i32(rl, rh, t0); 676 tcg_temp_free_i64(t0); 677 tcg_temp_free_i64(t1); 678 } 679 } 680 681 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg) 682 { 683 if (TCG_TARGET_HAS_ext8s_i32) { 684 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg); 685 } else { 686 tcg_gen_shli_i32(ret, arg, 24); 687 tcg_gen_sari_i32(ret, ret, 24); 688 } 689 } 690 691 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg) 692 { 693 if (TCG_TARGET_HAS_ext16s_i32) { 694 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg); 695 } else { 696 tcg_gen_shli_i32(ret, arg, 16); 697 tcg_gen_sari_i32(ret, ret, 16); 698 } 699 } 700 701 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg) 702 { 703 if (TCG_TARGET_HAS_ext8u_i32) { 704 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg); 705 } else { 706 tcg_gen_andi_i32(ret, arg, 0xffu); 707 } 708 } 709 710 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg) 711 { 712 if (TCG_TARGET_HAS_ext16u_i32) { 713 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg); 714 } else { 715 tcg_gen_andi_i32(ret, arg, 0xffffu); 716 } 717 } 718 719 /* Note: we assume the two high bytes are set to zero */ 720 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg) 721 { 722 if (TCG_TARGET_HAS_bswap16_i32) { 723 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg); 724 } else { 725 TCGv_i32 t0 = tcg_temp_new_i32(); 726 727 tcg_gen_ext8u_i32(t0, arg); 728 tcg_gen_shli_i32(t0, t0, 8); 729 tcg_gen_shri_i32(ret, arg, 8); 730 tcg_gen_or_i32(ret, ret, t0); 731 tcg_temp_free_i32(t0); 732 } 733 } 734 735 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg) 736 { 737 if (TCG_TARGET_HAS_bswap32_i32) { 738 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg); 739 } else { 740 TCGv_i32 t0, t1; 741 t0 = tcg_temp_new_i32(); 742 t1 = tcg_temp_new_i32(); 743 744 tcg_gen_shli_i32(t0, arg, 24); 745 746 tcg_gen_andi_i32(t1, arg, 0x0000ff00); 747 tcg_gen_shli_i32(t1, t1, 8); 748 tcg_gen_or_i32(t0, t0, t1); 749 750 tcg_gen_shri_i32(t1, arg, 8); 751 tcg_gen_andi_i32(t1, t1, 0x0000ff00); 752 tcg_gen_or_i32(t0, t0, t1); 753 754 tcg_gen_shri_i32(t1, arg, 24); 755 tcg_gen_or_i32(ret, t0, t1); 756 tcg_temp_free_i32(t0); 757 tcg_temp_free_i32(t1); 758 } 759 } 760 761 /* 64-bit ops */ 762 763 #if TCG_TARGET_REG_BITS == 32 764 /* These are all inline for TCG_TARGET_REG_BITS == 64. */ 765 766 void tcg_gen_discard_i64(TCGv_i64 arg) 767 { 768 tcg_gen_discard_i32(TCGV_LOW(arg)); 769 tcg_gen_discard_i32(TCGV_HIGH(arg)); 770 } 771 772 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg) 773 { 774 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 775 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 776 } 777 778 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg) 779 { 780 tcg_gen_movi_i32(TCGV_LOW(ret), arg); 781 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32); 782 } 783 784 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 785 { 786 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset); 787 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 788 } 789 790 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 791 { 792 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset); 793 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), 31); 794 } 795 796 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 797 { 798 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset); 799 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 800 } 801 802 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 803 { 804 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset); 805 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 806 } 807 808 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 809 { 810 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 811 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 812 } 813 814 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 815 { 816 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 817 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 818 } 819 820 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) 821 { 822 /* Since arg2 and ret have different types, 823 they cannot be the same temporary */ 824 #ifdef HOST_WORDS_BIGENDIAN 825 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset); 826 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4); 827 #else 828 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset); 829 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4); 830 #endif 831 } 832 833 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) 834 { 835 #ifdef HOST_WORDS_BIGENDIAN 836 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset); 837 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4); 838 #else 839 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); 840 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4); 841 #endif 842 } 843 844 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 845 { 846 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 847 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 848 } 849 850 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 851 { 852 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 853 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 854 } 855 856 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 857 { 858 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 859 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 860 } 861 862 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 863 { 864 gen_helper_shl_i64(ret, arg1, arg2); 865 } 866 867 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 868 { 869 gen_helper_shr_i64(ret, arg1, arg2); 870 } 871 872 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 873 { 874 gen_helper_sar_i64(ret, arg1, arg2); 875 } 876 877 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 878 { 879 TCGv_i64 t0; 880 TCGv_i32 t1; 881 882 t0 = tcg_temp_new_i64(); 883 t1 = tcg_temp_new_i32(); 884 885 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0), 886 TCGV_LOW(arg1), TCGV_LOW(arg2)); 887 888 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2)); 889 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 890 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2)); 891 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1); 892 893 tcg_gen_mov_i64(ret, t0); 894 tcg_temp_free_i64(t0); 895 tcg_temp_free_i32(t1); 896 } 897 #endif /* TCG_TARGET_REG_SIZE == 32 */ 898 899 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 900 { 901 /* some cases can be optimized here */ 902 if (arg2 == 0) { 903 tcg_gen_mov_i64(ret, arg1); 904 } else { 905 TCGv_i64 t0 = tcg_const_i64(arg2); 906 tcg_gen_add_i64(ret, arg1, t0); 907 tcg_temp_free_i64(t0); 908 } 909 } 910 911 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2) 912 { 913 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) { 914 /* Don't recurse with tcg_gen_neg_i64. */ 915 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2); 916 } else { 917 TCGv_i64 t0 = tcg_const_i64(arg1); 918 tcg_gen_sub_i64(ret, t0, arg2); 919 tcg_temp_free_i64(t0); 920 } 921 } 922 923 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 924 { 925 /* some cases can be optimized here */ 926 if (arg2 == 0) { 927 tcg_gen_mov_i64(ret, arg1); 928 } else { 929 TCGv_i64 t0 = tcg_const_i64(arg2); 930 tcg_gen_sub_i64(ret, arg1, t0); 931 tcg_temp_free_i64(t0); 932 } 933 } 934 935 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2) 936 { 937 TCGv_i64 t0; 938 939 if (TCG_TARGET_REG_BITS == 32) { 940 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 941 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 942 return; 943 } 944 945 /* Some cases can be optimized here. */ 946 switch (arg2) { 947 case 0: 948 tcg_gen_movi_i64(ret, 0); 949 return; 950 case 0xffffffffffffffffull: 951 tcg_gen_mov_i64(ret, arg1); 952 return; 953 case 0xffull: 954 /* Don't recurse with tcg_gen_ext8u_i64. */ 955 if (TCG_TARGET_HAS_ext8u_i64) { 956 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1); 957 return; 958 } 959 break; 960 case 0xffffu: 961 if (TCG_TARGET_HAS_ext16u_i64) { 962 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1); 963 return; 964 } 965 break; 966 case 0xffffffffull: 967 if (TCG_TARGET_HAS_ext32u_i64) { 968 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1); 969 return; 970 } 971 break; 972 } 973 t0 = tcg_const_i64(arg2); 974 tcg_gen_and_i64(ret, arg1, t0); 975 tcg_temp_free_i64(t0); 976 } 977 978 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 979 { 980 if (TCG_TARGET_REG_BITS == 32) { 981 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 982 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 983 return; 984 } 985 /* Some cases can be optimized here. */ 986 if (arg2 == -1) { 987 tcg_gen_movi_i64(ret, -1); 988 } else if (arg2 == 0) { 989 tcg_gen_mov_i64(ret, arg1); 990 } else { 991 TCGv_i64 t0 = tcg_const_i64(arg2); 992 tcg_gen_or_i64(ret, arg1, t0); 993 tcg_temp_free_i64(t0); 994 } 995 } 996 997 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 998 { 999 if (TCG_TARGET_REG_BITS == 32) { 1000 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2); 1001 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32); 1002 return; 1003 } 1004 /* Some cases can be optimized here. */ 1005 if (arg2 == 0) { 1006 tcg_gen_mov_i64(ret, arg1); 1007 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) { 1008 /* Don't recurse with tcg_gen_not_i64. */ 1009 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1); 1010 } else { 1011 TCGv_i64 t0 = tcg_const_i64(arg2); 1012 tcg_gen_xor_i64(ret, arg1, t0); 1013 tcg_temp_free_i64(t0); 1014 } 1015 } 1016 1017 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 1018 unsigned c, bool right, bool arith) 1019 { 1020 tcg_debug_assert(c < 64); 1021 if (c == 0) { 1022 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1023 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1024 } else if (c >= 32) { 1025 c -= 32; 1026 if (right) { 1027 if (arith) { 1028 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1029 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 1030 } else { 1031 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 1032 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1033 } 1034 } else { 1035 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 1036 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 1037 } 1038 } else { 1039 TCGv_i32 t0, t1; 1040 1041 t0 = tcg_temp_new_i32(); 1042 t1 = tcg_temp_new_i32(); 1043 if (right) { 1044 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 1045 if (arith) { 1046 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 1047 } else { 1048 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 1049 } 1050 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 1051 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 1052 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 1053 } else { 1054 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 1055 /* Note: ret can be the same as arg1, so we use t1 */ 1056 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 1057 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 1058 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 1059 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1060 } 1061 tcg_temp_free_i32(t0); 1062 tcg_temp_free_i32(t1); 1063 } 1064 } 1065 1066 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1067 { 1068 tcg_debug_assert(arg2 < 64); 1069 if (TCG_TARGET_REG_BITS == 32) { 1070 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0); 1071 } else if (arg2 == 0) { 1072 tcg_gen_mov_i64(ret, arg1); 1073 } else { 1074 TCGv_i64 t0 = tcg_const_i64(arg2); 1075 tcg_gen_shl_i64(ret, arg1, t0); 1076 tcg_temp_free_i64(t0); 1077 } 1078 } 1079 1080 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1081 { 1082 tcg_debug_assert(arg2 < 64); 1083 if (TCG_TARGET_REG_BITS == 32) { 1084 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0); 1085 } else if (arg2 == 0) { 1086 tcg_gen_mov_i64(ret, arg1); 1087 } else { 1088 TCGv_i64 t0 = tcg_const_i64(arg2); 1089 tcg_gen_shr_i64(ret, arg1, t0); 1090 tcg_temp_free_i64(t0); 1091 } 1092 } 1093 1094 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1095 { 1096 tcg_debug_assert(arg2 < 64); 1097 if (TCG_TARGET_REG_BITS == 32) { 1098 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1); 1099 } else if (arg2 == 0) { 1100 tcg_gen_mov_i64(ret, arg1); 1101 } else { 1102 TCGv_i64 t0 = tcg_const_i64(arg2); 1103 tcg_gen_sar_i64(ret, arg1, t0); 1104 tcg_temp_free_i64(t0); 1105 } 1106 } 1107 1108 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) 1109 { 1110 if (cond == TCG_COND_ALWAYS) { 1111 tcg_gen_br(l); 1112 } else if (cond != TCG_COND_NEVER) { 1113 if (TCG_TARGET_REG_BITS == 32) { 1114 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), 1115 TCGV_HIGH(arg1), TCGV_LOW(arg2), 1116 TCGV_HIGH(arg2), cond, label_arg(l)); 1117 } else { 1118 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, 1119 label_arg(l)); 1120 } 1121 } 1122 } 1123 1124 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) 1125 { 1126 if (cond == TCG_COND_ALWAYS) { 1127 tcg_gen_br(l); 1128 } else if (cond != TCG_COND_NEVER) { 1129 TCGv_i64 t0 = tcg_const_i64(arg2); 1130 tcg_gen_brcond_i64(cond, arg1, t0, l); 1131 tcg_temp_free_i64(t0); 1132 } 1133 } 1134 1135 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret, 1136 TCGv_i64 arg1, TCGv_i64 arg2) 1137 { 1138 if (cond == TCG_COND_ALWAYS) { 1139 tcg_gen_movi_i64(ret, 1); 1140 } else if (cond == TCG_COND_NEVER) { 1141 tcg_gen_movi_i64(ret, 0); 1142 } else { 1143 if (TCG_TARGET_REG_BITS == 32) { 1144 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret), 1145 TCGV_LOW(arg1), TCGV_HIGH(arg1), 1146 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond); 1147 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1148 } else { 1149 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond); 1150 } 1151 } 1152 } 1153 1154 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret, 1155 TCGv_i64 arg1, int64_t arg2) 1156 { 1157 TCGv_i64 t0 = tcg_const_i64(arg2); 1158 tcg_gen_setcond_i64(cond, ret, arg1, t0); 1159 tcg_temp_free_i64(t0); 1160 } 1161 1162 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2) 1163 { 1164 TCGv_i64 t0 = tcg_const_i64(arg2); 1165 tcg_gen_mul_i64(ret, arg1, t0); 1166 tcg_temp_free_i64(t0); 1167 } 1168 1169 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1170 { 1171 if (TCG_TARGET_HAS_div_i64) { 1172 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2); 1173 } else if (TCG_TARGET_HAS_div2_i64) { 1174 TCGv_i64 t0 = tcg_temp_new_i64(); 1175 tcg_gen_sari_i64(t0, arg1, 63); 1176 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2); 1177 tcg_temp_free_i64(t0); 1178 } else { 1179 gen_helper_div_i64(ret, arg1, arg2); 1180 } 1181 } 1182 1183 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1184 { 1185 if (TCG_TARGET_HAS_rem_i64) { 1186 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2); 1187 } else if (TCG_TARGET_HAS_div_i64) { 1188 TCGv_i64 t0 = tcg_temp_new_i64(); 1189 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2); 1190 tcg_gen_mul_i64(t0, t0, arg2); 1191 tcg_gen_sub_i64(ret, arg1, t0); 1192 tcg_temp_free_i64(t0); 1193 } else if (TCG_TARGET_HAS_div2_i64) { 1194 TCGv_i64 t0 = tcg_temp_new_i64(); 1195 tcg_gen_sari_i64(t0, arg1, 63); 1196 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2); 1197 tcg_temp_free_i64(t0); 1198 } else { 1199 gen_helper_rem_i64(ret, arg1, arg2); 1200 } 1201 } 1202 1203 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1204 { 1205 if (TCG_TARGET_HAS_div_i64) { 1206 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2); 1207 } else if (TCG_TARGET_HAS_div2_i64) { 1208 TCGv_i64 t0 = tcg_temp_new_i64(); 1209 tcg_gen_movi_i64(t0, 0); 1210 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2); 1211 tcg_temp_free_i64(t0); 1212 } else { 1213 gen_helper_divu_i64(ret, arg1, arg2); 1214 } 1215 } 1216 1217 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1218 { 1219 if (TCG_TARGET_HAS_rem_i64) { 1220 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2); 1221 } else if (TCG_TARGET_HAS_div_i64) { 1222 TCGv_i64 t0 = tcg_temp_new_i64(); 1223 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2); 1224 tcg_gen_mul_i64(t0, t0, arg2); 1225 tcg_gen_sub_i64(ret, arg1, t0); 1226 tcg_temp_free_i64(t0); 1227 } else if (TCG_TARGET_HAS_div2_i64) { 1228 TCGv_i64 t0 = tcg_temp_new_i64(); 1229 tcg_gen_movi_i64(t0, 0); 1230 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2); 1231 tcg_temp_free_i64(t0); 1232 } else { 1233 gen_helper_remu_i64(ret, arg1, arg2); 1234 } 1235 } 1236 1237 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg) 1238 { 1239 if (TCG_TARGET_REG_BITS == 32) { 1240 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1241 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1242 } else if (TCG_TARGET_HAS_ext8s_i64) { 1243 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg); 1244 } else { 1245 tcg_gen_shli_i64(ret, arg, 56); 1246 tcg_gen_sari_i64(ret, ret, 56); 1247 } 1248 } 1249 1250 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg) 1251 { 1252 if (TCG_TARGET_REG_BITS == 32) { 1253 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1254 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1255 } else if (TCG_TARGET_HAS_ext16s_i64) { 1256 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg); 1257 } else { 1258 tcg_gen_shli_i64(ret, arg, 48); 1259 tcg_gen_sari_i64(ret, ret, 48); 1260 } 1261 } 1262 1263 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg) 1264 { 1265 if (TCG_TARGET_REG_BITS == 32) { 1266 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1267 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1268 } else if (TCG_TARGET_HAS_ext32s_i64) { 1269 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg); 1270 } else { 1271 tcg_gen_shli_i64(ret, arg, 32); 1272 tcg_gen_sari_i64(ret, ret, 32); 1273 } 1274 } 1275 1276 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg) 1277 { 1278 if (TCG_TARGET_REG_BITS == 32) { 1279 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1280 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1281 } else if (TCG_TARGET_HAS_ext8u_i64) { 1282 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg); 1283 } else { 1284 tcg_gen_andi_i64(ret, arg, 0xffu); 1285 } 1286 } 1287 1288 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg) 1289 { 1290 if (TCG_TARGET_REG_BITS == 32) { 1291 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1292 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1293 } else if (TCG_TARGET_HAS_ext16u_i64) { 1294 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg); 1295 } else { 1296 tcg_gen_andi_i64(ret, arg, 0xffffu); 1297 } 1298 } 1299 1300 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg) 1301 { 1302 if (TCG_TARGET_REG_BITS == 32) { 1303 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1304 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1305 } else if (TCG_TARGET_HAS_ext32u_i64) { 1306 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg); 1307 } else { 1308 tcg_gen_andi_i64(ret, arg, 0xffffffffu); 1309 } 1310 } 1311 1312 /* Note: we assume the six high bytes are set to zero */ 1313 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg) 1314 { 1315 if (TCG_TARGET_REG_BITS == 32) { 1316 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1317 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1318 } else if (TCG_TARGET_HAS_bswap16_i64) { 1319 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg); 1320 } else { 1321 TCGv_i64 t0 = tcg_temp_new_i64(); 1322 1323 tcg_gen_ext8u_i64(t0, arg); 1324 tcg_gen_shli_i64(t0, t0, 8); 1325 tcg_gen_shri_i64(ret, arg, 8); 1326 tcg_gen_or_i64(ret, ret, t0); 1327 tcg_temp_free_i64(t0); 1328 } 1329 } 1330 1331 /* Note: we assume the four high bytes are set to zero */ 1332 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg) 1333 { 1334 if (TCG_TARGET_REG_BITS == 32) { 1335 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1336 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1337 } else if (TCG_TARGET_HAS_bswap32_i64) { 1338 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg); 1339 } else { 1340 TCGv_i64 t0, t1; 1341 t0 = tcg_temp_new_i64(); 1342 t1 = tcg_temp_new_i64(); 1343 1344 tcg_gen_shli_i64(t0, arg, 24); 1345 tcg_gen_ext32u_i64(t0, t0); 1346 1347 tcg_gen_andi_i64(t1, arg, 0x0000ff00); 1348 tcg_gen_shli_i64(t1, t1, 8); 1349 tcg_gen_or_i64(t0, t0, t1); 1350 1351 tcg_gen_shri_i64(t1, arg, 8); 1352 tcg_gen_andi_i64(t1, t1, 0x0000ff00); 1353 tcg_gen_or_i64(t0, t0, t1); 1354 1355 tcg_gen_shri_i64(t1, arg, 24); 1356 tcg_gen_or_i64(ret, t0, t1); 1357 tcg_temp_free_i64(t0); 1358 tcg_temp_free_i64(t1); 1359 } 1360 } 1361 1362 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg) 1363 { 1364 if (TCG_TARGET_REG_BITS == 32) { 1365 TCGv_i32 t0, t1; 1366 t0 = tcg_temp_new_i32(); 1367 t1 = tcg_temp_new_i32(); 1368 1369 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg)); 1370 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg)); 1371 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 1372 tcg_gen_mov_i32(TCGV_HIGH(ret), t0); 1373 tcg_temp_free_i32(t0); 1374 tcg_temp_free_i32(t1); 1375 } else if (TCG_TARGET_HAS_bswap64_i64) { 1376 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg); 1377 } else { 1378 TCGv_i64 t0 = tcg_temp_new_i64(); 1379 TCGv_i64 t1 = tcg_temp_new_i64(); 1380 1381 tcg_gen_shli_i64(t0, arg, 56); 1382 1383 tcg_gen_andi_i64(t1, arg, 0x0000ff00); 1384 tcg_gen_shli_i64(t1, t1, 40); 1385 tcg_gen_or_i64(t0, t0, t1); 1386 1387 tcg_gen_andi_i64(t1, arg, 0x00ff0000); 1388 tcg_gen_shli_i64(t1, t1, 24); 1389 tcg_gen_or_i64(t0, t0, t1); 1390 1391 tcg_gen_andi_i64(t1, arg, 0xff000000); 1392 tcg_gen_shli_i64(t1, t1, 8); 1393 tcg_gen_or_i64(t0, t0, t1); 1394 1395 tcg_gen_shri_i64(t1, arg, 8); 1396 tcg_gen_andi_i64(t1, t1, 0xff000000); 1397 tcg_gen_or_i64(t0, t0, t1); 1398 1399 tcg_gen_shri_i64(t1, arg, 24); 1400 tcg_gen_andi_i64(t1, t1, 0x00ff0000); 1401 tcg_gen_or_i64(t0, t0, t1); 1402 1403 tcg_gen_shri_i64(t1, arg, 40); 1404 tcg_gen_andi_i64(t1, t1, 0x0000ff00); 1405 tcg_gen_or_i64(t0, t0, t1); 1406 1407 tcg_gen_shri_i64(t1, arg, 56); 1408 tcg_gen_or_i64(ret, t0, t1); 1409 tcg_temp_free_i64(t0); 1410 tcg_temp_free_i64(t1); 1411 } 1412 } 1413 1414 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg) 1415 { 1416 if (TCG_TARGET_REG_BITS == 32) { 1417 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg)); 1418 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg)); 1419 } else if (TCG_TARGET_HAS_not_i64) { 1420 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg); 1421 } else { 1422 tcg_gen_xori_i64(ret, arg, -1); 1423 } 1424 } 1425 1426 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1427 { 1428 if (TCG_TARGET_REG_BITS == 32) { 1429 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1430 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1431 } else if (TCG_TARGET_HAS_andc_i64) { 1432 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2); 1433 } else { 1434 TCGv_i64 t0 = tcg_temp_new_i64(); 1435 tcg_gen_not_i64(t0, arg2); 1436 tcg_gen_and_i64(ret, arg1, t0); 1437 tcg_temp_free_i64(t0); 1438 } 1439 } 1440 1441 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1442 { 1443 if (TCG_TARGET_REG_BITS == 32) { 1444 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1445 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1446 } else if (TCG_TARGET_HAS_eqv_i64) { 1447 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2); 1448 } else { 1449 tcg_gen_xor_i64(ret, arg1, arg2); 1450 tcg_gen_not_i64(ret, ret); 1451 } 1452 } 1453 1454 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1455 { 1456 if (TCG_TARGET_REG_BITS == 32) { 1457 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1458 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1459 } else if (TCG_TARGET_HAS_nand_i64) { 1460 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2); 1461 } else { 1462 tcg_gen_and_i64(ret, arg1, arg2); 1463 tcg_gen_not_i64(ret, ret); 1464 } 1465 } 1466 1467 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1468 { 1469 if (TCG_TARGET_REG_BITS == 32) { 1470 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1471 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1472 } else if (TCG_TARGET_HAS_nor_i64) { 1473 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2); 1474 } else { 1475 tcg_gen_or_i64(ret, arg1, arg2); 1476 tcg_gen_not_i64(ret, ret); 1477 } 1478 } 1479 1480 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1481 { 1482 if (TCG_TARGET_REG_BITS == 32) { 1483 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); 1484 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2)); 1485 } else if (TCG_TARGET_HAS_orc_i64) { 1486 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2); 1487 } else { 1488 TCGv_i64 t0 = tcg_temp_new_i64(); 1489 tcg_gen_not_i64(t0, arg2); 1490 tcg_gen_or_i64(ret, arg1, t0); 1491 tcg_temp_free_i64(t0); 1492 } 1493 } 1494 1495 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1496 { 1497 if (TCG_TARGET_HAS_rot_i64) { 1498 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2); 1499 } else { 1500 TCGv_i64 t0, t1; 1501 t0 = tcg_temp_new_i64(); 1502 t1 = tcg_temp_new_i64(); 1503 tcg_gen_shl_i64(t0, arg1, arg2); 1504 tcg_gen_subfi_i64(t1, 64, arg2); 1505 tcg_gen_shr_i64(t1, arg1, t1); 1506 tcg_gen_or_i64(ret, t0, t1); 1507 tcg_temp_free_i64(t0); 1508 tcg_temp_free_i64(t1); 1509 } 1510 } 1511 1512 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1513 { 1514 tcg_debug_assert(arg2 < 64); 1515 /* some cases can be optimized here */ 1516 if (arg2 == 0) { 1517 tcg_gen_mov_i64(ret, arg1); 1518 } else if (TCG_TARGET_HAS_rot_i64) { 1519 TCGv_i64 t0 = tcg_const_i64(arg2); 1520 tcg_gen_rotl_i64(ret, arg1, t0); 1521 tcg_temp_free_i64(t0); 1522 } else { 1523 TCGv_i64 t0, t1; 1524 t0 = tcg_temp_new_i64(); 1525 t1 = tcg_temp_new_i64(); 1526 tcg_gen_shli_i64(t0, arg1, arg2); 1527 tcg_gen_shri_i64(t1, arg1, 64 - arg2); 1528 tcg_gen_or_i64(ret, t0, t1); 1529 tcg_temp_free_i64(t0); 1530 tcg_temp_free_i64(t1); 1531 } 1532 } 1533 1534 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) 1535 { 1536 if (TCG_TARGET_HAS_rot_i64) { 1537 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2); 1538 } else { 1539 TCGv_i64 t0, t1; 1540 t0 = tcg_temp_new_i64(); 1541 t1 = tcg_temp_new_i64(); 1542 tcg_gen_shr_i64(t0, arg1, arg2); 1543 tcg_gen_subfi_i64(t1, 64, arg2); 1544 tcg_gen_shl_i64(t1, arg1, t1); 1545 tcg_gen_or_i64(ret, t0, t1); 1546 tcg_temp_free_i64(t0); 1547 tcg_temp_free_i64(t1); 1548 } 1549 } 1550 1551 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2) 1552 { 1553 tcg_debug_assert(arg2 < 64); 1554 /* some cases can be optimized here */ 1555 if (arg2 == 0) { 1556 tcg_gen_mov_i64(ret, arg1); 1557 } else { 1558 tcg_gen_rotli_i64(ret, arg1, 64 - arg2); 1559 } 1560 } 1561 1562 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2, 1563 unsigned int ofs, unsigned int len) 1564 { 1565 uint64_t mask; 1566 TCGv_i64 t1; 1567 1568 tcg_debug_assert(ofs < 64); 1569 tcg_debug_assert(len <= 64); 1570 tcg_debug_assert(ofs + len <= 64); 1571 1572 if (ofs == 0 && len == 64) { 1573 tcg_gen_mov_i64(ret, arg2); 1574 return; 1575 } 1576 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) { 1577 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len); 1578 return; 1579 } 1580 1581 if (TCG_TARGET_REG_BITS == 32) { 1582 if (ofs >= 32) { 1583 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 1584 TCGV_LOW(arg2), ofs - 32, len); 1585 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 1586 return; 1587 } 1588 if (ofs + len <= 32) { 1589 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1), 1590 TCGV_LOW(arg2), ofs, len); 1591 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 1592 return; 1593 } 1594 } 1595 1596 mask = (1ull << len) - 1; 1597 t1 = tcg_temp_new_i64(); 1598 1599 if (ofs + len < 64) { 1600 tcg_gen_andi_i64(t1, arg2, mask); 1601 tcg_gen_shli_i64(t1, t1, ofs); 1602 } else { 1603 tcg_gen_shli_i64(t1, arg2, ofs); 1604 } 1605 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs)); 1606 tcg_gen_or_i64(ret, ret, t1); 1607 1608 tcg_temp_free_i64(t1); 1609 } 1610 1611 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1, 1612 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2) 1613 { 1614 if (cond == TCG_COND_ALWAYS) { 1615 tcg_gen_mov_i64(ret, v1); 1616 } else if (cond == TCG_COND_NEVER) { 1617 tcg_gen_mov_i64(ret, v2); 1618 } else if (TCG_TARGET_REG_BITS == 32) { 1619 TCGv_i32 t0 = tcg_temp_new_i32(); 1620 TCGv_i32 t1 = tcg_temp_new_i32(); 1621 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0, 1622 TCGV_LOW(c1), TCGV_HIGH(c1), 1623 TCGV_LOW(c2), TCGV_HIGH(c2), cond); 1624 1625 if (TCG_TARGET_HAS_movcond_i32) { 1626 tcg_gen_movi_i32(t1, 0); 1627 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1, 1628 TCGV_LOW(v1), TCGV_LOW(v2)); 1629 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1, 1630 TCGV_HIGH(v1), TCGV_HIGH(v2)); 1631 } else { 1632 tcg_gen_neg_i32(t0, t0); 1633 1634 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0); 1635 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0); 1636 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1); 1637 1638 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0); 1639 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0); 1640 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1); 1641 } 1642 tcg_temp_free_i32(t0); 1643 tcg_temp_free_i32(t1); 1644 } else if (TCG_TARGET_HAS_movcond_i64) { 1645 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond); 1646 } else { 1647 TCGv_i64 t0 = tcg_temp_new_i64(); 1648 TCGv_i64 t1 = tcg_temp_new_i64(); 1649 tcg_gen_setcond_i64(cond, t0, c1, c2); 1650 tcg_gen_neg_i64(t0, t0); 1651 tcg_gen_and_i64(t1, v1, t0); 1652 tcg_gen_andc_i64(ret, v2, t0); 1653 tcg_gen_or_i64(ret, ret, t1); 1654 tcg_temp_free_i64(t0); 1655 tcg_temp_free_i64(t1); 1656 } 1657 } 1658 1659 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 1660 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 1661 { 1662 if (TCG_TARGET_HAS_add2_i64) { 1663 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh); 1664 } else { 1665 TCGv_i64 t0 = tcg_temp_new_i64(); 1666 TCGv_i64 t1 = tcg_temp_new_i64(); 1667 tcg_gen_add_i64(t0, al, bl); 1668 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al); 1669 tcg_gen_add_i64(rh, ah, bh); 1670 tcg_gen_add_i64(rh, rh, t1); 1671 tcg_gen_mov_i64(rl, t0); 1672 tcg_temp_free_i64(t0); 1673 tcg_temp_free_i64(t1); 1674 } 1675 } 1676 1677 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al, 1678 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh) 1679 { 1680 if (TCG_TARGET_HAS_sub2_i64) { 1681 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh); 1682 } else { 1683 TCGv_i64 t0 = tcg_temp_new_i64(); 1684 TCGv_i64 t1 = tcg_temp_new_i64(); 1685 tcg_gen_sub_i64(t0, al, bl); 1686 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl); 1687 tcg_gen_sub_i64(rh, ah, bh); 1688 tcg_gen_sub_i64(rh, rh, t1); 1689 tcg_gen_mov_i64(rl, t0); 1690 tcg_temp_free_i64(t0); 1691 tcg_temp_free_i64(t1); 1692 } 1693 } 1694 1695 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 1696 { 1697 if (TCG_TARGET_HAS_mulu2_i64) { 1698 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2); 1699 } else if (TCG_TARGET_HAS_muluh_i64) { 1700 TCGv_i64 t = tcg_temp_new_i64(); 1701 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 1702 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2); 1703 tcg_gen_mov_i64(rl, t); 1704 tcg_temp_free_i64(t); 1705 } else { 1706 TCGv_i64 t0 = tcg_temp_new_i64(); 1707 tcg_gen_mul_i64(t0, arg1, arg2); 1708 gen_helper_muluh_i64(rh, arg1, arg2); 1709 tcg_gen_mov_i64(rl, t0); 1710 tcg_temp_free_i64(t0); 1711 } 1712 } 1713 1714 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2) 1715 { 1716 if (TCG_TARGET_HAS_muls2_i64) { 1717 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2); 1718 } else if (TCG_TARGET_HAS_mulsh_i64) { 1719 TCGv_i64 t = tcg_temp_new_i64(); 1720 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2); 1721 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2); 1722 tcg_gen_mov_i64(rl, t); 1723 tcg_temp_free_i64(t); 1724 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) { 1725 TCGv_i64 t0 = tcg_temp_new_i64(); 1726 TCGv_i64 t1 = tcg_temp_new_i64(); 1727 TCGv_i64 t2 = tcg_temp_new_i64(); 1728 TCGv_i64 t3 = tcg_temp_new_i64(); 1729 tcg_gen_mulu2_i64(t0, t1, arg1, arg2); 1730 /* Adjust for negative inputs. */ 1731 tcg_gen_sari_i64(t2, arg1, 63); 1732 tcg_gen_sari_i64(t3, arg2, 63); 1733 tcg_gen_and_i64(t2, t2, arg2); 1734 tcg_gen_and_i64(t3, t3, arg1); 1735 tcg_gen_sub_i64(rh, t1, t2); 1736 tcg_gen_sub_i64(rh, rh, t3); 1737 tcg_gen_mov_i64(rl, t0); 1738 tcg_temp_free_i64(t0); 1739 tcg_temp_free_i64(t1); 1740 tcg_temp_free_i64(t2); 1741 tcg_temp_free_i64(t3); 1742 } else { 1743 TCGv_i64 t0 = tcg_temp_new_i64(); 1744 tcg_gen_mul_i64(t0, arg1, arg2); 1745 gen_helper_mulsh_i64(rh, arg1, arg2); 1746 tcg_gen_mov_i64(rl, t0); 1747 tcg_temp_free_i64(t0); 1748 } 1749 } 1750 1751 /* Size changing operations. */ 1752 1753 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 1754 { 1755 if (TCG_TARGET_REG_BITS == 32) { 1756 tcg_gen_mov_i32(ret, TCGV_LOW(arg)); 1757 } else if (TCG_TARGET_HAS_extrl_i64_i32) { 1758 tcg_gen_op2(&tcg_ctx, INDEX_op_extrl_i64_i32, 1759 GET_TCGV_I32(ret), GET_TCGV_I64(arg)); 1760 } else { 1761 tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(arg))); 1762 } 1763 } 1764 1765 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg) 1766 { 1767 if (TCG_TARGET_REG_BITS == 32) { 1768 tcg_gen_mov_i32(ret, TCGV_HIGH(arg)); 1769 } else if (TCG_TARGET_HAS_extrh_i64_i32) { 1770 tcg_gen_op2(&tcg_ctx, INDEX_op_extrh_i64_i32, 1771 GET_TCGV_I32(ret), GET_TCGV_I64(arg)); 1772 } else { 1773 TCGv_i64 t = tcg_temp_new_i64(); 1774 tcg_gen_shri_i64(t, arg, 32); 1775 tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(t))); 1776 tcg_temp_free_i64(t); 1777 } 1778 } 1779 1780 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 1781 { 1782 if (TCG_TARGET_REG_BITS == 32) { 1783 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 1784 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 1785 } else { 1786 tcg_gen_op2(&tcg_ctx, INDEX_op_extu_i32_i64, 1787 GET_TCGV_I64(ret), GET_TCGV_I32(arg)); 1788 } 1789 } 1790 1791 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg) 1792 { 1793 if (TCG_TARGET_REG_BITS == 32) { 1794 tcg_gen_mov_i32(TCGV_LOW(ret), arg); 1795 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31); 1796 } else { 1797 tcg_gen_op2(&tcg_ctx, INDEX_op_ext_i32_i64, 1798 GET_TCGV_I64(ret), GET_TCGV_I32(arg)); 1799 } 1800 } 1801 1802 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high) 1803 { 1804 TCGv_i64 tmp; 1805 1806 if (TCG_TARGET_REG_BITS == 32) { 1807 tcg_gen_mov_i32(TCGV_LOW(dest), low); 1808 tcg_gen_mov_i32(TCGV_HIGH(dest), high); 1809 return; 1810 } 1811 1812 tmp = tcg_temp_new_i64(); 1813 /* These extensions are only needed for type correctness. 1814 We may be able to do better given target specific information. */ 1815 tcg_gen_extu_i32_i64(tmp, high); 1816 tcg_gen_extu_i32_i64(dest, low); 1817 /* If deposit is available, use it. Otherwise use the extra 1818 knowledge that we have of the zero-extensions above. */ 1819 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) { 1820 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32); 1821 } else { 1822 tcg_gen_shli_i64(tmp, tmp, 32); 1823 tcg_gen_or_i64(dest, dest, tmp); 1824 } 1825 tcg_temp_free_i64(tmp); 1826 } 1827 1828 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg) 1829 { 1830 if (TCG_TARGET_REG_BITS == 32) { 1831 tcg_gen_mov_i32(lo, TCGV_LOW(arg)); 1832 tcg_gen_mov_i32(hi, TCGV_HIGH(arg)); 1833 } else { 1834 tcg_gen_extrl_i64_i32(lo, arg); 1835 tcg_gen_extrh_i64_i32(hi, arg); 1836 } 1837 } 1838 1839 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg) 1840 { 1841 tcg_gen_ext32u_i64(lo, arg); 1842 tcg_gen_shri_i64(hi, arg, 32); 1843 } 1844 1845 /* QEMU specific operations. */ 1846 1847 void tcg_gen_goto_tb(unsigned idx) 1848 { 1849 /* We only support two chained exits. */ 1850 tcg_debug_assert(idx <= 1); 1851 #ifdef CONFIG_DEBUG_TCG 1852 /* Verify that we havn't seen this numbered exit before. */ 1853 tcg_debug_assert((tcg_ctx.goto_tb_issue_mask & (1 << idx)) == 0); 1854 tcg_ctx.goto_tb_issue_mask |= 1 << idx; 1855 #endif 1856 tcg_gen_op1i(INDEX_op_goto_tb, idx); 1857 } 1858 1859 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 1860 { 1861 /* Trigger the asserts within as early as possible. */ 1862 (void)get_alignment_bits(op); 1863 1864 switch (op & MO_SIZE) { 1865 case MO_8: 1866 op &= ~MO_BSWAP; 1867 break; 1868 case MO_16: 1869 break; 1870 case MO_32: 1871 if (!is64) { 1872 op &= ~MO_SIGN; 1873 } 1874 break; 1875 case MO_64: 1876 if (!is64) { 1877 tcg_abort(); 1878 } 1879 break; 1880 } 1881 if (st) { 1882 op &= ~MO_SIGN; 1883 } 1884 return op; 1885 } 1886 1887 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr, 1888 TCGMemOp memop, TCGArg idx) 1889 { 1890 TCGMemOpIdx oi = make_memop_idx(memop, idx); 1891 #if TARGET_LONG_BITS == 32 1892 tcg_gen_op3i_i32(opc, val, addr, oi); 1893 #else 1894 if (TCG_TARGET_REG_BITS == 32) { 1895 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi); 1896 } else { 1897 tcg_gen_op3(&tcg_ctx, opc, GET_TCGV_I32(val), GET_TCGV_I64(addr), oi); 1898 } 1899 #endif 1900 } 1901 1902 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr, 1903 TCGMemOp memop, TCGArg idx) 1904 { 1905 TCGMemOpIdx oi = make_memop_idx(memop, idx); 1906 #if TARGET_LONG_BITS == 32 1907 if (TCG_TARGET_REG_BITS == 32) { 1908 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi); 1909 } else { 1910 tcg_gen_op3(&tcg_ctx, opc, GET_TCGV_I64(val), GET_TCGV_I32(addr), oi); 1911 } 1912 #else 1913 if (TCG_TARGET_REG_BITS == 32) { 1914 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), 1915 TCGV_LOW(addr), TCGV_HIGH(addr), oi); 1916 } else { 1917 tcg_gen_op3i_i64(opc, val, addr, oi); 1918 } 1919 #endif 1920 } 1921 1922 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1923 { 1924 memop = tcg_canonicalize_memop(memop, 0, 0); 1925 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env, 1926 addr, trace_mem_get_info(memop, 0)); 1927 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx); 1928 } 1929 1930 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1931 { 1932 memop = tcg_canonicalize_memop(memop, 0, 1); 1933 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env, 1934 addr, trace_mem_get_info(memop, 1)); 1935 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx); 1936 } 1937 1938 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1939 { 1940 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 1941 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 1942 if (memop & MO_SIGN) { 1943 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 1944 } else { 1945 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 1946 } 1947 return; 1948 } 1949 1950 memop = tcg_canonicalize_memop(memop, 1, 0); 1951 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env, 1952 addr, trace_mem_get_info(memop, 0)); 1953 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx); 1954 } 1955 1956 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1957 { 1958 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) { 1959 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 1960 return; 1961 } 1962 1963 memop = tcg_canonicalize_memop(memop, 1, 1); 1964 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env, 1965 addr, trace_mem_get_info(memop, 1)); 1966 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx); 1967 } 1968 1969 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc) 1970 { 1971 switch (opc & MO_SSIZE) { 1972 case MO_SB: 1973 tcg_gen_ext8s_i32(ret, val); 1974 break; 1975 case MO_UB: 1976 tcg_gen_ext8u_i32(ret, val); 1977 break; 1978 case MO_SW: 1979 tcg_gen_ext16s_i32(ret, val); 1980 break; 1981 case MO_UW: 1982 tcg_gen_ext16u_i32(ret, val); 1983 break; 1984 default: 1985 tcg_gen_mov_i32(ret, val); 1986 break; 1987 } 1988 } 1989 1990 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc) 1991 { 1992 switch (opc & MO_SSIZE) { 1993 case MO_SB: 1994 tcg_gen_ext8s_i64(ret, val); 1995 break; 1996 case MO_UB: 1997 tcg_gen_ext8u_i64(ret, val); 1998 break; 1999 case MO_SW: 2000 tcg_gen_ext16s_i64(ret, val); 2001 break; 2002 case MO_UW: 2003 tcg_gen_ext16u_i64(ret, val); 2004 break; 2005 case MO_SL: 2006 tcg_gen_ext32s_i64(ret, val); 2007 break; 2008 case MO_UL: 2009 tcg_gen_ext32u_i64(ret, val); 2010 break; 2011 default: 2012 tcg_gen_mov_i64(ret, val); 2013 break; 2014 } 2015 } 2016 2017 #ifdef CONFIG_SOFTMMU 2018 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, 2019 TCGv_i32, TCGv_i32, TCGv_i32); 2020 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, 2021 TCGv_i64, TCGv_i64, TCGv_i32); 2022 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, 2023 TCGv_i32, TCGv_i32); 2024 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, 2025 TCGv_i64, TCGv_i32); 2026 #else 2027 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32); 2028 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64); 2029 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32); 2030 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64); 2031 #endif 2032 2033 #ifdef CONFIG_ATOMIC64 2034 # define WITH_ATOMIC64(X) X, 2035 #else 2036 # define WITH_ATOMIC64(X) 2037 #endif 2038 2039 static void * const table_cmpxchg[16] = { 2040 [MO_8] = gen_helper_atomic_cmpxchgb, 2041 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le, 2042 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be, 2043 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le, 2044 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be, 2045 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le) 2046 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be) 2047 }; 2048 2049 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv, 2050 TCGv_i32 newv, TCGArg idx, TCGMemOp memop) 2051 { 2052 memop = tcg_canonicalize_memop(memop, 0, 0); 2053 2054 if (!parallel_cpus) { 2055 TCGv_i32 t1 = tcg_temp_new_i32(); 2056 TCGv_i32 t2 = tcg_temp_new_i32(); 2057 2058 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE); 2059 2060 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2061 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1); 2062 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2063 tcg_temp_free_i32(t2); 2064 2065 if (memop & MO_SIGN) { 2066 tcg_gen_ext_i32(retv, t1, memop); 2067 } else { 2068 tcg_gen_mov_i32(retv, t1); 2069 } 2070 tcg_temp_free_i32(t1); 2071 } else { 2072 gen_atomic_cx_i32 gen; 2073 2074 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2075 tcg_debug_assert(gen != NULL); 2076 2077 #ifdef CONFIG_SOFTMMU 2078 { 2079 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2080 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi); 2081 tcg_temp_free_i32(oi); 2082 } 2083 #else 2084 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv); 2085 #endif 2086 2087 if (memop & MO_SIGN) { 2088 tcg_gen_ext_i32(retv, retv, memop); 2089 } 2090 } 2091 } 2092 2093 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv, 2094 TCGv_i64 newv, TCGArg idx, TCGMemOp memop) 2095 { 2096 memop = tcg_canonicalize_memop(memop, 1, 0); 2097 2098 if (!parallel_cpus) { 2099 TCGv_i64 t1 = tcg_temp_new_i64(); 2100 TCGv_i64 t2 = tcg_temp_new_i64(); 2101 2102 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE); 2103 2104 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2105 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1); 2106 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2107 tcg_temp_free_i64(t2); 2108 2109 if (memop & MO_SIGN) { 2110 tcg_gen_ext_i64(retv, t1, memop); 2111 } else { 2112 tcg_gen_mov_i64(retv, t1); 2113 } 2114 tcg_temp_free_i64(t1); 2115 } else if ((memop & MO_SIZE) == MO_64) { 2116 #ifdef CONFIG_ATOMIC64 2117 gen_atomic_cx_i64 gen; 2118 2119 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)]; 2120 tcg_debug_assert(gen != NULL); 2121 2122 #ifdef CONFIG_SOFTMMU 2123 { 2124 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx)); 2125 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi); 2126 tcg_temp_free_i32(oi); 2127 } 2128 #else 2129 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv); 2130 #endif 2131 #else 2132 gen_helper_exit_atomic(tcg_ctx.tcg_env); 2133 #endif /* CONFIG_ATOMIC64 */ 2134 } else { 2135 TCGv_i32 c32 = tcg_temp_new_i32(); 2136 TCGv_i32 n32 = tcg_temp_new_i32(); 2137 TCGv_i32 r32 = tcg_temp_new_i32(); 2138 2139 tcg_gen_extrl_i64_i32(c32, cmpv); 2140 tcg_gen_extrl_i64_i32(n32, newv); 2141 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN); 2142 tcg_temp_free_i32(c32); 2143 tcg_temp_free_i32(n32); 2144 2145 tcg_gen_extu_i32_i64(retv, r32); 2146 tcg_temp_free_i32(r32); 2147 2148 if (memop & MO_SIGN) { 2149 tcg_gen_ext_i64(retv, retv, memop); 2150 } 2151 } 2152 } 2153 2154 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 2155 TCGArg idx, TCGMemOp memop, bool new_val, 2156 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32)) 2157 { 2158 TCGv_i32 t1 = tcg_temp_new_i32(); 2159 TCGv_i32 t2 = tcg_temp_new_i32(); 2160 2161 memop = tcg_canonicalize_memop(memop, 0, 0); 2162 2163 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN); 2164 gen(t2, t1, val); 2165 tcg_gen_qemu_st_i32(t2, addr, idx, memop); 2166 2167 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop); 2168 tcg_temp_free_i32(t1); 2169 tcg_temp_free_i32(t2); 2170 } 2171 2172 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val, 2173 TCGArg idx, TCGMemOp memop, void * const table[]) 2174 { 2175 gen_atomic_op_i32 gen; 2176 2177 memop = tcg_canonicalize_memop(memop, 0, 0); 2178 2179 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 2180 tcg_debug_assert(gen != NULL); 2181 2182 #ifdef CONFIG_SOFTMMU 2183 { 2184 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2185 gen(ret, tcg_ctx.tcg_env, addr, val, oi); 2186 tcg_temp_free_i32(oi); 2187 } 2188 #else 2189 gen(ret, tcg_ctx.tcg_env, addr, val); 2190 #endif 2191 2192 if (memop & MO_SIGN) { 2193 tcg_gen_ext_i32(ret, ret, memop); 2194 } 2195 } 2196 2197 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 2198 TCGArg idx, TCGMemOp memop, bool new_val, 2199 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64)) 2200 { 2201 TCGv_i64 t1 = tcg_temp_new_i64(); 2202 TCGv_i64 t2 = tcg_temp_new_i64(); 2203 2204 memop = tcg_canonicalize_memop(memop, 1, 0); 2205 2206 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN); 2207 gen(t2, t1, val); 2208 tcg_gen_qemu_st_i64(t2, addr, idx, memop); 2209 2210 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop); 2211 tcg_temp_free_i64(t1); 2212 tcg_temp_free_i64(t2); 2213 } 2214 2215 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val, 2216 TCGArg idx, TCGMemOp memop, void * const table[]) 2217 { 2218 memop = tcg_canonicalize_memop(memop, 1, 0); 2219 2220 if ((memop & MO_SIZE) == MO_64) { 2221 #ifdef CONFIG_ATOMIC64 2222 gen_atomic_op_i64 gen; 2223 2224 gen = table[memop & (MO_SIZE | MO_BSWAP)]; 2225 tcg_debug_assert(gen != NULL); 2226 2227 #ifdef CONFIG_SOFTMMU 2228 { 2229 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx)); 2230 gen(ret, tcg_ctx.tcg_env, addr, val, oi); 2231 tcg_temp_free_i32(oi); 2232 } 2233 #else 2234 gen(ret, tcg_ctx.tcg_env, addr, val); 2235 #endif 2236 #else 2237 gen_helper_exit_atomic(tcg_ctx.tcg_env); 2238 #endif /* CONFIG_ATOMIC64 */ 2239 } else { 2240 TCGv_i32 v32 = tcg_temp_new_i32(); 2241 TCGv_i32 r32 = tcg_temp_new_i32(); 2242 2243 tcg_gen_extrl_i64_i32(v32, val); 2244 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table); 2245 tcg_temp_free_i32(v32); 2246 2247 tcg_gen_extu_i32_i64(ret, r32); 2248 tcg_temp_free_i32(r32); 2249 2250 if (memop & MO_SIGN) { 2251 tcg_gen_ext_i64(ret, ret, memop); 2252 } 2253 } 2254 } 2255 2256 #define GEN_ATOMIC_HELPER(NAME, OP, NEW) \ 2257 static void * const table_##NAME[16] = { \ 2258 [MO_8] = gen_helper_atomic_##NAME##b, \ 2259 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \ 2260 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \ 2261 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \ 2262 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \ 2263 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \ 2264 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \ 2265 }; \ 2266 void tcg_gen_atomic_##NAME##_i32 \ 2267 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \ 2268 { \ 2269 if (parallel_cpus) { \ 2270 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \ 2271 } else { \ 2272 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \ 2273 tcg_gen_##OP##_i32); \ 2274 } \ 2275 } \ 2276 void tcg_gen_atomic_##NAME##_i64 \ 2277 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \ 2278 { \ 2279 if (parallel_cpus) { \ 2280 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \ 2281 } else { \ 2282 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \ 2283 tcg_gen_##OP##_i64); \ 2284 } \ 2285 } 2286 2287 GEN_ATOMIC_HELPER(fetch_add, add, 0) 2288 GEN_ATOMIC_HELPER(fetch_and, and, 0) 2289 GEN_ATOMIC_HELPER(fetch_or, or, 0) 2290 GEN_ATOMIC_HELPER(fetch_xor, xor, 0) 2291 2292 GEN_ATOMIC_HELPER(add_fetch, add, 1) 2293 GEN_ATOMIC_HELPER(and_fetch, and, 1) 2294 GEN_ATOMIC_HELPER(or_fetch, or, 1) 2295 GEN_ATOMIC_HELPER(xor_fetch, xor, 1) 2296 2297 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b) 2298 { 2299 tcg_gen_mov_i32(r, b); 2300 } 2301 2302 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b) 2303 { 2304 tcg_gen_mov_i64(r, b); 2305 } 2306 2307 GEN_ATOMIC_HELPER(xchg, mov2, 0) 2308 2309 #undef GEN_ATOMIC_HELPER 2310