1/* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2018 SiFive, Inc 5 * Copyright (c) 2008-2009 Arnaud Patard <arnaud.patard@rtp-net.org> 6 * Copyright (c) 2009 Aurelien Jarno <aurelien@aurel32.net> 7 * Copyright (c) 2008 Fabrice Bellard 8 * 9 * Based on i386/tcg-target.c and mips/tcg-target.c 10 * 11 * Permission is hereby granted, free of charge, to any person obtaining a copy 12 * of this software and associated documentation files (the "Software"), to deal 13 * in the Software without restriction, including without limitation the rights 14 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 15 * copies of the Software, and to permit persons to whom the Software is 16 * furnished to do so, subject to the following conditions: 17 * 18 * The above copyright notice and this permission notice shall be included in 19 * all copies or substantial portions of the Software. 20 * 21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 26 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 27 * THE SOFTWARE. 28 */ 29 30#include "../tcg-ldst.c.inc" 31#include "../tcg-pool.c.inc" 32 33#ifdef CONFIG_DEBUG_TCG 34static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { 35 "zero", 36 "ra", 37 "sp", 38 "gp", 39 "tp", 40 "t0", 41 "t1", 42 "t2", 43 "s0", 44 "s1", 45 "a0", 46 "a1", 47 "a2", 48 "a3", 49 "a4", 50 "a5", 51 "a6", 52 "a7", 53 "s2", 54 "s3", 55 "s4", 56 "s5", 57 "s6", 58 "s7", 59 "s8", 60 "s9", 61 "s10", 62 "s11", 63 "t3", 64 "t4", 65 "t5", 66 "t6" 67}; 68#endif 69 70static const int tcg_target_reg_alloc_order[] = { 71 /* Call saved registers */ 72 /* TCG_REG_S0 reservered for TCG_AREG0 */ 73 TCG_REG_S1, 74 TCG_REG_S2, 75 TCG_REG_S3, 76 TCG_REG_S4, 77 TCG_REG_S5, 78 TCG_REG_S6, 79 TCG_REG_S7, 80 TCG_REG_S8, 81 TCG_REG_S9, 82 TCG_REG_S10, 83 TCG_REG_S11, 84 85 /* Call clobbered registers */ 86 TCG_REG_T0, 87 TCG_REG_T1, 88 TCG_REG_T2, 89 TCG_REG_T3, 90 TCG_REG_T4, 91 TCG_REG_T5, 92 TCG_REG_T6, 93 94 /* Argument registers */ 95 TCG_REG_A0, 96 TCG_REG_A1, 97 TCG_REG_A2, 98 TCG_REG_A3, 99 TCG_REG_A4, 100 TCG_REG_A5, 101 TCG_REG_A6, 102 TCG_REG_A7, 103}; 104 105static const int tcg_target_call_iarg_regs[] = { 106 TCG_REG_A0, 107 TCG_REG_A1, 108 TCG_REG_A2, 109 TCG_REG_A3, 110 TCG_REG_A4, 111 TCG_REG_A5, 112 TCG_REG_A6, 113 TCG_REG_A7, 114}; 115 116static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) 117{ 118 tcg_debug_assert(kind == TCG_CALL_RET_NORMAL); 119 tcg_debug_assert(slot >= 0 && slot <= 1); 120 return TCG_REG_A0 + slot; 121} 122 123#define TCG_CT_CONST_ZERO 0x100 124#define TCG_CT_CONST_S12 0x200 125#define TCG_CT_CONST_N12 0x400 126#define TCG_CT_CONST_M12 0x800 127 128#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) 129 130#define sextreg sextract64 131 132/* test if a constant matches the constraint */ 133static bool tcg_target_const_match(int64_t val, TCGType type, int ct) 134{ 135 if (ct & TCG_CT_CONST) { 136 return 1; 137 } 138 if ((ct & TCG_CT_CONST_ZERO) && val == 0) { 139 return 1; 140 } 141 /* 142 * Sign extended from 12 bits: [-0x800, 0x7ff]. 143 * Used for most arithmetic, as this is the isa field. 144 */ 145 if ((ct & TCG_CT_CONST_S12) && val >= -0x800 && val <= 0x7ff) { 146 return 1; 147 } 148 /* 149 * Sign extended from 12 bits, negated: [-0x7ff, 0x800]. 150 * Used for subtraction, where a constant must be handled by ADDI. 151 */ 152 if ((ct & TCG_CT_CONST_N12) && val >= -0x7ff && val <= 0x800) { 153 return 1; 154 } 155 /* 156 * Sign extended from 12 bits, +/- matching: [-0x7ff, 0x7ff]. 157 * Used by addsub2, which may need the negative operation, 158 * and requires the modified constant to be representable. 159 */ 160 if ((ct & TCG_CT_CONST_M12) && val >= -0x7ff && val <= 0x7ff) { 161 return 1; 162 } 163 return 0; 164} 165 166/* 167 * RISC-V Base ISA opcodes (IM) 168 */ 169 170typedef enum { 171 OPC_ADD = 0x33, 172 OPC_ADDI = 0x13, 173 OPC_AND = 0x7033, 174 OPC_ANDI = 0x7013, 175 OPC_AUIPC = 0x17, 176 OPC_BEQ = 0x63, 177 OPC_BGE = 0x5063, 178 OPC_BGEU = 0x7063, 179 OPC_BLT = 0x4063, 180 OPC_BLTU = 0x6063, 181 OPC_BNE = 0x1063, 182 OPC_DIV = 0x2004033, 183 OPC_DIVU = 0x2005033, 184 OPC_JAL = 0x6f, 185 OPC_JALR = 0x67, 186 OPC_LB = 0x3, 187 OPC_LBU = 0x4003, 188 OPC_LD = 0x3003, 189 OPC_LH = 0x1003, 190 OPC_LHU = 0x5003, 191 OPC_LUI = 0x37, 192 OPC_LW = 0x2003, 193 OPC_LWU = 0x6003, 194 OPC_MUL = 0x2000033, 195 OPC_MULH = 0x2001033, 196 OPC_MULHSU = 0x2002033, 197 OPC_MULHU = 0x2003033, 198 OPC_OR = 0x6033, 199 OPC_ORI = 0x6013, 200 OPC_REM = 0x2006033, 201 OPC_REMU = 0x2007033, 202 OPC_SB = 0x23, 203 OPC_SD = 0x3023, 204 OPC_SH = 0x1023, 205 OPC_SLL = 0x1033, 206 OPC_SLLI = 0x1013, 207 OPC_SLT = 0x2033, 208 OPC_SLTI = 0x2013, 209 OPC_SLTIU = 0x3013, 210 OPC_SLTU = 0x3033, 211 OPC_SRA = 0x40005033, 212 OPC_SRAI = 0x40005013, 213 OPC_SRL = 0x5033, 214 OPC_SRLI = 0x5013, 215 OPC_SUB = 0x40000033, 216 OPC_SW = 0x2023, 217 OPC_XOR = 0x4033, 218 OPC_XORI = 0x4013, 219 220 OPC_ADDIW = 0x1b, 221 OPC_ADDW = 0x3b, 222 OPC_DIVUW = 0x200503b, 223 OPC_DIVW = 0x200403b, 224 OPC_MULW = 0x200003b, 225 OPC_REMUW = 0x200703b, 226 OPC_REMW = 0x200603b, 227 OPC_SLLIW = 0x101b, 228 OPC_SLLW = 0x103b, 229 OPC_SRAIW = 0x4000501b, 230 OPC_SRAW = 0x4000503b, 231 OPC_SRLIW = 0x501b, 232 OPC_SRLW = 0x503b, 233 OPC_SUBW = 0x4000003b, 234 235 OPC_FENCE = 0x0000000f, 236 OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ 237} RISCVInsn; 238 239/* 240 * RISC-V immediate and instruction encoders (excludes 16-bit RVC) 241 */ 242 243/* Type-R */ 244 245static int32_t encode_r(RISCVInsn opc, TCGReg rd, TCGReg rs1, TCGReg rs2) 246{ 247 return opc | (rd & 0x1f) << 7 | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20; 248} 249 250/* Type-I */ 251 252static int32_t encode_imm12(uint32_t imm) 253{ 254 return (imm & 0xfff) << 20; 255} 256 257static int32_t encode_i(RISCVInsn opc, TCGReg rd, TCGReg rs1, uint32_t imm) 258{ 259 return opc | (rd & 0x1f) << 7 | (rs1 & 0x1f) << 15 | encode_imm12(imm); 260} 261 262/* Type-S */ 263 264static int32_t encode_simm12(uint32_t imm) 265{ 266 int32_t ret = 0; 267 268 ret |= (imm & 0xFE0) << 20; 269 ret |= (imm & 0x1F) << 7; 270 271 return ret; 272} 273 274static int32_t encode_s(RISCVInsn opc, TCGReg rs1, TCGReg rs2, uint32_t imm) 275{ 276 return opc | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20 | encode_simm12(imm); 277} 278 279/* Type-SB */ 280 281static int32_t encode_sbimm12(uint32_t imm) 282{ 283 int32_t ret = 0; 284 285 ret |= (imm & 0x1000) << 19; 286 ret |= (imm & 0x7e0) << 20; 287 ret |= (imm & 0x1e) << 7; 288 ret |= (imm & 0x800) >> 4; 289 290 return ret; 291} 292 293static int32_t encode_sb(RISCVInsn opc, TCGReg rs1, TCGReg rs2, uint32_t imm) 294{ 295 return opc | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20 | encode_sbimm12(imm); 296} 297 298/* Type-U */ 299 300static int32_t encode_uimm20(uint32_t imm) 301{ 302 return imm & 0xfffff000; 303} 304 305static int32_t encode_u(RISCVInsn opc, TCGReg rd, uint32_t imm) 306{ 307 return opc | (rd & 0x1f) << 7 | encode_uimm20(imm); 308} 309 310/* Type-UJ */ 311 312static int32_t encode_ujimm20(uint32_t imm) 313{ 314 int32_t ret = 0; 315 316 ret |= (imm & 0x0007fe) << (21 - 1); 317 ret |= (imm & 0x000800) << (20 - 11); 318 ret |= (imm & 0x0ff000) << (12 - 12); 319 ret |= (imm & 0x100000) << (31 - 20); 320 321 return ret; 322} 323 324static int32_t encode_uj(RISCVInsn opc, TCGReg rd, uint32_t imm) 325{ 326 return opc | (rd & 0x1f) << 7 | encode_ujimm20(imm); 327} 328 329/* 330 * RISC-V instruction emitters 331 */ 332 333static void tcg_out_opc_reg(TCGContext *s, RISCVInsn opc, 334 TCGReg rd, TCGReg rs1, TCGReg rs2) 335{ 336 tcg_out32(s, encode_r(opc, rd, rs1, rs2)); 337} 338 339static void tcg_out_opc_imm(TCGContext *s, RISCVInsn opc, 340 TCGReg rd, TCGReg rs1, TCGArg imm) 341{ 342 tcg_out32(s, encode_i(opc, rd, rs1, imm)); 343} 344 345static void tcg_out_opc_store(TCGContext *s, RISCVInsn opc, 346 TCGReg rs1, TCGReg rs2, uint32_t imm) 347{ 348 tcg_out32(s, encode_s(opc, rs1, rs2, imm)); 349} 350 351static void tcg_out_opc_branch(TCGContext *s, RISCVInsn opc, 352 TCGReg rs1, TCGReg rs2, uint32_t imm) 353{ 354 tcg_out32(s, encode_sb(opc, rs1, rs2, imm)); 355} 356 357static void tcg_out_opc_upper(TCGContext *s, RISCVInsn opc, 358 TCGReg rd, uint32_t imm) 359{ 360 tcg_out32(s, encode_u(opc, rd, imm)); 361} 362 363static void tcg_out_opc_jump(TCGContext *s, RISCVInsn opc, 364 TCGReg rd, uint32_t imm) 365{ 366 tcg_out32(s, encode_uj(opc, rd, imm)); 367} 368 369static void tcg_out_nop_fill(tcg_insn_unit *p, int count) 370{ 371 int i; 372 for (i = 0; i < count; ++i) { 373 p[i] = OPC_NOP; 374 } 375} 376 377/* 378 * Relocations 379 */ 380 381static bool reloc_sbimm12(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 382{ 383 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 384 intptr_t offset = (intptr_t)target - (intptr_t)src_rx; 385 386 tcg_debug_assert((offset & 1) == 0); 387 if (offset == sextreg(offset, 0, 12)) { 388 *src_rw |= encode_sbimm12(offset); 389 return true; 390 } 391 392 return false; 393} 394 395static bool reloc_jimm20(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 396{ 397 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 398 intptr_t offset = (intptr_t)target - (intptr_t)src_rx; 399 400 tcg_debug_assert((offset & 1) == 0); 401 if (offset == sextreg(offset, 0, 20)) { 402 *src_rw |= encode_ujimm20(offset); 403 return true; 404 } 405 406 return false; 407} 408 409static bool reloc_call(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 410{ 411 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 412 intptr_t offset = (intptr_t)target - (intptr_t)src_rx; 413 int32_t lo = sextreg(offset, 0, 12); 414 int32_t hi = offset - lo; 415 416 if (offset == hi + lo) { 417 src_rw[0] |= encode_uimm20(hi); 418 src_rw[1] |= encode_imm12(lo); 419 return true; 420 } 421 422 return false; 423} 424 425static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 426 intptr_t value, intptr_t addend) 427{ 428 tcg_debug_assert(addend == 0); 429 switch (type) { 430 case R_RISCV_BRANCH: 431 return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value); 432 case R_RISCV_JAL: 433 return reloc_jimm20(code_ptr, (tcg_insn_unit *)value); 434 case R_RISCV_CALL: 435 return reloc_call(code_ptr, (tcg_insn_unit *)value); 436 default: 437 g_assert_not_reached(); 438 } 439} 440 441/* 442 * TCG intrinsics 443 */ 444 445static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 446{ 447 if (ret == arg) { 448 return true; 449 } 450 switch (type) { 451 case TCG_TYPE_I32: 452 case TCG_TYPE_I64: 453 tcg_out_opc_imm(s, OPC_ADDI, ret, arg, 0); 454 break; 455 default: 456 g_assert_not_reached(); 457 } 458 return true; 459} 460 461static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, 462 tcg_target_long val) 463{ 464 tcg_target_long lo, hi, tmp; 465 int shift, ret; 466 467 if (type == TCG_TYPE_I32) { 468 val = (int32_t)val; 469 } 470 471 lo = sextreg(val, 0, 12); 472 if (val == lo) { 473 tcg_out_opc_imm(s, OPC_ADDI, rd, TCG_REG_ZERO, lo); 474 return; 475 } 476 477 hi = val - lo; 478 if (val == (int32_t)val) { 479 tcg_out_opc_upper(s, OPC_LUI, rd, hi); 480 if (lo != 0) { 481 tcg_out_opc_imm(s, OPC_ADDIW, rd, rd, lo); 482 } 483 return; 484 } 485 486 tmp = tcg_pcrel_diff(s, (void *)val); 487 if (tmp == (int32_t)tmp) { 488 tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); 489 tcg_out_opc_imm(s, OPC_ADDI, rd, rd, 0); 490 ret = reloc_call(s->code_ptr - 2, (const tcg_insn_unit *)val); 491 tcg_debug_assert(ret == true); 492 return; 493 } 494 495 /* Look for a single 20-bit section. */ 496 shift = ctz64(val); 497 tmp = val >> shift; 498 if (tmp == sextreg(tmp, 0, 20)) { 499 tcg_out_opc_upper(s, OPC_LUI, rd, tmp << 12); 500 if (shift > 12) { 501 tcg_out_opc_imm(s, OPC_SLLI, rd, rd, shift - 12); 502 } else { 503 tcg_out_opc_imm(s, OPC_SRAI, rd, rd, 12 - shift); 504 } 505 return; 506 } 507 508 /* Look for a few high zero bits, with lots of bits set in the middle. */ 509 shift = clz64(val); 510 tmp = val << shift; 511 if (tmp == sextreg(tmp, 12, 20) << 12) { 512 tcg_out_opc_upper(s, OPC_LUI, rd, tmp); 513 tcg_out_opc_imm(s, OPC_SRLI, rd, rd, shift); 514 return; 515 } else if (tmp == sextreg(tmp, 0, 12)) { 516 tcg_out_opc_imm(s, OPC_ADDI, rd, TCG_REG_ZERO, tmp); 517 tcg_out_opc_imm(s, OPC_SRLI, rd, rd, shift); 518 return; 519 } 520 521 /* Drop into the constant pool. */ 522 new_pool_label(s, val, R_RISCV_CALL, s->code_ptr, 0); 523 tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); 524 tcg_out_opc_imm(s, OPC_LD, rd, rd, 0); 525} 526 527static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2) 528{ 529 return false; 530} 531 532static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, 533 tcg_target_long imm) 534{ 535 /* This function is only used for passing structs by reference. */ 536 g_assert_not_reached(); 537} 538 539static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg) 540{ 541 tcg_out_opc_imm(s, OPC_ANDI, ret, arg, 0xff); 542} 543 544static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg) 545{ 546 tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); 547 tcg_out_opc_imm(s, OPC_SRLIW, ret, ret, 16); 548} 549 550static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg) 551{ 552 tcg_out_opc_imm(s, OPC_SLLI, ret, arg, 32); 553 tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32); 554} 555 556static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 557{ 558 tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24); 559 tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24); 560} 561 562static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 563{ 564 tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); 565 tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16); 566} 567 568static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) 569{ 570 tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0); 571} 572 573static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) 574{ 575 if (ret != arg) { 576 tcg_out_ext32s(s, ret, arg); 577 } 578} 579 580static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) 581{ 582 tcg_out_ext32u(s, ret, arg); 583} 584 585static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg) 586{ 587 tcg_out_ext32s(s, ret, arg); 588} 589 590static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, 591 TCGReg addr, intptr_t offset) 592{ 593 intptr_t imm12 = sextreg(offset, 0, 12); 594 595 if (offset != imm12) { 596 intptr_t diff = tcg_pcrel_diff(s, (void *)offset); 597 598 if (addr == TCG_REG_ZERO && diff == (int32_t)diff) { 599 imm12 = sextreg(diff, 0, 12); 600 tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP2, diff - imm12); 601 } else { 602 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP2, offset - imm12); 603 if (addr != TCG_REG_ZERO) { 604 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, addr); 605 } 606 } 607 addr = TCG_REG_TMP2; 608 } 609 610 switch (opc) { 611 case OPC_SB: 612 case OPC_SH: 613 case OPC_SW: 614 case OPC_SD: 615 tcg_out_opc_store(s, opc, addr, data, imm12); 616 break; 617 case OPC_LB: 618 case OPC_LBU: 619 case OPC_LH: 620 case OPC_LHU: 621 case OPC_LW: 622 case OPC_LWU: 623 case OPC_LD: 624 tcg_out_opc_imm(s, opc, data, addr, imm12); 625 break; 626 default: 627 g_assert_not_reached(); 628 } 629} 630 631static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg, 632 TCGReg arg1, intptr_t arg2) 633{ 634 RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_LW : OPC_LD; 635 tcg_out_ldst(s, insn, arg, arg1, arg2); 636} 637 638static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, 639 TCGReg arg1, intptr_t arg2) 640{ 641 RISCVInsn insn = type == TCG_TYPE_I32 ? OPC_SW : OPC_SD; 642 tcg_out_ldst(s, insn, arg, arg1, arg2); 643} 644 645static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 646 TCGReg base, intptr_t ofs) 647{ 648 if (val == 0) { 649 tcg_out_st(s, type, TCG_REG_ZERO, base, ofs); 650 return true; 651 } 652 return false; 653} 654 655static void tcg_out_addsub2(TCGContext *s, 656 TCGReg rl, TCGReg rh, 657 TCGReg al, TCGReg ah, 658 TCGArg bl, TCGArg bh, 659 bool cbl, bool cbh, bool is_sub, bool is32bit) 660{ 661 const RISCVInsn opc_add = is32bit ? OPC_ADDW : OPC_ADD; 662 const RISCVInsn opc_addi = is32bit ? OPC_ADDIW : OPC_ADDI; 663 const RISCVInsn opc_sub = is32bit ? OPC_SUBW : OPC_SUB; 664 TCGReg th = TCG_REG_TMP1; 665 666 /* If we have a negative constant such that negating it would 667 make the high part zero, we can (usually) eliminate one insn. */ 668 if (cbl && cbh && bh == -1 && bl != 0) { 669 bl = -bl; 670 bh = 0; 671 is_sub = !is_sub; 672 } 673 674 /* By operating on the high part first, we get to use the final 675 carry operation to move back from the temporary. */ 676 if (!cbh) { 677 tcg_out_opc_reg(s, (is_sub ? opc_sub : opc_add), th, ah, bh); 678 } else if (bh != 0 || ah == rl) { 679 tcg_out_opc_imm(s, opc_addi, th, ah, (is_sub ? -bh : bh)); 680 } else { 681 th = ah; 682 } 683 684 /* Note that tcg optimization should eliminate the bl == 0 case. */ 685 if (is_sub) { 686 if (cbl) { 687 tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, al, bl); 688 tcg_out_opc_imm(s, opc_addi, rl, al, -bl); 689 } else { 690 tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, al, bl); 691 tcg_out_opc_reg(s, opc_sub, rl, al, bl); 692 } 693 tcg_out_opc_reg(s, opc_sub, rh, th, TCG_REG_TMP0); 694 } else { 695 if (cbl) { 696 tcg_out_opc_imm(s, opc_addi, rl, al, bl); 697 tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, rl, bl); 698 } else if (al == bl) { 699 /* 700 * If the input regs overlap, this is a simple doubling 701 * and carry-out is the input msb. This special case is 702 * required when the output reg overlaps the input, 703 * but we might as well use it always. 704 */ 705 tcg_out_opc_imm(s, OPC_SLTI, TCG_REG_TMP0, al, 0); 706 tcg_out_opc_reg(s, opc_add, rl, al, al); 707 } else { 708 tcg_out_opc_reg(s, opc_add, rl, al, bl); 709 tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, 710 rl, (rl == bl ? al : bl)); 711 } 712 tcg_out_opc_reg(s, opc_add, rh, th, TCG_REG_TMP0); 713 } 714} 715 716static const struct { 717 RISCVInsn op; 718 bool swap; 719} tcg_brcond_to_riscv[] = { 720 [TCG_COND_EQ] = { OPC_BEQ, false }, 721 [TCG_COND_NE] = { OPC_BNE, false }, 722 [TCG_COND_LT] = { OPC_BLT, false }, 723 [TCG_COND_GE] = { OPC_BGE, false }, 724 [TCG_COND_LE] = { OPC_BGE, true }, 725 [TCG_COND_GT] = { OPC_BLT, true }, 726 [TCG_COND_LTU] = { OPC_BLTU, false }, 727 [TCG_COND_GEU] = { OPC_BGEU, false }, 728 [TCG_COND_LEU] = { OPC_BGEU, true }, 729 [TCG_COND_GTU] = { OPC_BLTU, true } 730}; 731 732static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, 733 TCGReg arg2, TCGLabel *l) 734{ 735 RISCVInsn op = tcg_brcond_to_riscv[cond].op; 736 737 tcg_debug_assert(op != 0); 738 739 if (tcg_brcond_to_riscv[cond].swap) { 740 TCGReg t = arg1; 741 arg1 = arg2; 742 arg2 = t; 743 } 744 745 tcg_out_reloc(s, s->code_ptr, R_RISCV_BRANCH, l, 0); 746 tcg_out_opc_branch(s, op, arg1, arg2, 0); 747} 748 749static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, 750 TCGReg arg1, TCGReg arg2) 751{ 752 switch (cond) { 753 case TCG_COND_EQ: 754 tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); 755 tcg_out_opc_imm(s, OPC_SLTIU, ret, ret, 1); 756 break; 757 case TCG_COND_NE: 758 tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); 759 tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, ret); 760 break; 761 case TCG_COND_LT: 762 tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); 763 break; 764 case TCG_COND_GE: 765 tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); 766 tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); 767 break; 768 case TCG_COND_LE: 769 tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); 770 tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); 771 break; 772 case TCG_COND_GT: 773 tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); 774 break; 775 case TCG_COND_LTU: 776 tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); 777 break; 778 case TCG_COND_GEU: 779 tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); 780 tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); 781 break; 782 case TCG_COND_LEU: 783 tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); 784 tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); 785 break; 786 case TCG_COND_GTU: 787 tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); 788 break; 789 default: 790 g_assert_not_reached(); 791 break; 792 } 793} 794 795static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) 796{ 797 TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; 798 ptrdiff_t offset = tcg_pcrel_diff(s, arg); 799 int ret; 800 801 tcg_debug_assert((offset & 1) == 0); 802 if (offset == sextreg(offset, 0, 20)) { 803 /* short jump: -2097150 to 2097152 */ 804 tcg_out_opc_jump(s, OPC_JAL, link, offset); 805 } else if (offset == (int32_t)offset) { 806 /* long jump: -2147483646 to 2147483648 */ 807 tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0); 808 tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0); 809 ret = reloc_call(s->code_ptr - 2, arg); 810 tcg_debug_assert(ret == true); 811 } else { 812 /* far jump: 64-bit */ 813 tcg_target_long imm = sextreg((tcg_target_long)arg, 0, 12); 814 tcg_target_long base = (tcg_target_long)arg - imm; 815 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, base); 816 tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, imm); 817 } 818} 819 820static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg, 821 const TCGHelperInfo *info) 822{ 823 tcg_out_call_int(s, arg, false); 824} 825 826static void tcg_out_mb(TCGContext *s, TCGArg a0) 827{ 828 tcg_insn_unit insn = OPC_FENCE; 829 830 if (a0 & TCG_MO_LD_LD) { 831 insn |= 0x02200000; 832 } 833 if (a0 & TCG_MO_ST_LD) { 834 insn |= 0x01200000; 835 } 836 if (a0 & TCG_MO_LD_ST) { 837 insn |= 0x02100000; 838 } 839 if (a0 & TCG_MO_ST_ST) { 840 insn |= 0x02200000; 841 } 842 tcg_out32(s, insn); 843} 844 845/* 846 * Load/store and TLB 847 */ 848 849#if defined(CONFIG_SOFTMMU) 850/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, 851 * MemOpIdx oi, uintptr_t ra) 852 */ 853static void * const qemu_ld_helpers[MO_SSIZE + 1] = { 854 [MO_UB] = helper_ret_ldub_mmu, 855 [MO_SB] = helper_ret_ldsb_mmu, 856#if HOST_BIG_ENDIAN 857 [MO_UW] = helper_be_lduw_mmu, 858 [MO_SW] = helper_be_ldsw_mmu, 859 [MO_UL] = helper_be_ldul_mmu, 860#if TCG_TARGET_REG_BITS == 64 861 [MO_SL] = helper_be_ldsl_mmu, 862#endif 863 [MO_UQ] = helper_be_ldq_mmu, 864#else 865 [MO_UW] = helper_le_lduw_mmu, 866 [MO_SW] = helper_le_ldsw_mmu, 867 [MO_UL] = helper_le_ldul_mmu, 868#if TCG_TARGET_REG_BITS == 64 869 [MO_SL] = helper_le_ldsl_mmu, 870#endif 871 [MO_UQ] = helper_le_ldq_mmu, 872#endif 873}; 874 875/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, 876 * uintxx_t val, MemOpIdx oi, 877 * uintptr_t ra) 878 */ 879static void * const qemu_st_helpers[MO_SIZE + 1] = { 880 [MO_8] = helper_ret_stb_mmu, 881#if HOST_BIG_ENDIAN 882 [MO_16] = helper_be_stw_mmu, 883 [MO_32] = helper_be_stl_mmu, 884 [MO_64] = helper_be_stq_mmu, 885#else 886 [MO_16] = helper_le_stw_mmu, 887 [MO_32] = helper_le_stl_mmu, 888 [MO_64] = helper_le_stq_mmu, 889#endif 890}; 891 892static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) 893{ 894 tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); 895 bool ok = reloc_jimm20(s->code_ptr - 1, target); 896 tcg_debug_assert(ok); 897} 898 899/* We have three temps, we might as well expose them. */ 900static const TCGLdstHelperParam ldst_helper_param = { 901 .ntmp = 3, .tmp = { TCG_REG_TMP0, TCG_REG_TMP1, TCG_REG_TMP2 } 902}; 903 904static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 905{ 906 MemOp opc = get_memop(l->oi); 907 908 /* resolve label address */ 909 if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 910 return false; 911 } 912 913 /* call load helper */ 914 tcg_out_ld_helper_args(s, l, &ldst_helper_param); 915 tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SSIZE], false); 916 tcg_out_ld_helper_ret(s, l, true, &ldst_helper_param); 917 918 tcg_out_goto(s, l->raddr); 919 return true; 920} 921 922static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 923{ 924 MemOp opc = get_memop(l->oi); 925 926 /* resolve label address */ 927 if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 928 return false; 929 } 930 931 /* call store helper */ 932 tcg_out_st_helper_args(s, l, &ldst_helper_param); 933 tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE], false); 934 935 tcg_out_goto(s, l->raddr); 936 return true; 937} 938#else 939static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) 940{ 941 /* resolve label address */ 942 if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 943 return false; 944 } 945 946 tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_A1, l->addrlo_reg); 947 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0); 948 949 /* tail call, with the return address back inline. */ 950 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (uintptr_t)l->raddr); 951 tcg_out_call_int(s, (const void *)(l->is_ld ? helper_unaligned_ld 952 : helper_unaligned_st), true); 953 return true; 954} 955 956static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 957{ 958 return tcg_out_fail_alignment(s, l); 959} 960 961static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 962{ 963 return tcg_out_fail_alignment(s, l); 964} 965#endif /* CONFIG_SOFTMMU */ 966 967/* 968 * For softmmu, perform the TLB load and compare. 969 * For useronly, perform any required alignment tests. 970 * In both cases, return a TCGLabelQemuLdst structure if the slow path 971 * is required and fill in @h with the host address for the fast path. 972 */ 973static TCGLabelQemuLdst *prepare_host_addr(TCGContext *s, TCGReg *pbase, 974 TCGReg addr_reg, MemOpIdx oi, 975 bool is_ld) 976{ 977 TCGLabelQemuLdst *ldst = NULL; 978 MemOp opc = get_memop(oi); 979 unsigned a_bits = get_alignment_bits(opc); 980 unsigned a_mask = (1u << a_bits) - 1; 981 982#ifdef CONFIG_SOFTMMU 983 unsigned s_bits = opc & MO_SIZE; 984 int mem_index = get_mmuidx(oi); 985 int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); 986 int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); 987 int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); 988 TCGReg mask_base = TCG_AREG0, table_base = TCG_AREG0; 989 tcg_target_long compare_mask; 990 991 ldst = new_ldst_label(s); 992 ldst->is_ld = is_ld; 993 ldst->oi = oi; 994 ldst->addrlo_reg = addr_reg; 995 996 QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); 997 QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); 998 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, mask_base, mask_ofs); 999 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, table_base, table_ofs); 1000 1001 tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addr_reg, 1002 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); 1003 tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); 1004 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); 1005 1006 /* Load the tlb comparator and the addend. */ 1007 tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP0, TCG_REG_TMP2, 1008 is_ld ? offsetof(CPUTLBEntry, addr_read) 1009 : offsetof(CPUTLBEntry, addr_write)); 1010 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, 1011 offsetof(CPUTLBEntry, addend)); 1012 1013 /* We don't support unaligned accesses. */ 1014 if (a_bits < s_bits) { 1015 a_bits = s_bits; 1016 } 1017 /* Clear the non-page, non-alignment bits from the address. */ 1018 compare_mask = (tcg_target_long)TARGET_PAGE_MASK | a_mask; 1019 if (compare_mask == sextreg(compare_mask, 0, 12)) { 1020 tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, compare_mask); 1021 } else { 1022 tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); 1023 tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addr_reg); 1024 } 1025 1026 /* Compare masked address with the TLB entry. */ 1027 ldst->label_ptr[0] = s->code_ptr; 1028 tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); 1029 1030 /* TLB Hit - translate address using addend. */ 1031 if (TARGET_LONG_BITS == 32) { 1032 tcg_out_ext32u(s, TCG_REG_TMP0, addr_reg); 1033 addr_reg = TCG_REG_TMP0; 1034 } 1035 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addr_reg); 1036 *pbase = TCG_REG_TMP0; 1037#else 1038 if (a_mask) { 1039 ldst = new_ldst_label(s); 1040 ldst->is_ld = is_ld; 1041 ldst->oi = oi; 1042 ldst->addrlo_reg = addr_reg; 1043 1044 /* We are expecting a_bits max 7, so we can always use andi. */ 1045 tcg_debug_assert(a_bits < 12); 1046 tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, a_mask); 1047 1048 ldst->label_ptr[0] = s->code_ptr; 1049 tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP1, TCG_REG_ZERO, 0); 1050 } 1051 1052 TCGReg base = addr_reg; 1053 if (TARGET_LONG_BITS == 32) { 1054 tcg_out_ext32u(s, TCG_REG_TMP0, base); 1055 base = TCG_REG_TMP0; 1056 } 1057 if (guest_base != 0) { 1058 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); 1059 base = TCG_REG_TMP0; 1060 } 1061 *pbase = base; 1062#endif 1063 1064 return ldst; 1065} 1066 1067static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg val, 1068 TCGReg base, MemOp opc, TCGType type) 1069{ 1070 /* Byte swapping is left to middle-end expansion. */ 1071 tcg_debug_assert((opc & MO_BSWAP) == 0); 1072 1073 switch (opc & (MO_SSIZE)) { 1074 case MO_UB: 1075 tcg_out_opc_imm(s, OPC_LBU, val, base, 0); 1076 break; 1077 case MO_SB: 1078 tcg_out_opc_imm(s, OPC_LB, val, base, 0); 1079 break; 1080 case MO_UW: 1081 tcg_out_opc_imm(s, OPC_LHU, val, base, 0); 1082 break; 1083 case MO_SW: 1084 tcg_out_opc_imm(s, OPC_LH, val, base, 0); 1085 break; 1086 case MO_UL: 1087 if (type == TCG_TYPE_I64) { 1088 tcg_out_opc_imm(s, OPC_LWU, val, base, 0); 1089 break; 1090 } 1091 /* FALLTHRU */ 1092 case MO_SL: 1093 tcg_out_opc_imm(s, OPC_LW, val, base, 0); 1094 break; 1095 case MO_UQ: 1096 tcg_out_opc_imm(s, OPC_LD, val, base, 0); 1097 break; 1098 default: 1099 g_assert_not_reached(); 1100 } 1101} 1102 1103static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, 1104 MemOpIdx oi, TCGType data_type) 1105{ 1106 TCGLabelQemuLdst *ldst; 1107 TCGReg base; 1108 1109 ldst = prepare_host_addr(s, &base, addr_reg, oi, true); 1110 tcg_out_qemu_ld_direct(s, data_reg, base, get_memop(oi), data_type); 1111 1112 if (ldst) { 1113 ldst->type = data_type; 1114 ldst->datalo_reg = data_reg; 1115 ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); 1116 } 1117} 1118 1119static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg val, 1120 TCGReg base, MemOp opc) 1121{ 1122 /* Byte swapping is left to middle-end expansion. */ 1123 tcg_debug_assert((opc & MO_BSWAP) == 0); 1124 1125 switch (opc & (MO_SSIZE)) { 1126 case MO_8: 1127 tcg_out_opc_store(s, OPC_SB, base, val, 0); 1128 break; 1129 case MO_16: 1130 tcg_out_opc_store(s, OPC_SH, base, val, 0); 1131 break; 1132 case MO_32: 1133 tcg_out_opc_store(s, OPC_SW, base, val, 0); 1134 break; 1135 case MO_64: 1136 tcg_out_opc_store(s, OPC_SD, base, val, 0); 1137 break; 1138 default: 1139 g_assert_not_reached(); 1140 } 1141} 1142 1143static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg, 1144 MemOpIdx oi, TCGType data_type) 1145{ 1146 TCGLabelQemuLdst *ldst; 1147 TCGReg base; 1148 1149 ldst = prepare_host_addr(s, &base, addr_reg, oi, false); 1150 tcg_out_qemu_st_direct(s, data_reg, base, get_memop(oi)); 1151 1152 if (ldst) { 1153 ldst->type = data_type; 1154 ldst->datalo_reg = data_reg; 1155 ldst->raddr = tcg_splitwx_to_rx(s->code_ptr); 1156 } 1157} 1158 1159static const tcg_insn_unit *tb_ret_addr; 1160 1161static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) 1162{ 1163 /* Reuse the zeroing that exists for goto_ptr. */ 1164 if (a0 == 0) { 1165 tcg_out_call_int(s, tcg_code_gen_epilogue, true); 1166 } else { 1167 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); 1168 tcg_out_call_int(s, tb_ret_addr, true); 1169 } 1170} 1171 1172static void tcg_out_goto_tb(TCGContext *s, int which) 1173{ 1174 /* Direct branch will be patched by tb_target_set_jmp_target. */ 1175 set_jmp_insn_offset(s, which); 1176 tcg_out32(s, OPC_JAL); 1177 1178 /* When branch is out of range, fall through to indirect. */ 1179 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, 1180 get_jmp_target_addr(s, which)); 1181 tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); 1182 set_jmp_reset_offset(s, which); 1183} 1184 1185void tb_target_set_jmp_target(const TranslationBlock *tb, int n, 1186 uintptr_t jmp_rx, uintptr_t jmp_rw) 1187{ 1188 uintptr_t addr = tb->jmp_target_addr[n]; 1189 ptrdiff_t offset = addr - jmp_rx; 1190 tcg_insn_unit insn; 1191 1192 /* Either directly branch, or fall through to indirect branch. */ 1193 if (offset == sextreg(offset, 0, 20)) { 1194 insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset); 1195 } else { 1196 insn = OPC_NOP; 1197 } 1198 qatomic_set((uint32_t *)jmp_rw, insn); 1199 flush_idcache_range(jmp_rx, jmp_rw, 4); 1200} 1201 1202static void tcg_out_op(TCGContext *s, TCGOpcode opc, 1203 const TCGArg args[TCG_MAX_OP_ARGS], 1204 const int const_args[TCG_MAX_OP_ARGS]) 1205{ 1206 TCGArg a0 = args[0]; 1207 TCGArg a1 = args[1]; 1208 TCGArg a2 = args[2]; 1209 int c2 = const_args[2]; 1210 1211 switch (opc) { 1212 case INDEX_op_goto_ptr: 1213 tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0); 1214 break; 1215 1216 case INDEX_op_br: 1217 tcg_out_reloc(s, s->code_ptr, R_RISCV_JAL, arg_label(a0), 0); 1218 tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); 1219 break; 1220 1221 case INDEX_op_ld8u_i32: 1222 case INDEX_op_ld8u_i64: 1223 tcg_out_ldst(s, OPC_LBU, a0, a1, a2); 1224 break; 1225 case INDEX_op_ld8s_i32: 1226 case INDEX_op_ld8s_i64: 1227 tcg_out_ldst(s, OPC_LB, a0, a1, a2); 1228 break; 1229 case INDEX_op_ld16u_i32: 1230 case INDEX_op_ld16u_i64: 1231 tcg_out_ldst(s, OPC_LHU, a0, a1, a2); 1232 break; 1233 case INDEX_op_ld16s_i32: 1234 case INDEX_op_ld16s_i64: 1235 tcg_out_ldst(s, OPC_LH, a0, a1, a2); 1236 break; 1237 case INDEX_op_ld32u_i64: 1238 tcg_out_ldst(s, OPC_LWU, a0, a1, a2); 1239 break; 1240 case INDEX_op_ld_i32: 1241 case INDEX_op_ld32s_i64: 1242 tcg_out_ldst(s, OPC_LW, a0, a1, a2); 1243 break; 1244 case INDEX_op_ld_i64: 1245 tcg_out_ldst(s, OPC_LD, a0, a1, a2); 1246 break; 1247 1248 case INDEX_op_st8_i32: 1249 case INDEX_op_st8_i64: 1250 tcg_out_ldst(s, OPC_SB, a0, a1, a2); 1251 break; 1252 case INDEX_op_st16_i32: 1253 case INDEX_op_st16_i64: 1254 tcg_out_ldst(s, OPC_SH, a0, a1, a2); 1255 break; 1256 case INDEX_op_st_i32: 1257 case INDEX_op_st32_i64: 1258 tcg_out_ldst(s, OPC_SW, a0, a1, a2); 1259 break; 1260 case INDEX_op_st_i64: 1261 tcg_out_ldst(s, OPC_SD, a0, a1, a2); 1262 break; 1263 1264 case INDEX_op_add_i32: 1265 if (c2) { 1266 tcg_out_opc_imm(s, OPC_ADDIW, a0, a1, a2); 1267 } else { 1268 tcg_out_opc_reg(s, OPC_ADDW, a0, a1, a2); 1269 } 1270 break; 1271 case INDEX_op_add_i64: 1272 if (c2) { 1273 tcg_out_opc_imm(s, OPC_ADDI, a0, a1, a2); 1274 } else { 1275 tcg_out_opc_reg(s, OPC_ADD, a0, a1, a2); 1276 } 1277 break; 1278 1279 case INDEX_op_sub_i32: 1280 if (c2) { 1281 tcg_out_opc_imm(s, OPC_ADDIW, a0, a1, -a2); 1282 } else { 1283 tcg_out_opc_reg(s, OPC_SUBW, a0, a1, a2); 1284 } 1285 break; 1286 case INDEX_op_sub_i64: 1287 if (c2) { 1288 tcg_out_opc_imm(s, OPC_ADDI, a0, a1, -a2); 1289 } else { 1290 tcg_out_opc_reg(s, OPC_SUB, a0, a1, a2); 1291 } 1292 break; 1293 1294 case INDEX_op_and_i32: 1295 case INDEX_op_and_i64: 1296 if (c2) { 1297 tcg_out_opc_imm(s, OPC_ANDI, a0, a1, a2); 1298 } else { 1299 tcg_out_opc_reg(s, OPC_AND, a0, a1, a2); 1300 } 1301 break; 1302 1303 case INDEX_op_or_i32: 1304 case INDEX_op_or_i64: 1305 if (c2) { 1306 tcg_out_opc_imm(s, OPC_ORI, a0, a1, a2); 1307 } else { 1308 tcg_out_opc_reg(s, OPC_OR, a0, a1, a2); 1309 } 1310 break; 1311 1312 case INDEX_op_xor_i32: 1313 case INDEX_op_xor_i64: 1314 if (c2) { 1315 tcg_out_opc_imm(s, OPC_XORI, a0, a1, a2); 1316 } else { 1317 tcg_out_opc_reg(s, OPC_XOR, a0, a1, a2); 1318 } 1319 break; 1320 1321 case INDEX_op_not_i32: 1322 case INDEX_op_not_i64: 1323 tcg_out_opc_imm(s, OPC_XORI, a0, a1, -1); 1324 break; 1325 1326 case INDEX_op_neg_i32: 1327 tcg_out_opc_reg(s, OPC_SUBW, a0, TCG_REG_ZERO, a1); 1328 break; 1329 case INDEX_op_neg_i64: 1330 tcg_out_opc_reg(s, OPC_SUB, a0, TCG_REG_ZERO, a1); 1331 break; 1332 1333 case INDEX_op_mul_i32: 1334 tcg_out_opc_reg(s, OPC_MULW, a0, a1, a2); 1335 break; 1336 case INDEX_op_mul_i64: 1337 tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2); 1338 break; 1339 1340 case INDEX_op_div_i32: 1341 tcg_out_opc_reg(s, OPC_DIVW, a0, a1, a2); 1342 break; 1343 case INDEX_op_div_i64: 1344 tcg_out_opc_reg(s, OPC_DIV, a0, a1, a2); 1345 break; 1346 1347 case INDEX_op_divu_i32: 1348 tcg_out_opc_reg(s, OPC_DIVUW, a0, a1, a2); 1349 break; 1350 case INDEX_op_divu_i64: 1351 tcg_out_opc_reg(s, OPC_DIVU, a0, a1, a2); 1352 break; 1353 1354 case INDEX_op_rem_i32: 1355 tcg_out_opc_reg(s, OPC_REMW, a0, a1, a2); 1356 break; 1357 case INDEX_op_rem_i64: 1358 tcg_out_opc_reg(s, OPC_REM, a0, a1, a2); 1359 break; 1360 1361 case INDEX_op_remu_i32: 1362 tcg_out_opc_reg(s, OPC_REMUW, a0, a1, a2); 1363 break; 1364 case INDEX_op_remu_i64: 1365 tcg_out_opc_reg(s, OPC_REMU, a0, a1, a2); 1366 break; 1367 1368 case INDEX_op_shl_i32: 1369 if (c2) { 1370 tcg_out_opc_imm(s, OPC_SLLIW, a0, a1, a2 & 0x1f); 1371 } else { 1372 tcg_out_opc_reg(s, OPC_SLLW, a0, a1, a2); 1373 } 1374 break; 1375 case INDEX_op_shl_i64: 1376 if (c2) { 1377 tcg_out_opc_imm(s, OPC_SLLI, a0, a1, a2 & 0x3f); 1378 } else { 1379 tcg_out_opc_reg(s, OPC_SLL, a0, a1, a2); 1380 } 1381 break; 1382 1383 case INDEX_op_shr_i32: 1384 if (c2) { 1385 tcg_out_opc_imm(s, OPC_SRLIW, a0, a1, a2 & 0x1f); 1386 } else { 1387 tcg_out_opc_reg(s, OPC_SRLW, a0, a1, a2); 1388 } 1389 break; 1390 case INDEX_op_shr_i64: 1391 if (c2) { 1392 tcg_out_opc_imm(s, OPC_SRLI, a0, a1, a2 & 0x3f); 1393 } else { 1394 tcg_out_opc_reg(s, OPC_SRL, a0, a1, a2); 1395 } 1396 break; 1397 1398 case INDEX_op_sar_i32: 1399 if (c2) { 1400 tcg_out_opc_imm(s, OPC_SRAIW, a0, a1, a2 & 0x1f); 1401 } else { 1402 tcg_out_opc_reg(s, OPC_SRAW, a0, a1, a2); 1403 } 1404 break; 1405 case INDEX_op_sar_i64: 1406 if (c2) { 1407 tcg_out_opc_imm(s, OPC_SRAI, a0, a1, a2 & 0x3f); 1408 } else { 1409 tcg_out_opc_reg(s, OPC_SRA, a0, a1, a2); 1410 } 1411 break; 1412 1413 case INDEX_op_add2_i32: 1414 tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], 1415 const_args[4], const_args[5], false, true); 1416 break; 1417 case INDEX_op_add2_i64: 1418 tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], 1419 const_args[4], const_args[5], false, false); 1420 break; 1421 case INDEX_op_sub2_i32: 1422 tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], 1423 const_args[4], const_args[5], true, true); 1424 break; 1425 case INDEX_op_sub2_i64: 1426 tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], 1427 const_args[4], const_args[5], true, false); 1428 break; 1429 1430 case INDEX_op_brcond_i32: 1431 case INDEX_op_brcond_i64: 1432 tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); 1433 break; 1434 1435 case INDEX_op_setcond_i32: 1436 case INDEX_op_setcond_i64: 1437 tcg_out_setcond(s, args[3], a0, a1, a2); 1438 break; 1439 1440 case INDEX_op_qemu_ld_i32: 1441 tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I32); 1442 break; 1443 case INDEX_op_qemu_ld_i64: 1444 tcg_out_qemu_ld(s, a0, a1, a2, TCG_TYPE_I64); 1445 break; 1446 case INDEX_op_qemu_st_i32: 1447 tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I32); 1448 break; 1449 case INDEX_op_qemu_st_i64: 1450 tcg_out_qemu_st(s, a0, a1, a2, TCG_TYPE_I64); 1451 break; 1452 1453 case INDEX_op_extrh_i64_i32: 1454 tcg_out_opc_imm(s, OPC_SRAI, a0, a1, 32); 1455 break; 1456 1457 case INDEX_op_mulsh_i32: 1458 case INDEX_op_mulsh_i64: 1459 tcg_out_opc_reg(s, OPC_MULH, a0, a1, a2); 1460 break; 1461 1462 case INDEX_op_muluh_i32: 1463 case INDEX_op_muluh_i64: 1464 tcg_out_opc_reg(s, OPC_MULHU, a0, a1, a2); 1465 break; 1466 1467 case INDEX_op_mb: 1468 tcg_out_mb(s, a0); 1469 break; 1470 1471 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ 1472 case INDEX_op_mov_i64: 1473 case INDEX_op_call: /* Always emitted via tcg_out_call. */ 1474 case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ 1475 case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ 1476 case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ 1477 case INDEX_op_ext8s_i64: 1478 case INDEX_op_ext8u_i32: 1479 case INDEX_op_ext8u_i64: 1480 case INDEX_op_ext16s_i32: 1481 case INDEX_op_ext16s_i64: 1482 case INDEX_op_ext16u_i32: 1483 case INDEX_op_ext16u_i64: 1484 case INDEX_op_ext32s_i64: 1485 case INDEX_op_ext32u_i64: 1486 case INDEX_op_ext_i32_i64: 1487 case INDEX_op_extu_i32_i64: 1488 case INDEX_op_extrl_i64_i32: 1489 default: 1490 g_assert_not_reached(); 1491 } 1492} 1493 1494static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) 1495{ 1496 switch (op) { 1497 case INDEX_op_goto_ptr: 1498 return C_O0_I1(r); 1499 1500 case INDEX_op_ld8u_i32: 1501 case INDEX_op_ld8s_i32: 1502 case INDEX_op_ld16u_i32: 1503 case INDEX_op_ld16s_i32: 1504 case INDEX_op_ld_i32: 1505 case INDEX_op_not_i32: 1506 case INDEX_op_neg_i32: 1507 case INDEX_op_ld8u_i64: 1508 case INDEX_op_ld8s_i64: 1509 case INDEX_op_ld16u_i64: 1510 case INDEX_op_ld16s_i64: 1511 case INDEX_op_ld32s_i64: 1512 case INDEX_op_ld32u_i64: 1513 case INDEX_op_ld_i64: 1514 case INDEX_op_not_i64: 1515 case INDEX_op_neg_i64: 1516 case INDEX_op_ext8u_i32: 1517 case INDEX_op_ext8u_i64: 1518 case INDEX_op_ext16u_i32: 1519 case INDEX_op_ext16u_i64: 1520 case INDEX_op_ext32u_i64: 1521 case INDEX_op_extu_i32_i64: 1522 case INDEX_op_ext8s_i32: 1523 case INDEX_op_ext8s_i64: 1524 case INDEX_op_ext16s_i32: 1525 case INDEX_op_ext16s_i64: 1526 case INDEX_op_ext32s_i64: 1527 case INDEX_op_extrl_i64_i32: 1528 case INDEX_op_extrh_i64_i32: 1529 case INDEX_op_ext_i32_i64: 1530 return C_O1_I1(r, r); 1531 1532 case INDEX_op_st8_i32: 1533 case INDEX_op_st16_i32: 1534 case INDEX_op_st_i32: 1535 case INDEX_op_st8_i64: 1536 case INDEX_op_st16_i64: 1537 case INDEX_op_st32_i64: 1538 case INDEX_op_st_i64: 1539 return C_O0_I2(rZ, r); 1540 1541 case INDEX_op_add_i32: 1542 case INDEX_op_and_i32: 1543 case INDEX_op_or_i32: 1544 case INDEX_op_xor_i32: 1545 case INDEX_op_add_i64: 1546 case INDEX_op_and_i64: 1547 case INDEX_op_or_i64: 1548 case INDEX_op_xor_i64: 1549 return C_O1_I2(r, r, rI); 1550 1551 case INDEX_op_sub_i32: 1552 case INDEX_op_sub_i64: 1553 return C_O1_I2(r, rZ, rN); 1554 1555 case INDEX_op_mul_i32: 1556 case INDEX_op_mulsh_i32: 1557 case INDEX_op_muluh_i32: 1558 case INDEX_op_div_i32: 1559 case INDEX_op_divu_i32: 1560 case INDEX_op_rem_i32: 1561 case INDEX_op_remu_i32: 1562 case INDEX_op_setcond_i32: 1563 case INDEX_op_mul_i64: 1564 case INDEX_op_mulsh_i64: 1565 case INDEX_op_muluh_i64: 1566 case INDEX_op_div_i64: 1567 case INDEX_op_divu_i64: 1568 case INDEX_op_rem_i64: 1569 case INDEX_op_remu_i64: 1570 case INDEX_op_setcond_i64: 1571 return C_O1_I2(r, rZ, rZ); 1572 1573 case INDEX_op_shl_i32: 1574 case INDEX_op_shr_i32: 1575 case INDEX_op_sar_i32: 1576 case INDEX_op_shl_i64: 1577 case INDEX_op_shr_i64: 1578 case INDEX_op_sar_i64: 1579 return C_O1_I2(r, r, ri); 1580 1581 case INDEX_op_brcond_i32: 1582 case INDEX_op_brcond_i64: 1583 return C_O0_I2(rZ, rZ); 1584 1585 case INDEX_op_add2_i32: 1586 case INDEX_op_add2_i64: 1587 case INDEX_op_sub2_i32: 1588 case INDEX_op_sub2_i64: 1589 return C_O2_I4(r, r, rZ, rZ, rM, rM); 1590 1591 case INDEX_op_qemu_ld_i32: 1592 case INDEX_op_qemu_ld_i64: 1593 return C_O1_I1(r, r); 1594 case INDEX_op_qemu_st_i32: 1595 case INDEX_op_qemu_st_i64: 1596 return C_O0_I2(rZ, r); 1597 1598 default: 1599 g_assert_not_reached(); 1600 } 1601} 1602 1603static const int tcg_target_callee_save_regs[] = { 1604 TCG_REG_S0, /* used for the global env (TCG_AREG0) */ 1605 TCG_REG_S1, 1606 TCG_REG_S2, 1607 TCG_REG_S3, 1608 TCG_REG_S4, 1609 TCG_REG_S5, 1610 TCG_REG_S6, 1611 TCG_REG_S7, 1612 TCG_REG_S8, 1613 TCG_REG_S9, 1614 TCG_REG_S10, 1615 TCG_REG_S11, 1616 TCG_REG_RA, /* should be last for ABI compliance */ 1617}; 1618 1619/* Stack frame parameters. */ 1620#define REG_SIZE (TCG_TARGET_REG_BITS / 8) 1621#define SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * REG_SIZE) 1622#define TEMP_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long)) 1623#define FRAME_SIZE ((TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE + SAVE_SIZE \ 1624 + TCG_TARGET_STACK_ALIGN - 1) \ 1625 & -TCG_TARGET_STACK_ALIGN) 1626#define SAVE_OFS (TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE) 1627 1628/* We're expecting to be able to use an immediate for frame allocation. */ 1629QEMU_BUILD_BUG_ON(FRAME_SIZE > 0x7ff); 1630 1631/* Generate global QEMU prologue and epilogue code */ 1632static void tcg_target_qemu_prologue(TCGContext *s) 1633{ 1634 int i; 1635 1636 tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE, TEMP_SIZE); 1637 1638 /* TB prologue */ 1639 tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_SP, TCG_REG_SP, -FRAME_SIZE); 1640 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { 1641 tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], 1642 TCG_REG_SP, SAVE_OFS + i * REG_SIZE); 1643 } 1644 1645#if !defined(CONFIG_SOFTMMU) 1646 tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base); 1647 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); 1648#endif 1649 1650 /* Call generated code */ 1651 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); 1652 tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0); 1653 1654 /* Return path for goto_ptr. Set return value to 0 */ 1655 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); 1656 tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO); 1657 1658 /* TB epilogue */ 1659 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); 1660 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { 1661 tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], 1662 TCG_REG_SP, SAVE_OFS + i * REG_SIZE); 1663 } 1664 1665 tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_SP, TCG_REG_SP, FRAME_SIZE); 1666 tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_RA, 0); 1667} 1668 1669static void tcg_target_init(TCGContext *s) 1670{ 1671 tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; 1672 tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; 1673 1674 tcg_target_call_clobber_regs = -1u; 1675 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0); 1676 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S1); 1677 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S2); 1678 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S3); 1679 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S4); 1680 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S5); 1681 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S6); 1682 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S7); 1683 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S8); 1684 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S9); 1685 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S10); 1686 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S11); 1687 1688 s->reserved_regs = 0; 1689 tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO); 1690 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP0); 1691 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); 1692 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP2); 1693 tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP); 1694 tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP); 1695 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TP); 1696} 1697 1698typedef struct { 1699 DebugFrameHeader h; 1700 uint8_t fde_def_cfa[4]; 1701 uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2]; 1702} DebugFrame; 1703 1704#define ELF_HOST_MACHINE EM_RISCV 1705 1706static const DebugFrame debug_frame = { 1707 .h.cie.len = sizeof(DebugFrameCIE) - 4, /* length after .len member */ 1708 .h.cie.id = -1, 1709 .h.cie.version = 1, 1710 .h.cie.code_align = 1, 1711 .h.cie.data_align = -(TCG_TARGET_REG_BITS / 8) & 0x7f, /* sleb128 */ 1712 .h.cie.return_column = TCG_REG_RA, 1713 1714 /* Total FDE size does not include the "len" member. */ 1715 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), 1716 1717 .fde_def_cfa = { 1718 12, TCG_REG_SP, /* DW_CFA_def_cfa sp, ... */ 1719 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ 1720 (FRAME_SIZE >> 7) 1721 }, 1722 .fde_reg_ofs = { 1723 0x80 + 9, 12, /* DW_CFA_offset, s1, -96 */ 1724 0x80 + 18, 11, /* DW_CFA_offset, s2, -88 */ 1725 0x80 + 19, 10, /* DW_CFA_offset, s3, -80 */ 1726 0x80 + 20, 9, /* DW_CFA_offset, s4, -72 */ 1727 0x80 + 21, 8, /* DW_CFA_offset, s5, -64 */ 1728 0x80 + 22, 7, /* DW_CFA_offset, s6, -56 */ 1729 0x80 + 23, 6, /* DW_CFA_offset, s7, -48 */ 1730 0x80 + 24, 5, /* DW_CFA_offset, s8, -40 */ 1731 0x80 + 25, 4, /* DW_CFA_offset, s9, -32 */ 1732 0x80 + 26, 3, /* DW_CFA_offset, s10, -24 */ 1733 0x80 + 27, 2, /* DW_CFA_offset, s11, -16 */ 1734 0x80 + 1 , 1, /* DW_CFA_offset, ra, -8 */ 1735 } 1736}; 1737 1738void tcg_register_jit(const void *buf, size_t buf_size) 1739{ 1740 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); 1741} 1742