1/* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2018 SiFive, Inc 5 * Copyright (c) 2008-2009 Arnaud Patard <arnaud.patard@rtp-net.org> 6 * Copyright (c) 2009 Aurelien Jarno <aurelien@aurel32.net> 7 * Copyright (c) 2008 Fabrice Bellard 8 * 9 * Based on i386/tcg-target.c and mips/tcg-target.c 10 * 11 * Permission is hereby granted, free of charge, to any person obtaining a copy 12 * of this software and associated documentation files (the "Software"), to deal 13 * in the Software without restriction, including without limitation the rights 14 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 15 * copies of the Software, and to permit persons to whom the Software is 16 * furnished to do so, subject to the following conditions: 17 * 18 * The above copyright notice and this permission notice shall be included in 19 * all copies or substantial portions of the Software. 20 * 21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 26 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 27 * THE SOFTWARE. 28 */ 29 30#include "../tcg-ldst.c.inc" 31#include "../tcg-pool.c.inc" 32 33#ifdef CONFIG_DEBUG_TCG 34static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = { 35 "zero", 36 "ra", 37 "sp", 38 "gp", 39 "tp", 40 "t0", 41 "t1", 42 "t2", 43 "s0", 44 "s1", 45 "a0", 46 "a1", 47 "a2", 48 "a3", 49 "a4", 50 "a5", 51 "a6", 52 "a7", 53 "s2", 54 "s3", 55 "s4", 56 "s5", 57 "s6", 58 "s7", 59 "s8", 60 "s9", 61 "s10", 62 "s11", 63 "t3", 64 "t4", 65 "t5", 66 "t6" 67}; 68#endif 69 70static const int tcg_target_reg_alloc_order[] = { 71 /* Call saved registers */ 72 /* TCG_REG_S0 reservered for TCG_AREG0 */ 73 TCG_REG_S1, 74 TCG_REG_S2, 75 TCG_REG_S3, 76 TCG_REG_S4, 77 TCG_REG_S5, 78 TCG_REG_S6, 79 TCG_REG_S7, 80 TCG_REG_S8, 81 TCG_REG_S9, 82 TCG_REG_S10, 83 TCG_REG_S11, 84 85 /* Call clobbered registers */ 86 TCG_REG_T0, 87 TCG_REG_T1, 88 TCG_REG_T2, 89 TCG_REG_T3, 90 TCG_REG_T4, 91 TCG_REG_T5, 92 TCG_REG_T6, 93 94 /* Argument registers */ 95 TCG_REG_A0, 96 TCG_REG_A1, 97 TCG_REG_A2, 98 TCG_REG_A3, 99 TCG_REG_A4, 100 TCG_REG_A5, 101 TCG_REG_A6, 102 TCG_REG_A7, 103}; 104 105static const int tcg_target_call_iarg_regs[] = { 106 TCG_REG_A0, 107 TCG_REG_A1, 108 TCG_REG_A2, 109 TCG_REG_A3, 110 TCG_REG_A4, 111 TCG_REG_A5, 112 TCG_REG_A6, 113 TCG_REG_A7, 114}; 115 116static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) 117{ 118 tcg_debug_assert(kind == TCG_CALL_RET_NORMAL); 119 tcg_debug_assert(slot >= 0 && slot <= 1); 120 return TCG_REG_A0 + slot; 121} 122 123#define TCG_CT_CONST_ZERO 0x100 124#define TCG_CT_CONST_S12 0x200 125#define TCG_CT_CONST_N12 0x400 126#define TCG_CT_CONST_M12 0x800 127 128#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32) 129/* 130 * For softmmu, we need to avoid conflicts with the first 5 131 * argument registers to call the helper. Some of these are 132 * also used for the tlb lookup. 133 */ 134#ifdef CONFIG_SOFTMMU 135#define SOFTMMU_RESERVE_REGS MAKE_64BIT_MASK(TCG_REG_A0, 5) 136#else 137#define SOFTMMU_RESERVE_REGS 0 138#endif 139 140 141static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len) 142{ 143 if (TCG_TARGET_REG_BITS == 32) { 144 return sextract32(val, pos, len); 145 } else { 146 return sextract64(val, pos, len); 147 } 148} 149 150/* test if a constant matches the constraint */ 151static bool tcg_target_const_match(int64_t val, TCGType type, int ct) 152{ 153 if (ct & TCG_CT_CONST) { 154 return 1; 155 } 156 if ((ct & TCG_CT_CONST_ZERO) && val == 0) { 157 return 1; 158 } 159 /* 160 * Sign extended from 12 bits: [-0x800, 0x7ff]. 161 * Used for most arithmetic, as this is the isa field. 162 */ 163 if ((ct & TCG_CT_CONST_S12) && val >= -0x800 && val <= 0x7ff) { 164 return 1; 165 } 166 /* 167 * Sign extended from 12 bits, negated: [-0x7ff, 0x800]. 168 * Used for subtraction, where a constant must be handled by ADDI. 169 */ 170 if ((ct & TCG_CT_CONST_N12) && val >= -0x7ff && val <= 0x800) { 171 return 1; 172 } 173 /* 174 * Sign extended from 12 bits, +/- matching: [-0x7ff, 0x7ff]. 175 * Used by addsub2, which may need the negative operation, 176 * and requires the modified constant to be representable. 177 */ 178 if ((ct & TCG_CT_CONST_M12) && val >= -0x7ff && val <= 0x7ff) { 179 return 1; 180 } 181 return 0; 182} 183 184/* 185 * RISC-V Base ISA opcodes (IM) 186 */ 187 188typedef enum { 189 OPC_ADD = 0x33, 190 OPC_ADDI = 0x13, 191 OPC_AND = 0x7033, 192 OPC_ANDI = 0x7013, 193 OPC_AUIPC = 0x17, 194 OPC_BEQ = 0x63, 195 OPC_BGE = 0x5063, 196 OPC_BGEU = 0x7063, 197 OPC_BLT = 0x4063, 198 OPC_BLTU = 0x6063, 199 OPC_BNE = 0x1063, 200 OPC_DIV = 0x2004033, 201 OPC_DIVU = 0x2005033, 202 OPC_JAL = 0x6f, 203 OPC_JALR = 0x67, 204 OPC_LB = 0x3, 205 OPC_LBU = 0x4003, 206 OPC_LD = 0x3003, 207 OPC_LH = 0x1003, 208 OPC_LHU = 0x5003, 209 OPC_LUI = 0x37, 210 OPC_LW = 0x2003, 211 OPC_LWU = 0x6003, 212 OPC_MUL = 0x2000033, 213 OPC_MULH = 0x2001033, 214 OPC_MULHSU = 0x2002033, 215 OPC_MULHU = 0x2003033, 216 OPC_OR = 0x6033, 217 OPC_ORI = 0x6013, 218 OPC_REM = 0x2006033, 219 OPC_REMU = 0x2007033, 220 OPC_SB = 0x23, 221 OPC_SD = 0x3023, 222 OPC_SH = 0x1023, 223 OPC_SLL = 0x1033, 224 OPC_SLLI = 0x1013, 225 OPC_SLT = 0x2033, 226 OPC_SLTI = 0x2013, 227 OPC_SLTIU = 0x3013, 228 OPC_SLTU = 0x3033, 229 OPC_SRA = 0x40005033, 230 OPC_SRAI = 0x40005013, 231 OPC_SRL = 0x5033, 232 OPC_SRLI = 0x5013, 233 OPC_SUB = 0x40000033, 234 OPC_SW = 0x2023, 235 OPC_XOR = 0x4033, 236 OPC_XORI = 0x4013, 237 238#if TCG_TARGET_REG_BITS == 64 239 OPC_ADDIW = 0x1b, 240 OPC_ADDW = 0x3b, 241 OPC_DIVUW = 0x200503b, 242 OPC_DIVW = 0x200403b, 243 OPC_MULW = 0x200003b, 244 OPC_REMUW = 0x200703b, 245 OPC_REMW = 0x200603b, 246 OPC_SLLIW = 0x101b, 247 OPC_SLLW = 0x103b, 248 OPC_SRAIW = 0x4000501b, 249 OPC_SRAW = 0x4000503b, 250 OPC_SRLIW = 0x501b, 251 OPC_SRLW = 0x503b, 252 OPC_SUBW = 0x4000003b, 253#else 254 /* Simplify code throughout by defining aliases for RV32. */ 255 OPC_ADDIW = OPC_ADDI, 256 OPC_ADDW = OPC_ADD, 257 OPC_DIVUW = OPC_DIVU, 258 OPC_DIVW = OPC_DIV, 259 OPC_MULW = OPC_MUL, 260 OPC_REMUW = OPC_REMU, 261 OPC_REMW = OPC_REM, 262 OPC_SLLIW = OPC_SLLI, 263 OPC_SLLW = OPC_SLL, 264 OPC_SRAIW = OPC_SRAI, 265 OPC_SRAW = OPC_SRA, 266 OPC_SRLIW = OPC_SRLI, 267 OPC_SRLW = OPC_SRL, 268 OPC_SUBW = OPC_SUB, 269#endif 270 271 OPC_FENCE = 0x0000000f, 272 OPC_NOP = OPC_ADDI, /* nop = addi r0,r0,0 */ 273} RISCVInsn; 274 275/* 276 * RISC-V immediate and instruction encoders (excludes 16-bit RVC) 277 */ 278 279/* Type-R */ 280 281static int32_t encode_r(RISCVInsn opc, TCGReg rd, TCGReg rs1, TCGReg rs2) 282{ 283 return opc | (rd & 0x1f) << 7 | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20; 284} 285 286/* Type-I */ 287 288static int32_t encode_imm12(uint32_t imm) 289{ 290 return (imm & 0xfff) << 20; 291} 292 293static int32_t encode_i(RISCVInsn opc, TCGReg rd, TCGReg rs1, uint32_t imm) 294{ 295 return opc | (rd & 0x1f) << 7 | (rs1 & 0x1f) << 15 | encode_imm12(imm); 296} 297 298/* Type-S */ 299 300static int32_t encode_simm12(uint32_t imm) 301{ 302 int32_t ret = 0; 303 304 ret |= (imm & 0xFE0) << 20; 305 ret |= (imm & 0x1F) << 7; 306 307 return ret; 308} 309 310static int32_t encode_s(RISCVInsn opc, TCGReg rs1, TCGReg rs2, uint32_t imm) 311{ 312 return opc | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20 | encode_simm12(imm); 313} 314 315/* Type-SB */ 316 317static int32_t encode_sbimm12(uint32_t imm) 318{ 319 int32_t ret = 0; 320 321 ret |= (imm & 0x1000) << 19; 322 ret |= (imm & 0x7e0) << 20; 323 ret |= (imm & 0x1e) << 7; 324 ret |= (imm & 0x800) >> 4; 325 326 return ret; 327} 328 329static int32_t encode_sb(RISCVInsn opc, TCGReg rs1, TCGReg rs2, uint32_t imm) 330{ 331 return opc | (rs1 & 0x1f) << 15 | (rs2 & 0x1f) << 20 | encode_sbimm12(imm); 332} 333 334/* Type-U */ 335 336static int32_t encode_uimm20(uint32_t imm) 337{ 338 return imm & 0xfffff000; 339} 340 341static int32_t encode_u(RISCVInsn opc, TCGReg rd, uint32_t imm) 342{ 343 return opc | (rd & 0x1f) << 7 | encode_uimm20(imm); 344} 345 346/* Type-UJ */ 347 348static int32_t encode_ujimm20(uint32_t imm) 349{ 350 int32_t ret = 0; 351 352 ret |= (imm & 0x0007fe) << (21 - 1); 353 ret |= (imm & 0x000800) << (20 - 11); 354 ret |= (imm & 0x0ff000) << (12 - 12); 355 ret |= (imm & 0x100000) << (31 - 20); 356 357 return ret; 358} 359 360static int32_t encode_uj(RISCVInsn opc, TCGReg rd, uint32_t imm) 361{ 362 return opc | (rd & 0x1f) << 7 | encode_ujimm20(imm); 363} 364 365/* 366 * RISC-V instruction emitters 367 */ 368 369static void tcg_out_opc_reg(TCGContext *s, RISCVInsn opc, 370 TCGReg rd, TCGReg rs1, TCGReg rs2) 371{ 372 tcg_out32(s, encode_r(opc, rd, rs1, rs2)); 373} 374 375static void tcg_out_opc_imm(TCGContext *s, RISCVInsn opc, 376 TCGReg rd, TCGReg rs1, TCGArg imm) 377{ 378 tcg_out32(s, encode_i(opc, rd, rs1, imm)); 379} 380 381static void tcg_out_opc_store(TCGContext *s, RISCVInsn opc, 382 TCGReg rs1, TCGReg rs2, uint32_t imm) 383{ 384 tcg_out32(s, encode_s(opc, rs1, rs2, imm)); 385} 386 387static void tcg_out_opc_branch(TCGContext *s, RISCVInsn opc, 388 TCGReg rs1, TCGReg rs2, uint32_t imm) 389{ 390 tcg_out32(s, encode_sb(opc, rs1, rs2, imm)); 391} 392 393static void tcg_out_opc_upper(TCGContext *s, RISCVInsn opc, 394 TCGReg rd, uint32_t imm) 395{ 396 tcg_out32(s, encode_u(opc, rd, imm)); 397} 398 399static void tcg_out_opc_jump(TCGContext *s, RISCVInsn opc, 400 TCGReg rd, uint32_t imm) 401{ 402 tcg_out32(s, encode_uj(opc, rd, imm)); 403} 404 405static void tcg_out_nop_fill(tcg_insn_unit *p, int count) 406{ 407 int i; 408 for (i = 0; i < count; ++i) { 409 p[i] = OPC_NOP; 410 } 411} 412 413/* 414 * Relocations 415 */ 416 417static bool reloc_sbimm12(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 418{ 419 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 420 intptr_t offset = (intptr_t)target - (intptr_t)src_rx; 421 422 tcg_debug_assert((offset & 1) == 0); 423 if (offset == sextreg(offset, 0, 12)) { 424 *src_rw |= encode_sbimm12(offset); 425 return true; 426 } 427 428 return false; 429} 430 431static bool reloc_jimm20(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 432{ 433 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 434 intptr_t offset = (intptr_t)target - (intptr_t)src_rx; 435 436 tcg_debug_assert((offset & 1) == 0); 437 if (offset == sextreg(offset, 0, 20)) { 438 *src_rw |= encode_ujimm20(offset); 439 return true; 440 } 441 442 return false; 443} 444 445static bool reloc_call(tcg_insn_unit *src_rw, const tcg_insn_unit *target) 446{ 447 const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw); 448 intptr_t offset = (intptr_t)target - (intptr_t)src_rx; 449 int32_t lo = sextreg(offset, 0, 12); 450 int32_t hi = offset - lo; 451 452 if (offset == hi + lo) { 453 src_rw[0] |= encode_uimm20(hi); 454 src_rw[1] |= encode_imm12(lo); 455 return true; 456 } 457 458 return false; 459} 460 461static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 462 intptr_t value, intptr_t addend) 463{ 464 tcg_debug_assert(addend == 0); 465 switch (type) { 466 case R_RISCV_BRANCH: 467 return reloc_sbimm12(code_ptr, (tcg_insn_unit *)value); 468 case R_RISCV_JAL: 469 return reloc_jimm20(code_ptr, (tcg_insn_unit *)value); 470 case R_RISCV_CALL: 471 return reloc_call(code_ptr, (tcg_insn_unit *)value); 472 default: 473 g_assert_not_reached(); 474 } 475} 476 477/* 478 * TCG intrinsics 479 */ 480 481static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 482{ 483 if (ret == arg) { 484 return true; 485 } 486 switch (type) { 487 case TCG_TYPE_I32: 488 case TCG_TYPE_I64: 489 tcg_out_opc_imm(s, OPC_ADDI, ret, arg, 0); 490 break; 491 default: 492 g_assert_not_reached(); 493 } 494 return true; 495} 496 497static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd, 498 tcg_target_long val) 499{ 500 tcg_target_long lo, hi, tmp; 501 int shift, ret; 502 503 if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) { 504 val = (int32_t)val; 505 } 506 507 lo = sextreg(val, 0, 12); 508 if (val == lo) { 509 tcg_out_opc_imm(s, OPC_ADDI, rd, TCG_REG_ZERO, lo); 510 return; 511 } 512 513 hi = val - lo; 514 if (TCG_TARGET_REG_BITS == 32 || val == (int32_t)val) { 515 tcg_out_opc_upper(s, OPC_LUI, rd, hi); 516 if (lo != 0) { 517 tcg_out_opc_imm(s, OPC_ADDIW, rd, rd, lo); 518 } 519 return; 520 } 521 522 /* We can only be here if TCG_TARGET_REG_BITS != 32 */ 523 tmp = tcg_pcrel_diff(s, (void *)val); 524 if (tmp == (int32_t)tmp) { 525 tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); 526 tcg_out_opc_imm(s, OPC_ADDI, rd, rd, 0); 527 ret = reloc_call(s->code_ptr - 2, (const tcg_insn_unit *)val); 528 tcg_debug_assert(ret == true); 529 return; 530 } 531 532 /* Look for a single 20-bit section. */ 533 shift = ctz64(val); 534 tmp = val >> shift; 535 if (tmp == sextreg(tmp, 0, 20)) { 536 tcg_out_opc_upper(s, OPC_LUI, rd, tmp << 12); 537 if (shift > 12) { 538 tcg_out_opc_imm(s, OPC_SLLI, rd, rd, shift - 12); 539 } else { 540 tcg_out_opc_imm(s, OPC_SRAI, rd, rd, 12 - shift); 541 } 542 return; 543 } 544 545 /* Look for a few high zero bits, with lots of bits set in the middle. */ 546 shift = clz64(val); 547 tmp = val << shift; 548 if (tmp == sextreg(tmp, 12, 20) << 12) { 549 tcg_out_opc_upper(s, OPC_LUI, rd, tmp); 550 tcg_out_opc_imm(s, OPC_SRLI, rd, rd, shift); 551 return; 552 } else if (tmp == sextreg(tmp, 0, 12)) { 553 tcg_out_opc_imm(s, OPC_ADDI, rd, TCG_REG_ZERO, tmp); 554 tcg_out_opc_imm(s, OPC_SRLI, rd, rd, shift); 555 return; 556 } 557 558 /* Drop into the constant pool. */ 559 new_pool_label(s, val, R_RISCV_CALL, s->code_ptr, 0); 560 tcg_out_opc_upper(s, OPC_AUIPC, rd, 0); 561 tcg_out_opc_imm(s, OPC_LD, rd, rd, 0); 562} 563 564static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2) 565{ 566 return false; 567} 568 569static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs, 570 tcg_target_long imm) 571{ 572 /* This function is only used for passing structs by reference. */ 573 g_assert_not_reached(); 574} 575 576static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg) 577{ 578 tcg_out_opc_imm(s, OPC_ANDI, ret, arg, 0xff); 579} 580 581static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg) 582{ 583 tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); 584 tcg_out_opc_imm(s, OPC_SRLIW, ret, ret, 16); 585} 586 587static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg) 588{ 589 tcg_out_opc_imm(s, OPC_SLLI, ret, arg, 32); 590 tcg_out_opc_imm(s, OPC_SRLI, ret, ret, 32); 591} 592 593static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 594{ 595 tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 24); 596 tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 24); 597} 598 599static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg) 600{ 601 tcg_out_opc_imm(s, OPC_SLLIW, ret, arg, 16); 602 tcg_out_opc_imm(s, OPC_SRAIW, ret, ret, 16); 603} 604 605static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg) 606{ 607 tcg_out_opc_imm(s, OPC_ADDIW, ret, arg, 0); 608} 609 610static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) 611{ 612 if (ret != arg) { 613 tcg_out_ext32s(s, ret, arg); 614 } 615} 616 617static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg) 618{ 619 tcg_out_ext32u(s, ret, arg); 620} 621 622static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg) 623{ 624 tcg_out_ext32s(s, ret, arg); 625} 626 627static void tcg_out_ldst(TCGContext *s, RISCVInsn opc, TCGReg data, 628 TCGReg addr, intptr_t offset) 629{ 630 intptr_t imm12 = sextreg(offset, 0, 12); 631 632 if (offset != imm12) { 633 intptr_t diff = tcg_pcrel_diff(s, (void *)offset); 634 635 if (addr == TCG_REG_ZERO && diff == (int32_t)diff) { 636 imm12 = sextreg(diff, 0, 12); 637 tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP2, diff - imm12); 638 } else { 639 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP2, offset - imm12); 640 if (addr != TCG_REG_ZERO) { 641 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, addr); 642 } 643 } 644 addr = TCG_REG_TMP2; 645 } 646 647 switch (opc) { 648 case OPC_SB: 649 case OPC_SH: 650 case OPC_SW: 651 case OPC_SD: 652 tcg_out_opc_store(s, opc, addr, data, imm12); 653 break; 654 case OPC_LB: 655 case OPC_LBU: 656 case OPC_LH: 657 case OPC_LHU: 658 case OPC_LW: 659 case OPC_LWU: 660 case OPC_LD: 661 tcg_out_opc_imm(s, opc, data, addr, imm12); 662 break; 663 default: 664 g_assert_not_reached(); 665 } 666} 667 668static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg, 669 TCGReg arg1, intptr_t arg2) 670{ 671 bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); 672 tcg_out_ldst(s, is32bit ? OPC_LW : OPC_LD, arg, arg1, arg2); 673} 674 675static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, 676 TCGReg arg1, intptr_t arg2) 677{ 678 bool is32bit = (TCG_TARGET_REG_BITS == 32 || type == TCG_TYPE_I32); 679 tcg_out_ldst(s, is32bit ? OPC_SW : OPC_SD, arg, arg1, arg2); 680} 681 682static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 683 TCGReg base, intptr_t ofs) 684{ 685 if (val == 0) { 686 tcg_out_st(s, type, TCG_REG_ZERO, base, ofs); 687 return true; 688 } 689 return false; 690} 691 692static void tcg_out_addsub2(TCGContext *s, 693 TCGReg rl, TCGReg rh, 694 TCGReg al, TCGReg ah, 695 TCGArg bl, TCGArg bh, 696 bool cbl, bool cbh, bool is_sub, bool is32bit) 697{ 698 const RISCVInsn opc_add = is32bit ? OPC_ADDW : OPC_ADD; 699 const RISCVInsn opc_addi = is32bit ? OPC_ADDIW : OPC_ADDI; 700 const RISCVInsn opc_sub = is32bit ? OPC_SUBW : OPC_SUB; 701 TCGReg th = TCG_REG_TMP1; 702 703 /* If we have a negative constant such that negating it would 704 make the high part zero, we can (usually) eliminate one insn. */ 705 if (cbl && cbh && bh == -1 && bl != 0) { 706 bl = -bl; 707 bh = 0; 708 is_sub = !is_sub; 709 } 710 711 /* By operating on the high part first, we get to use the final 712 carry operation to move back from the temporary. */ 713 if (!cbh) { 714 tcg_out_opc_reg(s, (is_sub ? opc_sub : opc_add), th, ah, bh); 715 } else if (bh != 0 || ah == rl) { 716 tcg_out_opc_imm(s, opc_addi, th, ah, (is_sub ? -bh : bh)); 717 } else { 718 th = ah; 719 } 720 721 /* Note that tcg optimization should eliminate the bl == 0 case. */ 722 if (is_sub) { 723 if (cbl) { 724 tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, al, bl); 725 tcg_out_opc_imm(s, opc_addi, rl, al, -bl); 726 } else { 727 tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, al, bl); 728 tcg_out_opc_reg(s, opc_sub, rl, al, bl); 729 } 730 tcg_out_opc_reg(s, opc_sub, rh, th, TCG_REG_TMP0); 731 } else { 732 if (cbl) { 733 tcg_out_opc_imm(s, opc_addi, rl, al, bl); 734 tcg_out_opc_imm(s, OPC_SLTIU, TCG_REG_TMP0, rl, bl); 735 } else if (al == bl) { 736 /* 737 * If the input regs overlap, this is a simple doubling 738 * and carry-out is the input msb. This special case is 739 * required when the output reg overlaps the input, 740 * but we might as well use it always. 741 */ 742 tcg_out_opc_imm(s, OPC_SLTI, TCG_REG_TMP0, al, 0); 743 tcg_out_opc_reg(s, opc_add, rl, al, al); 744 } else { 745 tcg_out_opc_reg(s, opc_add, rl, al, bl); 746 tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_TMP0, 747 rl, (rl == bl ? al : bl)); 748 } 749 tcg_out_opc_reg(s, opc_add, rh, th, TCG_REG_TMP0); 750 } 751} 752 753static const struct { 754 RISCVInsn op; 755 bool swap; 756} tcg_brcond_to_riscv[] = { 757 [TCG_COND_EQ] = { OPC_BEQ, false }, 758 [TCG_COND_NE] = { OPC_BNE, false }, 759 [TCG_COND_LT] = { OPC_BLT, false }, 760 [TCG_COND_GE] = { OPC_BGE, false }, 761 [TCG_COND_LE] = { OPC_BGE, true }, 762 [TCG_COND_GT] = { OPC_BLT, true }, 763 [TCG_COND_LTU] = { OPC_BLTU, false }, 764 [TCG_COND_GEU] = { OPC_BGEU, false }, 765 [TCG_COND_LEU] = { OPC_BGEU, true }, 766 [TCG_COND_GTU] = { OPC_BLTU, true } 767}; 768 769static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1, 770 TCGReg arg2, TCGLabel *l) 771{ 772 RISCVInsn op = tcg_brcond_to_riscv[cond].op; 773 774 tcg_debug_assert(op != 0); 775 776 if (tcg_brcond_to_riscv[cond].swap) { 777 TCGReg t = arg1; 778 arg1 = arg2; 779 arg2 = t; 780 } 781 782 tcg_out_reloc(s, s->code_ptr, R_RISCV_BRANCH, l, 0); 783 tcg_out_opc_branch(s, op, arg1, arg2, 0); 784} 785 786static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret, 787 TCGReg arg1, TCGReg arg2) 788{ 789 switch (cond) { 790 case TCG_COND_EQ: 791 tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); 792 tcg_out_opc_imm(s, OPC_SLTIU, ret, ret, 1); 793 break; 794 case TCG_COND_NE: 795 tcg_out_opc_reg(s, OPC_SUB, ret, arg1, arg2); 796 tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, ret); 797 break; 798 case TCG_COND_LT: 799 tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); 800 break; 801 case TCG_COND_GE: 802 tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2); 803 tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); 804 break; 805 case TCG_COND_LE: 806 tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); 807 tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); 808 break; 809 case TCG_COND_GT: 810 tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1); 811 break; 812 case TCG_COND_LTU: 813 tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); 814 break; 815 case TCG_COND_GEU: 816 tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2); 817 tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); 818 break; 819 case TCG_COND_LEU: 820 tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); 821 tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1); 822 break; 823 case TCG_COND_GTU: 824 tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1); 825 break; 826 default: 827 g_assert_not_reached(); 828 break; 829 } 830} 831 832static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, 833 TCGReg bl, TCGReg bh, TCGLabel *l) 834{ 835 /* todo */ 836 g_assert_not_reached(); 837} 838 839static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret, 840 TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh) 841{ 842 /* todo */ 843 g_assert_not_reached(); 844} 845 846static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) 847{ 848 TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA; 849 ptrdiff_t offset = tcg_pcrel_diff(s, arg); 850 int ret; 851 852 tcg_debug_assert((offset & 1) == 0); 853 if (offset == sextreg(offset, 0, 20)) { 854 /* short jump: -2097150 to 2097152 */ 855 tcg_out_opc_jump(s, OPC_JAL, link, offset); 856 } else if (TCG_TARGET_REG_BITS == 32 || offset == (int32_t)offset) { 857 /* long jump: -2147483646 to 2147483648 */ 858 tcg_out_opc_upper(s, OPC_AUIPC, TCG_REG_TMP0, 0); 859 tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, 0); 860 ret = reloc_call(s->code_ptr - 2, arg); 861 tcg_debug_assert(ret == true); 862 } else if (TCG_TARGET_REG_BITS == 64) { 863 /* far jump: 64-bit */ 864 tcg_target_long imm = sextreg((tcg_target_long)arg, 0, 12); 865 tcg_target_long base = (tcg_target_long)arg - imm; 866 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, base); 867 tcg_out_opc_imm(s, OPC_JALR, link, TCG_REG_TMP0, imm); 868 } else { 869 g_assert_not_reached(); 870 } 871} 872 873static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg, 874 const TCGHelperInfo *info) 875{ 876 tcg_out_call_int(s, arg, false); 877} 878 879static void tcg_out_mb(TCGContext *s, TCGArg a0) 880{ 881 tcg_insn_unit insn = OPC_FENCE; 882 883 if (a0 & TCG_MO_LD_LD) { 884 insn |= 0x02200000; 885 } 886 if (a0 & TCG_MO_ST_LD) { 887 insn |= 0x01200000; 888 } 889 if (a0 & TCG_MO_LD_ST) { 890 insn |= 0x02100000; 891 } 892 if (a0 & TCG_MO_ST_ST) { 893 insn |= 0x02200000; 894 } 895 tcg_out32(s, insn); 896} 897 898/* 899 * Load/store and TLB 900 */ 901 902#if defined(CONFIG_SOFTMMU) 903/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr, 904 * MemOpIdx oi, uintptr_t ra) 905 */ 906static void * const qemu_ld_helpers[MO_SSIZE + 1] = { 907 [MO_UB] = helper_ret_ldub_mmu, 908 [MO_SB] = helper_ret_ldsb_mmu, 909#if HOST_BIG_ENDIAN 910 [MO_UW] = helper_be_lduw_mmu, 911 [MO_SW] = helper_be_ldsw_mmu, 912 [MO_UL] = helper_be_ldul_mmu, 913#if TCG_TARGET_REG_BITS == 64 914 [MO_SL] = helper_be_ldsl_mmu, 915#endif 916 [MO_UQ] = helper_be_ldq_mmu, 917#else 918 [MO_UW] = helper_le_lduw_mmu, 919 [MO_SW] = helper_le_ldsw_mmu, 920 [MO_UL] = helper_le_ldul_mmu, 921#if TCG_TARGET_REG_BITS == 64 922 [MO_SL] = helper_le_ldsl_mmu, 923#endif 924 [MO_UQ] = helper_le_ldq_mmu, 925#endif 926}; 927 928/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr, 929 * uintxx_t val, MemOpIdx oi, 930 * uintptr_t ra) 931 */ 932static void * const qemu_st_helpers[MO_SIZE + 1] = { 933 [MO_8] = helper_ret_stb_mmu, 934#if HOST_BIG_ENDIAN 935 [MO_16] = helper_be_stw_mmu, 936 [MO_32] = helper_be_stl_mmu, 937 [MO_64] = helper_be_stq_mmu, 938#else 939 [MO_16] = helper_le_stw_mmu, 940 [MO_32] = helper_le_stl_mmu, 941 [MO_64] = helper_le_stq_mmu, 942#endif 943}; 944 945/* We don't support oversize guests */ 946QEMU_BUILD_BUG_ON(TCG_TARGET_REG_BITS < TARGET_LONG_BITS); 947 948/* We expect to use a 12-bit negative offset from ENV. */ 949QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) > 0); 950QEMU_BUILD_BUG_ON(TLB_MASK_TABLE_OFS(0) < -(1 << 11)); 951 952static void tcg_out_goto(TCGContext *s, const tcg_insn_unit *target) 953{ 954 tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); 955 bool ok = reloc_jimm20(s->code_ptr - 1, target); 956 tcg_debug_assert(ok); 957} 958 959static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addrl, 960 TCGReg addrh, MemOpIdx oi, 961 tcg_insn_unit **label_ptr, bool is_load) 962{ 963 MemOp opc = get_memop(oi); 964 unsigned s_bits = opc & MO_SIZE; 965 unsigned a_bits = get_alignment_bits(opc); 966 tcg_target_long compare_mask; 967 int mem_index = get_mmuidx(oi); 968 int fast_ofs = TLB_MASK_TABLE_OFS(mem_index); 969 int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask); 970 int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table); 971 TCGReg mask_base = TCG_AREG0, table_base = TCG_AREG0; 972 973 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, mask_base, mask_ofs); 974 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, table_base, table_ofs); 975 976 tcg_out_opc_imm(s, OPC_SRLI, TCG_REG_TMP2, addrl, 977 TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); 978 tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0); 979 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1); 980 981 /* Load the tlb comparator and the addend. */ 982 tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP0, TCG_REG_TMP2, 983 is_load ? offsetof(CPUTLBEntry, addr_read) 984 : offsetof(CPUTLBEntry, addr_write)); 985 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2, 986 offsetof(CPUTLBEntry, addend)); 987 988 /* We don't support unaligned accesses. */ 989 if (a_bits < s_bits) { 990 a_bits = s_bits; 991 } 992 /* Clear the non-page, non-alignment bits from the address. */ 993 compare_mask = (tcg_target_long)TARGET_PAGE_MASK | ((1 << a_bits) - 1); 994 if (compare_mask == sextreg(compare_mask, 0, 12)) { 995 tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addrl, compare_mask); 996 } else { 997 tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_TMP1, compare_mask); 998 tcg_out_opc_reg(s, OPC_AND, TCG_REG_TMP1, TCG_REG_TMP1, addrl); 999 } 1000 1001 /* Compare masked address with the TLB entry. */ 1002 label_ptr[0] = s->code_ptr; 1003 tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP0, TCG_REG_TMP1, 0); 1004 1005 /* TLB Hit - translate address using addend. */ 1006 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { 1007 tcg_out_ext32u(s, TCG_REG_TMP0, addrl); 1008 addrl = TCG_REG_TMP0; 1009 } 1010 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_REG_TMP2, addrl); 1011 return TCG_REG_TMP0; 1012} 1013 1014static void add_qemu_ldst_label(TCGContext *s, int is_ld, MemOpIdx oi, 1015 TCGType ext, 1016 TCGReg datalo, TCGReg datahi, 1017 TCGReg addrlo, TCGReg addrhi, 1018 void *raddr, tcg_insn_unit **label_ptr) 1019{ 1020 TCGLabelQemuLdst *label = new_ldst_label(s); 1021 1022 label->is_ld = is_ld; 1023 label->oi = oi; 1024 label->type = ext; 1025 label->datalo_reg = datalo; 1026 label->datahi_reg = datahi; 1027 label->addrlo_reg = addrlo; 1028 label->addrhi_reg = addrhi; 1029 label->raddr = tcg_splitwx_to_rx(raddr); 1030 label->label_ptr[0] = label_ptr[0]; 1031} 1032 1033static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 1034{ 1035 MemOpIdx oi = l->oi; 1036 MemOp opc = get_memop(oi); 1037 TCGReg a0 = tcg_target_call_iarg_regs[0]; 1038 TCGReg a1 = tcg_target_call_iarg_regs[1]; 1039 TCGReg a2 = tcg_target_call_iarg_regs[2]; 1040 TCGReg a3 = tcg_target_call_iarg_regs[3]; 1041 1042 /* We don't support oversize guests */ 1043 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { 1044 g_assert_not_reached(); 1045 } 1046 1047 /* resolve label address */ 1048 if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 1049 return false; 1050 } 1051 1052 /* call load helper */ 1053 tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0); 1054 tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg); 1055 tcg_out_movi(s, TCG_TYPE_PTR, a2, oi); 1056 tcg_out_movi(s, TCG_TYPE_PTR, a3, (tcg_target_long)l->raddr); 1057 1058 tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SSIZE], false); 1059 tcg_out_mov(s, (opc & MO_SIZE) == MO_64, l->datalo_reg, a0); 1060 1061 tcg_out_goto(s, l->raddr); 1062 return true; 1063} 1064 1065static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 1066{ 1067 MemOpIdx oi = l->oi; 1068 MemOp opc = get_memop(oi); 1069 MemOp s_bits = opc & MO_SIZE; 1070 TCGReg a0 = tcg_target_call_iarg_regs[0]; 1071 TCGReg a1 = tcg_target_call_iarg_regs[1]; 1072 TCGReg a2 = tcg_target_call_iarg_regs[2]; 1073 TCGReg a3 = tcg_target_call_iarg_regs[3]; 1074 TCGReg a4 = tcg_target_call_iarg_regs[4]; 1075 1076 /* We don't support oversize guests */ 1077 if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { 1078 g_assert_not_reached(); 1079 } 1080 1081 /* resolve label address */ 1082 if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 1083 return false; 1084 } 1085 1086 /* call store helper */ 1087 tcg_out_mov(s, TCG_TYPE_PTR, a0, TCG_AREG0); 1088 tcg_out_mov(s, TCG_TYPE_PTR, a1, l->addrlo_reg); 1089 tcg_out_movext(s, s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32, a2, 1090 l->type, s_bits, l->datalo_reg); 1091 tcg_out_movi(s, TCG_TYPE_PTR, a3, oi); 1092 tcg_out_movi(s, TCG_TYPE_PTR, a4, (tcg_target_long)l->raddr); 1093 1094 tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE], false); 1095 1096 tcg_out_goto(s, l->raddr); 1097 return true; 1098} 1099#else 1100 1101static void tcg_out_test_alignment(TCGContext *s, bool is_ld, TCGReg addr_reg, 1102 unsigned a_bits) 1103{ 1104 unsigned a_mask = (1 << a_bits) - 1; 1105 TCGLabelQemuLdst *l = new_ldst_label(s); 1106 1107 l->is_ld = is_ld; 1108 l->addrlo_reg = addr_reg; 1109 1110 /* We are expecting a_bits to max out at 7, so we can always use andi. */ 1111 tcg_debug_assert(a_bits < 12); 1112 tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_TMP1, addr_reg, a_mask); 1113 1114 l->label_ptr[0] = s->code_ptr; 1115 tcg_out_opc_branch(s, OPC_BNE, TCG_REG_TMP1, TCG_REG_ZERO, 0); 1116 1117 l->raddr = tcg_splitwx_to_rx(s->code_ptr); 1118} 1119 1120static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) 1121{ 1122 /* resolve label address */ 1123 if (!reloc_sbimm12(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) { 1124 return false; 1125 } 1126 1127 tcg_out_mov(s, TCG_TYPE_TL, TCG_REG_A1, l->addrlo_reg); 1128 tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_A0, TCG_AREG0); 1129 1130 /* tail call, with the return address back inline. */ 1131 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (uintptr_t)l->raddr); 1132 tcg_out_call_int(s, (const void *)(l->is_ld ? helper_unaligned_ld 1133 : helper_unaligned_st), true); 1134 return true; 1135} 1136 1137static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 1138{ 1139 return tcg_out_fail_alignment(s, l); 1140} 1141 1142static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) 1143{ 1144 return tcg_out_fail_alignment(s, l); 1145} 1146 1147#endif /* CONFIG_SOFTMMU */ 1148 1149static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg lo, TCGReg hi, 1150 TCGReg base, MemOp opc, bool is_64) 1151{ 1152 /* Byte swapping is left to middle-end expansion. */ 1153 tcg_debug_assert((opc & MO_BSWAP) == 0); 1154 1155 switch (opc & (MO_SSIZE)) { 1156 case MO_UB: 1157 tcg_out_opc_imm(s, OPC_LBU, lo, base, 0); 1158 break; 1159 case MO_SB: 1160 tcg_out_opc_imm(s, OPC_LB, lo, base, 0); 1161 break; 1162 case MO_UW: 1163 tcg_out_opc_imm(s, OPC_LHU, lo, base, 0); 1164 break; 1165 case MO_SW: 1166 tcg_out_opc_imm(s, OPC_LH, lo, base, 0); 1167 break; 1168 case MO_UL: 1169 if (TCG_TARGET_REG_BITS == 64 && is_64) { 1170 tcg_out_opc_imm(s, OPC_LWU, lo, base, 0); 1171 break; 1172 } 1173 /* FALLTHRU */ 1174 case MO_SL: 1175 tcg_out_opc_imm(s, OPC_LW, lo, base, 0); 1176 break; 1177 case MO_UQ: 1178 /* Prefer to load from offset 0 first, but allow for overlap. */ 1179 if (TCG_TARGET_REG_BITS == 64) { 1180 tcg_out_opc_imm(s, OPC_LD, lo, base, 0); 1181 } else if (lo != base) { 1182 tcg_out_opc_imm(s, OPC_LW, lo, base, 0); 1183 tcg_out_opc_imm(s, OPC_LW, hi, base, 4); 1184 } else { 1185 tcg_out_opc_imm(s, OPC_LW, hi, base, 4); 1186 tcg_out_opc_imm(s, OPC_LW, lo, base, 0); 1187 } 1188 break; 1189 default: 1190 g_assert_not_reached(); 1191 } 1192} 1193 1194static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64) 1195{ 1196 TCGReg addr_regl, addr_regh __attribute__((unused)); 1197 TCGReg data_regl, data_regh; 1198 MemOpIdx oi; 1199 MemOp opc; 1200#if defined(CONFIG_SOFTMMU) 1201 tcg_insn_unit *label_ptr[1]; 1202#else 1203 unsigned a_bits; 1204#endif 1205 TCGReg base; 1206 1207 data_regl = *args++; 1208 data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); 1209 addr_regl = *args++; 1210 addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); 1211 oi = *args++; 1212 opc = get_memop(oi); 1213 1214#if defined(CONFIG_SOFTMMU) 1215 base = tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 1); 1216 tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); 1217 add_qemu_ldst_label(s, 1, oi, 1218 (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), 1219 data_regl, data_regh, addr_regl, addr_regh, 1220 s->code_ptr, label_ptr); 1221#else 1222 a_bits = get_alignment_bits(opc); 1223 if (a_bits) { 1224 tcg_out_test_alignment(s, true, addr_regl, a_bits); 1225 } 1226 base = addr_regl; 1227 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { 1228 tcg_out_ext32u(s, TCG_REG_TMP0, base); 1229 base = TCG_REG_TMP0; 1230 } 1231 if (guest_base != 0) { 1232 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); 1233 base = TCG_REG_TMP0; 1234 } 1235 tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc, is_64); 1236#endif 1237} 1238 1239static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg lo, TCGReg hi, 1240 TCGReg base, MemOp opc) 1241{ 1242 /* Byte swapping is left to middle-end expansion. */ 1243 tcg_debug_assert((opc & MO_BSWAP) == 0); 1244 1245 switch (opc & (MO_SSIZE)) { 1246 case MO_8: 1247 tcg_out_opc_store(s, OPC_SB, base, lo, 0); 1248 break; 1249 case MO_16: 1250 tcg_out_opc_store(s, OPC_SH, base, lo, 0); 1251 break; 1252 case MO_32: 1253 tcg_out_opc_store(s, OPC_SW, base, lo, 0); 1254 break; 1255 case MO_64: 1256 if (TCG_TARGET_REG_BITS == 64) { 1257 tcg_out_opc_store(s, OPC_SD, base, lo, 0); 1258 } else { 1259 tcg_out_opc_store(s, OPC_SW, base, lo, 0); 1260 tcg_out_opc_store(s, OPC_SW, base, hi, 4); 1261 } 1262 break; 1263 default: 1264 g_assert_not_reached(); 1265 } 1266} 1267 1268static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64) 1269{ 1270 TCGReg addr_regl, addr_regh __attribute__((unused)); 1271 TCGReg data_regl, data_regh; 1272 MemOpIdx oi; 1273 MemOp opc; 1274#if defined(CONFIG_SOFTMMU) 1275 tcg_insn_unit *label_ptr[1]; 1276#else 1277 unsigned a_bits; 1278#endif 1279 TCGReg base; 1280 1281 data_regl = *args++; 1282 data_regh = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0); 1283 addr_regl = *args++; 1284 addr_regh = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0); 1285 oi = *args++; 1286 opc = get_memop(oi); 1287 1288#if defined(CONFIG_SOFTMMU) 1289 base = tcg_out_tlb_load(s, addr_regl, addr_regh, oi, label_ptr, 0); 1290 tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); 1291 add_qemu_ldst_label(s, 0, oi, 1292 (is_64 ? TCG_TYPE_I64 : TCG_TYPE_I32), 1293 data_regl, data_regh, addr_regl, addr_regh, 1294 s->code_ptr, label_ptr); 1295#else 1296 a_bits = get_alignment_bits(opc); 1297 if (a_bits) { 1298 tcg_out_test_alignment(s, false, addr_regl, a_bits); 1299 } 1300 base = addr_regl; 1301 if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) { 1302 tcg_out_ext32u(s, TCG_REG_TMP0, base); 1303 base = TCG_REG_TMP0; 1304 } 1305 if (guest_base != 0) { 1306 tcg_out_opc_reg(s, OPC_ADD, TCG_REG_TMP0, TCG_GUEST_BASE_REG, base); 1307 base = TCG_REG_TMP0; 1308 } 1309 tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc); 1310#endif 1311} 1312 1313static const tcg_insn_unit *tb_ret_addr; 1314 1315static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0) 1316{ 1317 /* Reuse the zeroing that exists for goto_ptr. */ 1318 if (a0 == 0) { 1319 tcg_out_call_int(s, tcg_code_gen_epilogue, true); 1320 } else { 1321 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0); 1322 tcg_out_call_int(s, tb_ret_addr, true); 1323 } 1324} 1325 1326static void tcg_out_goto_tb(TCGContext *s, int which) 1327{ 1328 /* Direct branch will be patched by tb_target_set_jmp_target. */ 1329 set_jmp_insn_offset(s, which); 1330 tcg_out32(s, OPC_JAL); 1331 1332 /* When branch is out of range, fall through to indirect. */ 1333 tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_ZERO, 1334 get_jmp_target_addr(s, which)); 1335 tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_TMP0, 0); 1336 set_jmp_reset_offset(s, which); 1337} 1338 1339void tb_target_set_jmp_target(const TranslationBlock *tb, int n, 1340 uintptr_t jmp_rx, uintptr_t jmp_rw) 1341{ 1342 uintptr_t addr = tb->jmp_target_addr[n]; 1343 ptrdiff_t offset = addr - jmp_rx; 1344 tcg_insn_unit insn; 1345 1346 /* Either directly branch, or fall through to indirect branch. */ 1347 if (offset == sextreg(offset, 0, 20)) { 1348 insn = encode_uj(OPC_JAL, TCG_REG_ZERO, offset); 1349 } else { 1350 insn = OPC_NOP; 1351 } 1352 qatomic_set((uint32_t *)jmp_rw, insn); 1353 flush_idcache_range(jmp_rx, jmp_rw, 4); 1354} 1355 1356static void tcg_out_op(TCGContext *s, TCGOpcode opc, 1357 const TCGArg args[TCG_MAX_OP_ARGS], 1358 const int const_args[TCG_MAX_OP_ARGS]) 1359{ 1360 TCGArg a0 = args[0]; 1361 TCGArg a1 = args[1]; 1362 TCGArg a2 = args[2]; 1363 int c2 = const_args[2]; 1364 1365 switch (opc) { 1366 case INDEX_op_goto_ptr: 1367 tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, a0, 0); 1368 break; 1369 1370 case INDEX_op_br: 1371 tcg_out_reloc(s, s->code_ptr, R_RISCV_JAL, arg_label(a0), 0); 1372 tcg_out_opc_jump(s, OPC_JAL, TCG_REG_ZERO, 0); 1373 break; 1374 1375 case INDEX_op_ld8u_i32: 1376 case INDEX_op_ld8u_i64: 1377 tcg_out_ldst(s, OPC_LBU, a0, a1, a2); 1378 break; 1379 case INDEX_op_ld8s_i32: 1380 case INDEX_op_ld8s_i64: 1381 tcg_out_ldst(s, OPC_LB, a0, a1, a2); 1382 break; 1383 case INDEX_op_ld16u_i32: 1384 case INDEX_op_ld16u_i64: 1385 tcg_out_ldst(s, OPC_LHU, a0, a1, a2); 1386 break; 1387 case INDEX_op_ld16s_i32: 1388 case INDEX_op_ld16s_i64: 1389 tcg_out_ldst(s, OPC_LH, a0, a1, a2); 1390 break; 1391 case INDEX_op_ld32u_i64: 1392 tcg_out_ldst(s, OPC_LWU, a0, a1, a2); 1393 break; 1394 case INDEX_op_ld_i32: 1395 case INDEX_op_ld32s_i64: 1396 tcg_out_ldst(s, OPC_LW, a0, a1, a2); 1397 break; 1398 case INDEX_op_ld_i64: 1399 tcg_out_ldst(s, OPC_LD, a0, a1, a2); 1400 break; 1401 1402 case INDEX_op_st8_i32: 1403 case INDEX_op_st8_i64: 1404 tcg_out_ldst(s, OPC_SB, a0, a1, a2); 1405 break; 1406 case INDEX_op_st16_i32: 1407 case INDEX_op_st16_i64: 1408 tcg_out_ldst(s, OPC_SH, a0, a1, a2); 1409 break; 1410 case INDEX_op_st_i32: 1411 case INDEX_op_st32_i64: 1412 tcg_out_ldst(s, OPC_SW, a0, a1, a2); 1413 break; 1414 case INDEX_op_st_i64: 1415 tcg_out_ldst(s, OPC_SD, a0, a1, a2); 1416 break; 1417 1418 case INDEX_op_add_i32: 1419 if (c2) { 1420 tcg_out_opc_imm(s, OPC_ADDIW, a0, a1, a2); 1421 } else { 1422 tcg_out_opc_reg(s, OPC_ADDW, a0, a1, a2); 1423 } 1424 break; 1425 case INDEX_op_add_i64: 1426 if (c2) { 1427 tcg_out_opc_imm(s, OPC_ADDI, a0, a1, a2); 1428 } else { 1429 tcg_out_opc_reg(s, OPC_ADD, a0, a1, a2); 1430 } 1431 break; 1432 1433 case INDEX_op_sub_i32: 1434 if (c2) { 1435 tcg_out_opc_imm(s, OPC_ADDIW, a0, a1, -a2); 1436 } else { 1437 tcg_out_opc_reg(s, OPC_SUBW, a0, a1, a2); 1438 } 1439 break; 1440 case INDEX_op_sub_i64: 1441 if (c2) { 1442 tcg_out_opc_imm(s, OPC_ADDI, a0, a1, -a2); 1443 } else { 1444 tcg_out_opc_reg(s, OPC_SUB, a0, a1, a2); 1445 } 1446 break; 1447 1448 case INDEX_op_and_i32: 1449 case INDEX_op_and_i64: 1450 if (c2) { 1451 tcg_out_opc_imm(s, OPC_ANDI, a0, a1, a2); 1452 } else { 1453 tcg_out_opc_reg(s, OPC_AND, a0, a1, a2); 1454 } 1455 break; 1456 1457 case INDEX_op_or_i32: 1458 case INDEX_op_or_i64: 1459 if (c2) { 1460 tcg_out_opc_imm(s, OPC_ORI, a0, a1, a2); 1461 } else { 1462 tcg_out_opc_reg(s, OPC_OR, a0, a1, a2); 1463 } 1464 break; 1465 1466 case INDEX_op_xor_i32: 1467 case INDEX_op_xor_i64: 1468 if (c2) { 1469 tcg_out_opc_imm(s, OPC_XORI, a0, a1, a2); 1470 } else { 1471 tcg_out_opc_reg(s, OPC_XOR, a0, a1, a2); 1472 } 1473 break; 1474 1475 case INDEX_op_not_i32: 1476 case INDEX_op_not_i64: 1477 tcg_out_opc_imm(s, OPC_XORI, a0, a1, -1); 1478 break; 1479 1480 case INDEX_op_neg_i32: 1481 tcg_out_opc_reg(s, OPC_SUBW, a0, TCG_REG_ZERO, a1); 1482 break; 1483 case INDEX_op_neg_i64: 1484 tcg_out_opc_reg(s, OPC_SUB, a0, TCG_REG_ZERO, a1); 1485 break; 1486 1487 case INDEX_op_mul_i32: 1488 tcg_out_opc_reg(s, OPC_MULW, a0, a1, a2); 1489 break; 1490 case INDEX_op_mul_i64: 1491 tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2); 1492 break; 1493 1494 case INDEX_op_div_i32: 1495 tcg_out_opc_reg(s, OPC_DIVW, a0, a1, a2); 1496 break; 1497 case INDEX_op_div_i64: 1498 tcg_out_opc_reg(s, OPC_DIV, a0, a1, a2); 1499 break; 1500 1501 case INDEX_op_divu_i32: 1502 tcg_out_opc_reg(s, OPC_DIVUW, a0, a1, a2); 1503 break; 1504 case INDEX_op_divu_i64: 1505 tcg_out_opc_reg(s, OPC_DIVU, a0, a1, a2); 1506 break; 1507 1508 case INDEX_op_rem_i32: 1509 tcg_out_opc_reg(s, OPC_REMW, a0, a1, a2); 1510 break; 1511 case INDEX_op_rem_i64: 1512 tcg_out_opc_reg(s, OPC_REM, a0, a1, a2); 1513 break; 1514 1515 case INDEX_op_remu_i32: 1516 tcg_out_opc_reg(s, OPC_REMUW, a0, a1, a2); 1517 break; 1518 case INDEX_op_remu_i64: 1519 tcg_out_opc_reg(s, OPC_REMU, a0, a1, a2); 1520 break; 1521 1522 case INDEX_op_shl_i32: 1523 if (c2) { 1524 tcg_out_opc_imm(s, OPC_SLLIW, a0, a1, a2 & 0x1f); 1525 } else { 1526 tcg_out_opc_reg(s, OPC_SLLW, a0, a1, a2); 1527 } 1528 break; 1529 case INDEX_op_shl_i64: 1530 if (c2) { 1531 tcg_out_opc_imm(s, OPC_SLLI, a0, a1, a2 & 0x3f); 1532 } else { 1533 tcg_out_opc_reg(s, OPC_SLL, a0, a1, a2); 1534 } 1535 break; 1536 1537 case INDEX_op_shr_i32: 1538 if (c2) { 1539 tcg_out_opc_imm(s, OPC_SRLIW, a0, a1, a2 & 0x1f); 1540 } else { 1541 tcg_out_opc_reg(s, OPC_SRLW, a0, a1, a2); 1542 } 1543 break; 1544 case INDEX_op_shr_i64: 1545 if (c2) { 1546 tcg_out_opc_imm(s, OPC_SRLI, a0, a1, a2 & 0x3f); 1547 } else { 1548 tcg_out_opc_reg(s, OPC_SRL, a0, a1, a2); 1549 } 1550 break; 1551 1552 case INDEX_op_sar_i32: 1553 if (c2) { 1554 tcg_out_opc_imm(s, OPC_SRAIW, a0, a1, a2 & 0x1f); 1555 } else { 1556 tcg_out_opc_reg(s, OPC_SRAW, a0, a1, a2); 1557 } 1558 break; 1559 case INDEX_op_sar_i64: 1560 if (c2) { 1561 tcg_out_opc_imm(s, OPC_SRAI, a0, a1, a2 & 0x3f); 1562 } else { 1563 tcg_out_opc_reg(s, OPC_SRA, a0, a1, a2); 1564 } 1565 break; 1566 1567 case INDEX_op_add2_i32: 1568 tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], 1569 const_args[4], const_args[5], false, true); 1570 break; 1571 case INDEX_op_add2_i64: 1572 tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], 1573 const_args[4], const_args[5], false, false); 1574 break; 1575 case INDEX_op_sub2_i32: 1576 tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], 1577 const_args[4], const_args[5], true, true); 1578 break; 1579 case INDEX_op_sub2_i64: 1580 tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5], 1581 const_args[4], const_args[5], true, false); 1582 break; 1583 1584 case INDEX_op_brcond_i32: 1585 case INDEX_op_brcond_i64: 1586 tcg_out_brcond(s, a2, a0, a1, arg_label(args[3])); 1587 break; 1588 case INDEX_op_brcond2_i32: 1589 tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], arg_label(args[5])); 1590 break; 1591 1592 case INDEX_op_setcond_i32: 1593 case INDEX_op_setcond_i64: 1594 tcg_out_setcond(s, args[3], a0, a1, a2); 1595 break; 1596 case INDEX_op_setcond2_i32: 1597 tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]); 1598 break; 1599 1600 case INDEX_op_qemu_ld_i32: 1601 tcg_out_qemu_ld(s, args, false); 1602 break; 1603 case INDEX_op_qemu_ld_i64: 1604 tcg_out_qemu_ld(s, args, true); 1605 break; 1606 case INDEX_op_qemu_st_i32: 1607 tcg_out_qemu_st(s, args, false); 1608 break; 1609 case INDEX_op_qemu_st_i64: 1610 tcg_out_qemu_st(s, args, true); 1611 break; 1612 1613 case INDEX_op_extrh_i64_i32: 1614 tcg_out_opc_imm(s, OPC_SRAI, a0, a1, 32); 1615 break; 1616 1617 case INDEX_op_mulsh_i32: 1618 case INDEX_op_mulsh_i64: 1619 tcg_out_opc_reg(s, OPC_MULH, a0, a1, a2); 1620 break; 1621 1622 case INDEX_op_muluh_i32: 1623 case INDEX_op_muluh_i64: 1624 tcg_out_opc_reg(s, OPC_MULHU, a0, a1, a2); 1625 break; 1626 1627 case INDEX_op_mb: 1628 tcg_out_mb(s, a0); 1629 break; 1630 1631 case INDEX_op_mov_i32: /* Always emitted via tcg_out_mov. */ 1632 case INDEX_op_mov_i64: 1633 case INDEX_op_call: /* Always emitted via tcg_out_call. */ 1634 case INDEX_op_exit_tb: /* Always emitted via tcg_out_exit_tb. */ 1635 case INDEX_op_goto_tb: /* Always emitted via tcg_out_goto_tb. */ 1636 case INDEX_op_ext8s_i32: /* Always emitted via tcg_reg_alloc_op. */ 1637 case INDEX_op_ext8s_i64: 1638 case INDEX_op_ext8u_i32: 1639 case INDEX_op_ext8u_i64: 1640 case INDEX_op_ext16s_i32: 1641 case INDEX_op_ext16s_i64: 1642 case INDEX_op_ext16u_i32: 1643 case INDEX_op_ext16u_i64: 1644 case INDEX_op_ext32s_i64: 1645 case INDEX_op_ext32u_i64: 1646 case INDEX_op_ext_i32_i64: 1647 case INDEX_op_extu_i32_i64: 1648 case INDEX_op_extrl_i64_i32: 1649 default: 1650 g_assert_not_reached(); 1651 } 1652} 1653 1654static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) 1655{ 1656 switch (op) { 1657 case INDEX_op_goto_ptr: 1658 return C_O0_I1(r); 1659 1660 case INDEX_op_ld8u_i32: 1661 case INDEX_op_ld8s_i32: 1662 case INDEX_op_ld16u_i32: 1663 case INDEX_op_ld16s_i32: 1664 case INDEX_op_ld_i32: 1665 case INDEX_op_not_i32: 1666 case INDEX_op_neg_i32: 1667 case INDEX_op_ld8u_i64: 1668 case INDEX_op_ld8s_i64: 1669 case INDEX_op_ld16u_i64: 1670 case INDEX_op_ld16s_i64: 1671 case INDEX_op_ld32s_i64: 1672 case INDEX_op_ld32u_i64: 1673 case INDEX_op_ld_i64: 1674 case INDEX_op_not_i64: 1675 case INDEX_op_neg_i64: 1676 case INDEX_op_ext8u_i32: 1677 case INDEX_op_ext8u_i64: 1678 case INDEX_op_ext16u_i32: 1679 case INDEX_op_ext16u_i64: 1680 case INDEX_op_ext32u_i64: 1681 case INDEX_op_extu_i32_i64: 1682 case INDEX_op_ext8s_i32: 1683 case INDEX_op_ext8s_i64: 1684 case INDEX_op_ext16s_i32: 1685 case INDEX_op_ext16s_i64: 1686 case INDEX_op_ext32s_i64: 1687 case INDEX_op_extrl_i64_i32: 1688 case INDEX_op_extrh_i64_i32: 1689 case INDEX_op_ext_i32_i64: 1690 return C_O1_I1(r, r); 1691 1692 case INDEX_op_st8_i32: 1693 case INDEX_op_st16_i32: 1694 case INDEX_op_st_i32: 1695 case INDEX_op_st8_i64: 1696 case INDEX_op_st16_i64: 1697 case INDEX_op_st32_i64: 1698 case INDEX_op_st_i64: 1699 return C_O0_I2(rZ, r); 1700 1701 case INDEX_op_add_i32: 1702 case INDEX_op_and_i32: 1703 case INDEX_op_or_i32: 1704 case INDEX_op_xor_i32: 1705 case INDEX_op_add_i64: 1706 case INDEX_op_and_i64: 1707 case INDEX_op_or_i64: 1708 case INDEX_op_xor_i64: 1709 return C_O1_I2(r, r, rI); 1710 1711 case INDEX_op_sub_i32: 1712 case INDEX_op_sub_i64: 1713 return C_O1_I2(r, rZ, rN); 1714 1715 case INDEX_op_mul_i32: 1716 case INDEX_op_mulsh_i32: 1717 case INDEX_op_muluh_i32: 1718 case INDEX_op_div_i32: 1719 case INDEX_op_divu_i32: 1720 case INDEX_op_rem_i32: 1721 case INDEX_op_remu_i32: 1722 case INDEX_op_setcond_i32: 1723 case INDEX_op_mul_i64: 1724 case INDEX_op_mulsh_i64: 1725 case INDEX_op_muluh_i64: 1726 case INDEX_op_div_i64: 1727 case INDEX_op_divu_i64: 1728 case INDEX_op_rem_i64: 1729 case INDEX_op_remu_i64: 1730 case INDEX_op_setcond_i64: 1731 return C_O1_I2(r, rZ, rZ); 1732 1733 case INDEX_op_shl_i32: 1734 case INDEX_op_shr_i32: 1735 case INDEX_op_sar_i32: 1736 case INDEX_op_shl_i64: 1737 case INDEX_op_shr_i64: 1738 case INDEX_op_sar_i64: 1739 return C_O1_I2(r, r, ri); 1740 1741 case INDEX_op_brcond_i32: 1742 case INDEX_op_brcond_i64: 1743 return C_O0_I2(rZ, rZ); 1744 1745 case INDEX_op_add2_i32: 1746 case INDEX_op_add2_i64: 1747 case INDEX_op_sub2_i32: 1748 case INDEX_op_sub2_i64: 1749 return C_O2_I4(r, r, rZ, rZ, rM, rM); 1750 1751 case INDEX_op_brcond2_i32: 1752 return C_O0_I4(rZ, rZ, rZ, rZ); 1753 1754 case INDEX_op_setcond2_i32: 1755 return C_O1_I4(r, rZ, rZ, rZ, rZ); 1756 1757 case INDEX_op_qemu_ld_i32: 1758 return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS 1759 ? C_O1_I1(r, L) : C_O1_I2(r, L, L)); 1760 case INDEX_op_qemu_st_i32: 1761 return (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS 1762 ? C_O0_I2(LZ, L) : C_O0_I3(LZ, L, L)); 1763 case INDEX_op_qemu_ld_i64: 1764 return (TCG_TARGET_REG_BITS == 64 ? C_O1_I1(r, L) 1765 : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O2_I1(r, r, L) 1766 : C_O2_I2(r, r, L, L)); 1767 case INDEX_op_qemu_st_i64: 1768 return (TCG_TARGET_REG_BITS == 64 ? C_O0_I2(LZ, L) 1769 : TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? C_O0_I3(LZ, LZ, L) 1770 : C_O0_I4(LZ, LZ, L, L)); 1771 1772 default: 1773 g_assert_not_reached(); 1774 } 1775} 1776 1777static const int tcg_target_callee_save_regs[] = { 1778 TCG_REG_S0, /* used for the global env (TCG_AREG0) */ 1779 TCG_REG_S1, 1780 TCG_REG_S2, 1781 TCG_REG_S3, 1782 TCG_REG_S4, 1783 TCG_REG_S5, 1784 TCG_REG_S6, 1785 TCG_REG_S7, 1786 TCG_REG_S8, 1787 TCG_REG_S9, 1788 TCG_REG_S10, 1789 TCG_REG_S11, 1790 TCG_REG_RA, /* should be last for ABI compliance */ 1791}; 1792 1793/* Stack frame parameters. */ 1794#define REG_SIZE (TCG_TARGET_REG_BITS / 8) 1795#define SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * REG_SIZE) 1796#define TEMP_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long)) 1797#define FRAME_SIZE ((TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE + SAVE_SIZE \ 1798 + TCG_TARGET_STACK_ALIGN - 1) \ 1799 & -TCG_TARGET_STACK_ALIGN) 1800#define SAVE_OFS (TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE) 1801 1802/* We're expecting to be able to use an immediate for frame allocation. */ 1803QEMU_BUILD_BUG_ON(FRAME_SIZE > 0x7ff); 1804 1805/* Generate global QEMU prologue and epilogue code */ 1806static void tcg_target_qemu_prologue(TCGContext *s) 1807{ 1808 int i; 1809 1810 tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE, TEMP_SIZE); 1811 1812 /* TB prologue */ 1813 tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_SP, TCG_REG_SP, -FRAME_SIZE); 1814 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { 1815 tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], 1816 TCG_REG_SP, SAVE_OFS + i * REG_SIZE); 1817 } 1818 1819#if !defined(CONFIG_SOFTMMU) 1820 tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base); 1821 tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG); 1822#endif 1823 1824 /* Call generated code */ 1825 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]); 1826 tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0); 1827 1828 /* Return path for goto_ptr. Set return value to 0 */ 1829 tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr); 1830 tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO); 1831 1832 /* TB epilogue */ 1833 tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr); 1834 for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) { 1835 tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i], 1836 TCG_REG_SP, SAVE_OFS + i * REG_SIZE); 1837 } 1838 1839 tcg_out_opc_imm(s, OPC_ADDI, TCG_REG_SP, TCG_REG_SP, FRAME_SIZE); 1840 tcg_out_opc_imm(s, OPC_JALR, TCG_REG_ZERO, TCG_REG_RA, 0); 1841} 1842 1843static void tcg_target_init(TCGContext *s) 1844{ 1845 tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff; 1846 if (TCG_TARGET_REG_BITS == 64) { 1847 tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff; 1848 } 1849 1850 tcg_target_call_clobber_regs = -1u; 1851 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0); 1852 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S1); 1853 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S2); 1854 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S3); 1855 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S4); 1856 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S5); 1857 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S6); 1858 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S7); 1859 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S8); 1860 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S9); 1861 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S10); 1862 tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S11); 1863 1864 s->reserved_regs = 0; 1865 tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO); 1866 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP0); 1867 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); 1868 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP2); 1869 tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP); 1870 tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP); 1871 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TP); 1872} 1873 1874typedef struct { 1875 DebugFrameHeader h; 1876 uint8_t fde_def_cfa[4]; 1877 uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2]; 1878} DebugFrame; 1879 1880#define ELF_HOST_MACHINE EM_RISCV 1881 1882static const DebugFrame debug_frame = { 1883 .h.cie.len = sizeof(DebugFrameCIE) - 4, /* length after .len member */ 1884 .h.cie.id = -1, 1885 .h.cie.version = 1, 1886 .h.cie.code_align = 1, 1887 .h.cie.data_align = -(TCG_TARGET_REG_BITS / 8) & 0x7f, /* sleb128 */ 1888 .h.cie.return_column = TCG_REG_RA, 1889 1890 /* Total FDE size does not include the "len" member. */ 1891 .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset), 1892 1893 .fde_def_cfa = { 1894 12, TCG_REG_SP, /* DW_CFA_def_cfa sp, ... */ 1895 (FRAME_SIZE & 0x7f) | 0x80, /* ... uleb128 FRAME_SIZE */ 1896 (FRAME_SIZE >> 7) 1897 }, 1898 .fde_reg_ofs = { 1899 0x80 + 9, 12, /* DW_CFA_offset, s1, -96 */ 1900 0x80 + 18, 11, /* DW_CFA_offset, s2, -88 */ 1901 0x80 + 19, 10, /* DW_CFA_offset, s3, -80 */ 1902 0x80 + 20, 9, /* DW_CFA_offset, s4, -72 */ 1903 0x80 + 21, 8, /* DW_CFA_offset, s5, -64 */ 1904 0x80 + 22, 7, /* DW_CFA_offset, s6, -56 */ 1905 0x80 + 23, 6, /* DW_CFA_offset, s7, -48 */ 1906 0x80 + 24, 5, /* DW_CFA_offset, s8, -40 */ 1907 0x80 + 25, 4, /* DW_CFA_offset, s9, -32 */ 1908 0x80 + 26, 3, /* DW_CFA_offset, s10, -24 */ 1909 0x80 + 27, 2, /* DW_CFA_offset, s11, -16 */ 1910 0x80 + 1 , 1, /* DW_CFA_offset, ra, -8 */ 1911 } 1912}; 1913 1914void tcg_register_jit(const void *buf, size_t buf_size) 1915{ 1916 tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame)); 1917} 1918