1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_TCG_OPTIMIZATIONS 27 28 #include "qemu/osdep.h" 29 30 /* Define to jump the ELF file used to communicate with GDB. */ 31 #undef DEBUG_JIT 32 33 #include "qemu/error-report.h" 34 #include "qemu/cutils.h" 35 #include "qemu/host-utils.h" 36 #include "qemu/qemu-print.h" 37 #include "qemu/timer.h" 38 #include "qemu/cacheflush.h" 39 40 /* Note: the long term plan is to reduce the dependencies on the QEMU 41 CPU definitions. Currently they are used for qemu_ld/st 42 instructions */ 43 #define NO_CPU_IO_DEFS 44 45 #include "exec/exec-all.h" 46 #include "tcg/tcg-op.h" 47 48 #if UINTPTR_MAX == UINT32_MAX 49 # define ELF_CLASS ELFCLASS32 50 #else 51 # define ELF_CLASS ELFCLASS64 52 #endif 53 #ifdef HOST_WORDS_BIGENDIAN 54 # define ELF_DATA ELFDATA2MSB 55 #else 56 # define ELF_DATA ELFDATA2LSB 57 #endif 58 59 #include "elf.h" 60 #include "exec/log.h" 61 #include "tcg-internal.h" 62 63 #ifdef CONFIG_TCG_INTERPRETER 64 #include <ffi.h> 65 #endif 66 67 /* Forward declarations for functions declared in tcg-target.c.inc and 68 used here. */ 69 static void tcg_target_init(TCGContext *s); 70 static void tcg_target_qemu_prologue(TCGContext *s); 71 static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 72 intptr_t value, intptr_t addend); 73 74 /* The CIE and FDE header definitions will be common to all hosts. */ 75 typedef struct { 76 uint32_t len __attribute__((aligned((sizeof(void *))))); 77 uint32_t id; 78 uint8_t version; 79 char augmentation[1]; 80 uint8_t code_align; 81 uint8_t data_align; 82 uint8_t return_column; 83 } DebugFrameCIE; 84 85 typedef struct QEMU_PACKED { 86 uint32_t len __attribute__((aligned((sizeof(void *))))); 87 uint32_t cie_offset; 88 uintptr_t func_start; 89 uintptr_t func_len; 90 } DebugFrameFDEHeader; 91 92 typedef struct QEMU_PACKED { 93 DebugFrameCIE cie; 94 DebugFrameFDEHeader fde; 95 } DebugFrameHeader; 96 97 static void tcg_register_jit_int(const void *buf, size_t size, 98 const void *debug_frame, 99 size_t debug_frame_size) 100 __attribute__((unused)); 101 102 /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 103 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 104 intptr_t arg2); 105 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 106 static void tcg_out_movi(TCGContext *s, TCGType type, 107 TCGReg ret, tcg_target_long arg); 108 static void tcg_out_op(TCGContext *s, TCGOpcode opc, 109 const TCGArg args[TCG_MAX_OP_ARGS], 110 const int const_args[TCG_MAX_OP_ARGS]); 111 #if TCG_TARGET_MAYBE_vec 112 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 113 TCGReg dst, TCGReg src); 114 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 115 TCGReg dst, TCGReg base, intptr_t offset); 116 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 117 TCGReg dst, int64_t arg); 118 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 119 unsigned vecl, unsigned vece, 120 const TCGArg args[TCG_MAX_OP_ARGS], 121 const int const_args[TCG_MAX_OP_ARGS]); 122 #else 123 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 124 TCGReg dst, TCGReg src) 125 { 126 g_assert_not_reached(); 127 } 128 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 129 TCGReg dst, TCGReg base, intptr_t offset) 130 { 131 g_assert_not_reached(); 132 } 133 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 134 TCGReg dst, int64_t arg) 135 { 136 g_assert_not_reached(); 137 } 138 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 139 unsigned vecl, unsigned vece, 140 const TCGArg args[TCG_MAX_OP_ARGS], 141 const int const_args[TCG_MAX_OP_ARGS]) 142 { 143 g_assert_not_reached(); 144 } 145 #endif 146 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 147 intptr_t arg2); 148 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 149 TCGReg base, intptr_t ofs); 150 #ifdef CONFIG_TCG_INTERPRETER 151 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 152 ffi_cif *cif); 153 #else 154 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target); 155 #endif 156 static bool tcg_target_const_match(int64_t val, TCGType type, int ct); 157 #ifdef TCG_TARGET_NEED_LDST_LABELS 158 static int tcg_out_ldst_finalize(TCGContext *s); 159 #endif 160 161 TCGContext tcg_init_ctx; 162 __thread TCGContext *tcg_ctx; 163 164 TCGContext **tcg_ctxs; 165 unsigned int tcg_cur_ctxs; 166 unsigned int tcg_max_ctxs; 167 TCGv_env cpu_env = 0; 168 const void *tcg_code_gen_epilogue; 169 uintptr_t tcg_splitwx_diff; 170 171 #ifndef CONFIG_TCG_INTERPRETER 172 tcg_prologue_fn *tcg_qemu_tb_exec; 173 #endif 174 175 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 176 static TCGRegSet tcg_target_call_clobber_regs; 177 178 #if TCG_TARGET_INSN_UNIT_SIZE == 1 179 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 180 { 181 *s->code_ptr++ = v; 182 } 183 184 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 185 uint8_t v) 186 { 187 *p = v; 188 } 189 #endif 190 191 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 192 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 193 { 194 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 195 *s->code_ptr++ = v; 196 } else { 197 tcg_insn_unit *p = s->code_ptr; 198 memcpy(p, &v, sizeof(v)); 199 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 200 } 201 } 202 203 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 204 uint16_t v) 205 { 206 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 207 *p = v; 208 } else { 209 memcpy(p, &v, sizeof(v)); 210 } 211 } 212 #endif 213 214 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 215 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 216 { 217 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 218 *s->code_ptr++ = v; 219 } else { 220 tcg_insn_unit *p = s->code_ptr; 221 memcpy(p, &v, sizeof(v)); 222 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 223 } 224 } 225 226 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 227 uint32_t v) 228 { 229 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 230 *p = v; 231 } else { 232 memcpy(p, &v, sizeof(v)); 233 } 234 } 235 #endif 236 237 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 238 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 239 { 240 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 241 *s->code_ptr++ = v; 242 } else { 243 tcg_insn_unit *p = s->code_ptr; 244 memcpy(p, &v, sizeof(v)); 245 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 246 } 247 } 248 249 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 250 uint64_t v) 251 { 252 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 253 *p = v; 254 } else { 255 memcpy(p, &v, sizeof(v)); 256 } 257 } 258 #endif 259 260 /* label relocation processing */ 261 262 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 263 TCGLabel *l, intptr_t addend) 264 { 265 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 266 267 r->type = type; 268 r->ptr = code_ptr; 269 r->addend = addend; 270 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 271 } 272 273 static void tcg_out_label(TCGContext *s, TCGLabel *l) 274 { 275 tcg_debug_assert(!l->has_value); 276 l->has_value = 1; 277 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 278 } 279 280 TCGLabel *gen_new_label(void) 281 { 282 TCGContext *s = tcg_ctx; 283 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 284 285 memset(l, 0, sizeof(TCGLabel)); 286 l->id = s->nb_labels++; 287 QSIMPLEQ_INIT(&l->relocs); 288 289 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 290 291 return l; 292 } 293 294 static bool tcg_resolve_relocs(TCGContext *s) 295 { 296 TCGLabel *l; 297 298 QSIMPLEQ_FOREACH(l, &s->labels, next) { 299 TCGRelocation *r; 300 uintptr_t value = l->u.value; 301 302 QSIMPLEQ_FOREACH(r, &l->relocs, next) { 303 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 304 return false; 305 } 306 } 307 } 308 return true; 309 } 310 311 static void set_jmp_reset_offset(TCGContext *s, int which) 312 { 313 /* 314 * We will check for overflow at the end of the opcode loop in 315 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 316 */ 317 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s); 318 } 319 320 /* Signal overflow, starting over with fewer guest insns. */ 321 static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s) 322 { 323 siglongjmp(s->jmp_trans, -2); 324 } 325 326 #define C_PFX1(P, A) P##A 327 #define C_PFX2(P, A, B) P##A##_##B 328 #define C_PFX3(P, A, B, C) P##A##_##B##_##C 329 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 330 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 331 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 332 333 /* Define an enumeration for the various combinations. */ 334 335 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 336 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 337 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 338 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 339 340 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 341 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 342 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 343 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 344 345 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 346 347 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 348 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 349 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 350 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 351 352 typedef enum { 353 #include "tcg-target-con-set.h" 354 } TCGConstraintSetIndex; 355 356 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode); 357 358 #undef C_O0_I1 359 #undef C_O0_I2 360 #undef C_O0_I3 361 #undef C_O0_I4 362 #undef C_O1_I1 363 #undef C_O1_I2 364 #undef C_O1_I3 365 #undef C_O1_I4 366 #undef C_N1_I2 367 #undef C_O2_I1 368 #undef C_O2_I2 369 #undef C_O2_I3 370 #undef C_O2_I4 371 372 /* Put all of the constraint sets into an array, indexed by the enum. */ 373 374 #define C_O0_I1(I1) { .args_ct_str = { #I1 } }, 375 #define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } }, 376 #define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } }, 377 #define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } }, 378 379 #define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } }, 380 #define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } }, 381 #define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } }, 382 #define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } }, 383 384 #define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } }, 385 386 #define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } }, 387 #define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } }, 388 #define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } }, 389 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } }, 390 391 static const TCGTargetOpDef constraint_sets[] = { 392 #include "tcg-target-con-set.h" 393 }; 394 395 396 #undef C_O0_I1 397 #undef C_O0_I2 398 #undef C_O0_I3 399 #undef C_O0_I4 400 #undef C_O1_I1 401 #undef C_O1_I2 402 #undef C_O1_I3 403 #undef C_O1_I4 404 #undef C_N1_I2 405 #undef C_O2_I1 406 #undef C_O2_I2 407 #undef C_O2_I3 408 #undef C_O2_I4 409 410 /* Expand the enumerator to be returned from tcg_target_op_def(). */ 411 412 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 413 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 414 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 415 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 416 417 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 418 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 419 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 420 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 421 422 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 423 424 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 425 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 426 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 427 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 428 429 #include "tcg-target.c.inc" 430 431 static void alloc_tcg_plugin_context(TCGContext *s) 432 { 433 #ifdef CONFIG_PLUGIN 434 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1); 435 s->plugin_tb->insns = 436 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn); 437 #endif 438 } 439 440 /* 441 * All TCG threads except the parent (i.e. the one that called tcg_context_init 442 * and registered the target's TCG globals) must register with this function 443 * before initiating translation. 444 * 445 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 446 * of tcg_region_init() for the reasoning behind this. 447 * 448 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in 449 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context 450 * is not used anymore for translation once this function is called. 451 * 452 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates 453 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode. 454 */ 455 #ifdef CONFIG_USER_ONLY 456 void tcg_register_thread(void) 457 { 458 tcg_ctx = &tcg_init_ctx; 459 } 460 #else 461 void tcg_register_thread(void) 462 { 463 TCGContext *s = g_malloc(sizeof(*s)); 464 unsigned int i, n; 465 466 *s = tcg_init_ctx; 467 468 /* Relink mem_base. */ 469 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 470 if (tcg_init_ctx.temps[i].mem_base) { 471 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 472 tcg_debug_assert(b >= 0 && b < n); 473 s->temps[i].mem_base = &s->temps[b]; 474 } 475 } 476 477 /* Claim an entry in tcg_ctxs */ 478 n = qatomic_fetch_inc(&tcg_cur_ctxs); 479 g_assert(n < tcg_max_ctxs); 480 qatomic_set(&tcg_ctxs[n], s); 481 482 if (n > 0) { 483 alloc_tcg_plugin_context(s); 484 tcg_region_initial_alloc(s); 485 } 486 487 tcg_ctx = s; 488 } 489 #endif /* !CONFIG_USER_ONLY */ 490 491 /* pool based memory allocation */ 492 void *tcg_malloc_internal(TCGContext *s, int size) 493 { 494 TCGPool *p; 495 int pool_size; 496 497 if (size > TCG_POOL_CHUNK_SIZE) { 498 /* big malloc: insert a new pool (XXX: could optimize) */ 499 p = g_malloc(sizeof(TCGPool) + size); 500 p->size = size; 501 p->next = s->pool_first_large; 502 s->pool_first_large = p; 503 return p->data; 504 } else { 505 p = s->pool_current; 506 if (!p) { 507 p = s->pool_first; 508 if (!p) 509 goto new_pool; 510 } else { 511 if (!p->next) { 512 new_pool: 513 pool_size = TCG_POOL_CHUNK_SIZE; 514 p = g_malloc(sizeof(TCGPool) + pool_size); 515 p->size = pool_size; 516 p->next = NULL; 517 if (s->pool_current) 518 s->pool_current->next = p; 519 else 520 s->pool_first = p; 521 } else { 522 p = p->next; 523 } 524 } 525 } 526 s->pool_current = p; 527 s->pool_cur = p->data + size; 528 s->pool_end = p->data + p->size; 529 return p->data; 530 } 531 532 void tcg_pool_reset(TCGContext *s) 533 { 534 TCGPool *p, *t; 535 for (p = s->pool_first_large; p; p = t) { 536 t = p->next; 537 g_free(p); 538 } 539 s->pool_first_large = NULL; 540 s->pool_cur = s->pool_end = NULL; 541 s->pool_current = NULL; 542 } 543 544 #include "exec/helper-proto.h" 545 546 static const TCGHelperInfo all_helpers[] = { 547 #include "exec/helper-tcg.h" 548 }; 549 static GHashTable *helper_table; 550 551 #ifdef CONFIG_TCG_INTERPRETER 552 static GHashTable *ffi_table; 553 554 static ffi_type * const typecode_to_ffi[8] = { 555 [dh_typecode_void] = &ffi_type_void, 556 [dh_typecode_i32] = &ffi_type_uint32, 557 [dh_typecode_s32] = &ffi_type_sint32, 558 [dh_typecode_i64] = &ffi_type_uint64, 559 [dh_typecode_s64] = &ffi_type_sint64, 560 [dh_typecode_ptr] = &ffi_type_pointer, 561 }; 562 #endif 563 564 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 565 static void process_op_defs(TCGContext *s); 566 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 567 TCGReg reg, const char *name); 568 569 static void tcg_context_init(unsigned max_cpus) 570 { 571 TCGContext *s = &tcg_init_ctx; 572 int op, total_args, n, i; 573 TCGOpDef *def; 574 TCGArgConstraint *args_ct; 575 TCGTemp *ts; 576 577 memset(s, 0, sizeof(*s)); 578 s->nb_globals = 0; 579 580 /* Count total number of arguments and allocate the corresponding 581 space */ 582 total_args = 0; 583 for(op = 0; op < NB_OPS; op++) { 584 def = &tcg_op_defs[op]; 585 n = def->nb_iargs + def->nb_oargs; 586 total_args += n; 587 } 588 589 args_ct = g_new0(TCGArgConstraint, total_args); 590 591 for(op = 0; op < NB_OPS; op++) { 592 def = &tcg_op_defs[op]; 593 def->args_ct = args_ct; 594 n = def->nb_iargs + def->nb_oargs; 595 args_ct += n; 596 } 597 598 /* Register helpers. */ 599 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 600 helper_table = g_hash_table_new(NULL, NULL); 601 602 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 603 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 604 (gpointer)&all_helpers[i]); 605 } 606 607 #ifdef CONFIG_TCG_INTERPRETER 608 /* g_direct_hash/equal for direct comparisons on uint32_t. */ 609 ffi_table = g_hash_table_new(NULL, NULL); 610 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 611 struct { 612 ffi_cif cif; 613 ffi_type *args[]; 614 } *ca; 615 uint32_t typemask = all_helpers[i].typemask; 616 gpointer hash = (gpointer)(uintptr_t)typemask; 617 ffi_status status; 618 int nargs; 619 620 if (g_hash_table_lookup(ffi_table, hash)) { 621 continue; 622 } 623 624 /* Ignoring the return type, find the last non-zero field. */ 625 nargs = 32 - clz32(typemask >> 3); 626 nargs = DIV_ROUND_UP(nargs, 3); 627 628 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 629 ca->cif.rtype = typecode_to_ffi[typemask & 7]; 630 ca->cif.nargs = nargs; 631 632 if (nargs != 0) { 633 ca->cif.arg_types = ca->args; 634 for (i = 0; i < nargs; ++i) { 635 int typecode = extract32(typemask, (i + 1) * 3, 3); 636 ca->args[i] = typecode_to_ffi[typecode]; 637 } 638 } 639 640 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 641 ca->cif.rtype, ca->cif.arg_types); 642 assert(status == FFI_OK); 643 644 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif); 645 } 646 #endif 647 648 tcg_target_init(s); 649 process_op_defs(s); 650 651 /* Reverse the order of the saved registers, assuming they're all at 652 the start of tcg_target_reg_alloc_order. */ 653 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 654 int r = tcg_target_reg_alloc_order[n]; 655 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 656 break; 657 } 658 } 659 for (i = 0; i < n; ++i) { 660 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 661 } 662 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 663 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 664 } 665 666 alloc_tcg_plugin_context(s); 667 668 tcg_ctx = s; 669 /* 670 * In user-mode we simply share the init context among threads, since we 671 * use a single region. See the documentation tcg_region_init() for the 672 * reasoning behind this. 673 * In softmmu we will have at most max_cpus TCG threads. 674 */ 675 #ifdef CONFIG_USER_ONLY 676 tcg_ctxs = &tcg_ctx; 677 tcg_cur_ctxs = 1; 678 tcg_max_ctxs = 1; 679 #else 680 tcg_max_ctxs = max_cpus; 681 tcg_ctxs = g_new0(TCGContext *, max_cpus); 682 #endif 683 684 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 685 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 686 cpu_env = temp_tcgv_ptr(ts); 687 } 688 689 void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus) 690 { 691 tcg_context_init(max_cpus); 692 tcg_region_init(tb_size, splitwx, max_cpus); 693 } 694 695 /* 696 * Allocate TBs right before their corresponding translated code, making 697 * sure that TBs and code are on different cache lines. 698 */ 699 TranslationBlock *tcg_tb_alloc(TCGContext *s) 700 { 701 uintptr_t align = qemu_icache_linesize; 702 TranslationBlock *tb; 703 void *next; 704 705 retry: 706 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 707 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 708 709 if (unlikely(next > s->code_gen_highwater)) { 710 if (tcg_region_alloc(s)) { 711 return NULL; 712 } 713 goto retry; 714 } 715 qatomic_set(&s->code_gen_ptr, next); 716 s->data_gen_ptr = NULL; 717 return tb; 718 } 719 720 void tcg_prologue_init(TCGContext *s) 721 { 722 size_t prologue_size; 723 724 s->code_ptr = s->code_gen_ptr; 725 s->code_buf = s->code_gen_ptr; 726 s->data_gen_ptr = NULL; 727 728 #ifndef CONFIG_TCG_INTERPRETER 729 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 730 #endif 731 732 #ifdef TCG_TARGET_NEED_POOL_LABELS 733 s->pool_labels = NULL; 734 #endif 735 736 qemu_thread_jit_write(); 737 /* Generate the prologue. */ 738 tcg_target_qemu_prologue(s); 739 740 #ifdef TCG_TARGET_NEED_POOL_LABELS 741 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 742 { 743 int result = tcg_out_pool_finalize(s); 744 tcg_debug_assert(result == 0); 745 } 746 #endif 747 748 prologue_size = tcg_current_code_size(s); 749 750 #ifndef CONFIG_TCG_INTERPRETER 751 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 752 (uintptr_t)s->code_buf, prologue_size); 753 #endif 754 755 #ifdef DEBUG_DISAS 756 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 757 FILE *logfile = qemu_log_lock(); 758 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size); 759 if (s->data_gen_ptr) { 760 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 761 size_t data_size = prologue_size - code_size; 762 size_t i; 763 764 log_disas(s->code_gen_ptr, code_size); 765 766 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 767 if (sizeof(tcg_target_ulong) == 8) { 768 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 769 (uintptr_t)s->data_gen_ptr + i, 770 *(uint64_t *)(s->data_gen_ptr + i)); 771 } else { 772 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n", 773 (uintptr_t)s->data_gen_ptr + i, 774 *(uint32_t *)(s->data_gen_ptr + i)); 775 } 776 } 777 } else { 778 log_disas(s->code_gen_ptr, prologue_size); 779 } 780 qemu_log("\n"); 781 qemu_log_flush(); 782 qemu_log_unlock(logfile); 783 } 784 #endif 785 786 #ifndef CONFIG_TCG_INTERPRETER 787 /* 788 * Assert that goto_ptr is implemented completely, setting an epilogue. 789 * For tci, we use NULL as the signal to return from the interpreter, 790 * so skip this check. 791 */ 792 tcg_debug_assert(tcg_code_gen_epilogue != NULL); 793 #endif 794 795 tcg_region_prologue_set(s); 796 } 797 798 void tcg_func_start(TCGContext *s) 799 { 800 tcg_pool_reset(s); 801 s->nb_temps = s->nb_globals; 802 803 /* No temps have been previously allocated for size or locality. */ 804 memset(s->free_temps, 0, sizeof(s->free_temps)); 805 806 /* No constant temps have been previously allocated. */ 807 for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 808 if (s->const_table[i]) { 809 g_hash_table_remove_all(s->const_table[i]); 810 } 811 } 812 813 s->nb_ops = 0; 814 s->nb_labels = 0; 815 s->current_frame_offset = s->frame_start; 816 817 #ifdef CONFIG_DEBUG_TCG 818 s->goto_tb_issue_mask = 0; 819 #endif 820 821 QTAILQ_INIT(&s->ops); 822 QTAILQ_INIT(&s->free_ops); 823 QSIMPLEQ_INIT(&s->labels); 824 } 825 826 static TCGTemp *tcg_temp_alloc(TCGContext *s) 827 { 828 int n = s->nb_temps++; 829 830 if (n >= TCG_MAX_TEMPS) { 831 tcg_raise_tb_overflow(s); 832 } 833 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 834 } 835 836 static TCGTemp *tcg_global_alloc(TCGContext *s) 837 { 838 TCGTemp *ts; 839 840 tcg_debug_assert(s->nb_globals == s->nb_temps); 841 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 842 s->nb_globals++; 843 ts = tcg_temp_alloc(s); 844 ts->kind = TEMP_GLOBAL; 845 846 return ts; 847 } 848 849 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 850 TCGReg reg, const char *name) 851 { 852 TCGTemp *ts; 853 854 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 855 tcg_abort(); 856 } 857 858 ts = tcg_global_alloc(s); 859 ts->base_type = type; 860 ts->type = type; 861 ts->kind = TEMP_FIXED; 862 ts->reg = reg; 863 ts->name = name; 864 tcg_regset_set_reg(s->reserved_regs, reg); 865 866 return ts; 867 } 868 869 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 870 { 871 s->frame_start = start; 872 s->frame_end = start + size; 873 s->frame_temp 874 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 875 } 876 877 TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 878 intptr_t offset, const char *name) 879 { 880 TCGContext *s = tcg_ctx; 881 TCGTemp *base_ts = tcgv_ptr_temp(base); 882 TCGTemp *ts = tcg_global_alloc(s); 883 int indirect_reg = 0, bigendian = 0; 884 #ifdef HOST_WORDS_BIGENDIAN 885 bigendian = 1; 886 #endif 887 888 switch (base_ts->kind) { 889 case TEMP_FIXED: 890 break; 891 case TEMP_GLOBAL: 892 /* We do not support double-indirect registers. */ 893 tcg_debug_assert(!base_ts->indirect_reg); 894 base_ts->indirect_base = 1; 895 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 896 ? 2 : 1); 897 indirect_reg = 1; 898 break; 899 default: 900 g_assert_not_reached(); 901 } 902 903 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 904 TCGTemp *ts2 = tcg_global_alloc(s); 905 char buf[64]; 906 907 ts->base_type = TCG_TYPE_I64; 908 ts->type = TCG_TYPE_I32; 909 ts->indirect_reg = indirect_reg; 910 ts->mem_allocated = 1; 911 ts->mem_base = base_ts; 912 ts->mem_offset = offset + bigendian * 4; 913 pstrcpy(buf, sizeof(buf), name); 914 pstrcat(buf, sizeof(buf), "_0"); 915 ts->name = strdup(buf); 916 917 tcg_debug_assert(ts2 == ts + 1); 918 ts2->base_type = TCG_TYPE_I64; 919 ts2->type = TCG_TYPE_I32; 920 ts2->indirect_reg = indirect_reg; 921 ts2->mem_allocated = 1; 922 ts2->mem_base = base_ts; 923 ts2->mem_offset = offset + (1 - bigendian) * 4; 924 pstrcpy(buf, sizeof(buf), name); 925 pstrcat(buf, sizeof(buf), "_1"); 926 ts2->name = strdup(buf); 927 } else { 928 ts->base_type = type; 929 ts->type = type; 930 ts->indirect_reg = indirect_reg; 931 ts->mem_allocated = 1; 932 ts->mem_base = base_ts; 933 ts->mem_offset = offset; 934 ts->name = name; 935 } 936 return ts; 937 } 938 939 TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) 940 { 941 TCGContext *s = tcg_ctx; 942 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL; 943 TCGTemp *ts; 944 int idx, k; 945 946 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 947 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 948 if (idx < TCG_MAX_TEMPS) { 949 /* There is already an available temp with the right type. */ 950 clear_bit(idx, s->free_temps[k].l); 951 952 ts = &s->temps[idx]; 953 ts->temp_allocated = 1; 954 tcg_debug_assert(ts->base_type == type); 955 tcg_debug_assert(ts->kind == kind); 956 } else { 957 ts = tcg_temp_alloc(s); 958 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 959 TCGTemp *ts2 = tcg_temp_alloc(s); 960 961 ts->base_type = type; 962 ts->type = TCG_TYPE_I32; 963 ts->temp_allocated = 1; 964 ts->kind = kind; 965 966 tcg_debug_assert(ts2 == ts + 1); 967 ts2->base_type = TCG_TYPE_I64; 968 ts2->type = TCG_TYPE_I32; 969 ts2->temp_allocated = 1; 970 ts2->kind = kind; 971 } else { 972 ts->base_type = type; 973 ts->type = type; 974 ts->temp_allocated = 1; 975 ts->kind = kind; 976 } 977 } 978 979 #if defined(CONFIG_DEBUG_TCG) 980 s->temps_in_use++; 981 #endif 982 return ts; 983 } 984 985 TCGv_vec tcg_temp_new_vec(TCGType type) 986 { 987 TCGTemp *t; 988 989 #ifdef CONFIG_DEBUG_TCG 990 switch (type) { 991 case TCG_TYPE_V64: 992 assert(TCG_TARGET_HAS_v64); 993 break; 994 case TCG_TYPE_V128: 995 assert(TCG_TARGET_HAS_v128); 996 break; 997 case TCG_TYPE_V256: 998 assert(TCG_TARGET_HAS_v256); 999 break; 1000 default: 1001 g_assert_not_reached(); 1002 } 1003 #endif 1004 1005 t = tcg_temp_new_internal(type, 0); 1006 return temp_tcgv_vec(t); 1007 } 1008 1009 /* Create a new temp of the same type as an existing temp. */ 1010 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 1011 { 1012 TCGTemp *t = tcgv_vec_temp(match); 1013 1014 tcg_debug_assert(t->temp_allocated != 0); 1015 1016 t = tcg_temp_new_internal(t->base_type, 0); 1017 return temp_tcgv_vec(t); 1018 } 1019 1020 void tcg_temp_free_internal(TCGTemp *ts) 1021 { 1022 TCGContext *s = tcg_ctx; 1023 int k, idx; 1024 1025 /* In order to simplify users of tcg_constant_*, silently ignore free. */ 1026 if (ts->kind == TEMP_CONST) { 1027 return; 1028 } 1029 1030 #if defined(CONFIG_DEBUG_TCG) 1031 s->temps_in_use--; 1032 if (s->temps_in_use < 0) { 1033 fprintf(stderr, "More temporaries freed than allocated!\n"); 1034 } 1035 #endif 1036 1037 tcg_debug_assert(ts->kind < TEMP_GLOBAL); 1038 tcg_debug_assert(ts->temp_allocated != 0); 1039 ts->temp_allocated = 0; 1040 1041 idx = temp_idx(ts); 1042 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT); 1043 set_bit(idx, s->free_temps[k].l); 1044 } 1045 1046 TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 1047 { 1048 TCGContext *s = tcg_ctx; 1049 GHashTable *h = s->const_table[type]; 1050 TCGTemp *ts; 1051 1052 if (h == NULL) { 1053 h = g_hash_table_new(g_int64_hash, g_int64_equal); 1054 s->const_table[type] = h; 1055 } 1056 1057 ts = g_hash_table_lookup(h, &val); 1058 if (ts == NULL) { 1059 ts = tcg_temp_alloc(s); 1060 1061 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 1062 TCGTemp *ts2 = tcg_temp_alloc(s); 1063 1064 ts->base_type = TCG_TYPE_I64; 1065 ts->type = TCG_TYPE_I32; 1066 ts->kind = TEMP_CONST; 1067 ts->temp_allocated = 1; 1068 /* 1069 * Retain the full value of the 64-bit constant in the low 1070 * part, so that the hash table works. Actual uses will 1071 * truncate the value to the low part. 1072 */ 1073 ts->val = val; 1074 1075 tcg_debug_assert(ts2 == ts + 1); 1076 ts2->base_type = TCG_TYPE_I64; 1077 ts2->type = TCG_TYPE_I32; 1078 ts2->kind = TEMP_CONST; 1079 ts2->temp_allocated = 1; 1080 ts2->val = val >> 32; 1081 } else { 1082 ts->base_type = type; 1083 ts->type = type; 1084 ts->kind = TEMP_CONST; 1085 ts->temp_allocated = 1; 1086 ts->val = val; 1087 } 1088 g_hash_table_insert(h, &ts->val, ts); 1089 } 1090 1091 return ts; 1092 } 1093 1094 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 1095 { 1096 val = dup_const(vece, val); 1097 return temp_tcgv_vec(tcg_constant_internal(type, val)); 1098 } 1099 1100 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 1101 { 1102 TCGTemp *t = tcgv_vec_temp(match); 1103 1104 tcg_debug_assert(t->temp_allocated != 0); 1105 return tcg_constant_vec(t->base_type, vece, val); 1106 } 1107 1108 TCGv_i32 tcg_const_i32(int32_t val) 1109 { 1110 TCGv_i32 t0; 1111 t0 = tcg_temp_new_i32(); 1112 tcg_gen_movi_i32(t0, val); 1113 return t0; 1114 } 1115 1116 TCGv_i64 tcg_const_i64(int64_t val) 1117 { 1118 TCGv_i64 t0; 1119 t0 = tcg_temp_new_i64(); 1120 tcg_gen_movi_i64(t0, val); 1121 return t0; 1122 } 1123 1124 TCGv_i32 tcg_const_local_i32(int32_t val) 1125 { 1126 TCGv_i32 t0; 1127 t0 = tcg_temp_local_new_i32(); 1128 tcg_gen_movi_i32(t0, val); 1129 return t0; 1130 } 1131 1132 TCGv_i64 tcg_const_local_i64(int64_t val) 1133 { 1134 TCGv_i64 t0; 1135 t0 = tcg_temp_local_new_i64(); 1136 tcg_gen_movi_i64(t0, val); 1137 return t0; 1138 } 1139 1140 #if defined(CONFIG_DEBUG_TCG) 1141 void tcg_clear_temp_count(void) 1142 { 1143 TCGContext *s = tcg_ctx; 1144 s->temps_in_use = 0; 1145 } 1146 1147 int tcg_check_temp_count(void) 1148 { 1149 TCGContext *s = tcg_ctx; 1150 if (s->temps_in_use) { 1151 /* Clear the count so that we don't give another 1152 * warning immediately next time around. 1153 */ 1154 s->temps_in_use = 0; 1155 return 1; 1156 } 1157 return 0; 1158 } 1159 #endif 1160 1161 /* Return true if OP may appear in the opcode stream. 1162 Test the runtime variable that controls each opcode. */ 1163 bool tcg_op_supported(TCGOpcode op) 1164 { 1165 const bool have_vec 1166 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256; 1167 1168 switch (op) { 1169 case INDEX_op_discard: 1170 case INDEX_op_set_label: 1171 case INDEX_op_call: 1172 case INDEX_op_br: 1173 case INDEX_op_mb: 1174 case INDEX_op_insn_start: 1175 case INDEX_op_exit_tb: 1176 case INDEX_op_goto_tb: 1177 case INDEX_op_goto_ptr: 1178 case INDEX_op_qemu_ld_i32: 1179 case INDEX_op_qemu_st_i32: 1180 case INDEX_op_qemu_ld_i64: 1181 case INDEX_op_qemu_st_i64: 1182 return true; 1183 1184 case INDEX_op_qemu_st8_i32: 1185 return TCG_TARGET_HAS_qemu_st8_i32; 1186 1187 case INDEX_op_mov_i32: 1188 case INDEX_op_setcond_i32: 1189 case INDEX_op_brcond_i32: 1190 case INDEX_op_ld8u_i32: 1191 case INDEX_op_ld8s_i32: 1192 case INDEX_op_ld16u_i32: 1193 case INDEX_op_ld16s_i32: 1194 case INDEX_op_ld_i32: 1195 case INDEX_op_st8_i32: 1196 case INDEX_op_st16_i32: 1197 case INDEX_op_st_i32: 1198 case INDEX_op_add_i32: 1199 case INDEX_op_sub_i32: 1200 case INDEX_op_mul_i32: 1201 case INDEX_op_and_i32: 1202 case INDEX_op_or_i32: 1203 case INDEX_op_xor_i32: 1204 case INDEX_op_shl_i32: 1205 case INDEX_op_shr_i32: 1206 case INDEX_op_sar_i32: 1207 return true; 1208 1209 case INDEX_op_movcond_i32: 1210 return TCG_TARGET_HAS_movcond_i32; 1211 case INDEX_op_div_i32: 1212 case INDEX_op_divu_i32: 1213 return TCG_TARGET_HAS_div_i32; 1214 case INDEX_op_rem_i32: 1215 case INDEX_op_remu_i32: 1216 return TCG_TARGET_HAS_rem_i32; 1217 case INDEX_op_div2_i32: 1218 case INDEX_op_divu2_i32: 1219 return TCG_TARGET_HAS_div2_i32; 1220 case INDEX_op_rotl_i32: 1221 case INDEX_op_rotr_i32: 1222 return TCG_TARGET_HAS_rot_i32; 1223 case INDEX_op_deposit_i32: 1224 return TCG_TARGET_HAS_deposit_i32; 1225 case INDEX_op_extract_i32: 1226 return TCG_TARGET_HAS_extract_i32; 1227 case INDEX_op_sextract_i32: 1228 return TCG_TARGET_HAS_sextract_i32; 1229 case INDEX_op_extract2_i32: 1230 return TCG_TARGET_HAS_extract2_i32; 1231 case INDEX_op_add2_i32: 1232 return TCG_TARGET_HAS_add2_i32; 1233 case INDEX_op_sub2_i32: 1234 return TCG_TARGET_HAS_sub2_i32; 1235 case INDEX_op_mulu2_i32: 1236 return TCG_TARGET_HAS_mulu2_i32; 1237 case INDEX_op_muls2_i32: 1238 return TCG_TARGET_HAS_muls2_i32; 1239 case INDEX_op_muluh_i32: 1240 return TCG_TARGET_HAS_muluh_i32; 1241 case INDEX_op_mulsh_i32: 1242 return TCG_TARGET_HAS_mulsh_i32; 1243 case INDEX_op_ext8s_i32: 1244 return TCG_TARGET_HAS_ext8s_i32; 1245 case INDEX_op_ext16s_i32: 1246 return TCG_TARGET_HAS_ext16s_i32; 1247 case INDEX_op_ext8u_i32: 1248 return TCG_TARGET_HAS_ext8u_i32; 1249 case INDEX_op_ext16u_i32: 1250 return TCG_TARGET_HAS_ext16u_i32; 1251 case INDEX_op_bswap16_i32: 1252 return TCG_TARGET_HAS_bswap16_i32; 1253 case INDEX_op_bswap32_i32: 1254 return TCG_TARGET_HAS_bswap32_i32; 1255 case INDEX_op_not_i32: 1256 return TCG_TARGET_HAS_not_i32; 1257 case INDEX_op_neg_i32: 1258 return TCG_TARGET_HAS_neg_i32; 1259 case INDEX_op_andc_i32: 1260 return TCG_TARGET_HAS_andc_i32; 1261 case INDEX_op_orc_i32: 1262 return TCG_TARGET_HAS_orc_i32; 1263 case INDEX_op_eqv_i32: 1264 return TCG_TARGET_HAS_eqv_i32; 1265 case INDEX_op_nand_i32: 1266 return TCG_TARGET_HAS_nand_i32; 1267 case INDEX_op_nor_i32: 1268 return TCG_TARGET_HAS_nor_i32; 1269 case INDEX_op_clz_i32: 1270 return TCG_TARGET_HAS_clz_i32; 1271 case INDEX_op_ctz_i32: 1272 return TCG_TARGET_HAS_ctz_i32; 1273 case INDEX_op_ctpop_i32: 1274 return TCG_TARGET_HAS_ctpop_i32; 1275 1276 case INDEX_op_brcond2_i32: 1277 case INDEX_op_setcond2_i32: 1278 return TCG_TARGET_REG_BITS == 32; 1279 1280 case INDEX_op_mov_i64: 1281 case INDEX_op_setcond_i64: 1282 case INDEX_op_brcond_i64: 1283 case INDEX_op_ld8u_i64: 1284 case INDEX_op_ld8s_i64: 1285 case INDEX_op_ld16u_i64: 1286 case INDEX_op_ld16s_i64: 1287 case INDEX_op_ld32u_i64: 1288 case INDEX_op_ld32s_i64: 1289 case INDEX_op_ld_i64: 1290 case INDEX_op_st8_i64: 1291 case INDEX_op_st16_i64: 1292 case INDEX_op_st32_i64: 1293 case INDEX_op_st_i64: 1294 case INDEX_op_add_i64: 1295 case INDEX_op_sub_i64: 1296 case INDEX_op_mul_i64: 1297 case INDEX_op_and_i64: 1298 case INDEX_op_or_i64: 1299 case INDEX_op_xor_i64: 1300 case INDEX_op_shl_i64: 1301 case INDEX_op_shr_i64: 1302 case INDEX_op_sar_i64: 1303 case INDEX_op_ext_i32_i64: 1304 case INDEX_op_extu_i32_i64: 1305 return TCG_TARGET_REG_BITS == 64; 1306 1307 case INDEX_op_movcond_i64: 1308 return TCG_TARGET_HAS_movcond_i64; 1309 case INDEX_op_div_i64: 1310 case INDEX_op_divu_i64: 1311 return TCG_TARGET_HAS_div_i64; 1312 case INDEX_op_rem_i64: 1313 case INDEX_op_remu_i64: 1314 return TCG_TARGET_HAS_rem_i64; 1315 case INDEX_op_div2_i64: 1316 case INDEX_op_divu2_i64: 1317 return TCG_TARGET_HAS_div2_i64; 1318 case INDEX_op_rotl_i64: 1319 case INDEX_op_rotr_i64: 1320 return TCG_TARGET_HAS_rot_i64; 1321 case INDEX_op_deposit_i64: 1322 return TCG_TARGET_HAS_deposit_i64; 1323 case INDEX_op_extract_i64: 1324 return TCG_TARGET_HAS_extract_i64; 1325 case INDEX_op_sextract_i64: 1326 return TCG_TARGET_HAS_sextract_i64; 1327 case INDEX_op_extract2_i64: 1328 return TCG_TARGET_HAS_extract2_i64; 1329 case INDEX_op_extrl_i64_i32: 1330 return TCG_TARGET_HAS_extrl_i64_i32; 1331 case INDEX_op_extrh_i64_i32: 1332 return TCG_TARGET_HAS_extrh_i64_i32; 1333 case INDEX_op_ext8s_i64: 1334 return TCG_TARGET_HAS_ext8s_i64; 1335 case INDEX_op_ext16s_i64: 1336 return TCG_TARGET_HAS_ext16s_i64; 1337 case INDEX_op_ext32s_i64: 1338 return TCG_TARGET_HAS_ext32s_i64; 1339 case INDEX_op_ext8u_i64: 1340 return TCG_TARGET_HAS_ext8u_i64; 1341 case INDEX_op_ext16u_i64: 1342 return TCG_TARGET_HAS_ext16u_i64; 1343 case INDEX_op_ext32u_i64: 1344 return TCG_TARGET_HAS_ext32u_i64; 1345 case INDEX_op_bswap16_i64: 1346 return TCG_TARGET_HAS_bswap16_i64; 1347 case INDEX_op_bswap32_i64: 1348 return TCG_TARGET_HAS_bswap32_i64; 1349 case INDEX_op_bswap64_i64: 1350 return TCG_TARGET_HAS_bswap64_i64; 1351 case INDEX_op_not_i64: 1352 return TCG_TARGET_HAS_not_i64; 1353 case INDEX_op_neg_i64: 1354 return TCG_TARGET_HAS_neg_i64; 1355 case INDEX_op_andc_i64: 1356 return TCG_TARGET_HAS_andc_i64; 1357 case INDEX_op_orc_i64: 1358 return TCG_TARGET_HAS_orc_i64; 1359 case INDEX_op_eqv_i64: 1360 return TCG_TARGET_HAS_eqv_i64; 1361 case INDEX_op_nand_i64: 1362 return TCG_TARGET_HAS_nand_i64; 1363 case INDEX_op_nor_i64: 1364 return TCG_TARGET_HAS_nor_i64; 1365 case INDEX_op_clz_i64: 1366 return TCG_TARGET_HAS_clz_i64; 1367 case INDEX_op_ctz_i64: 1368 return TCG_TARGET_HAS_ctz_i64; 1369 case INDEX_op_ctpop_i64: 1370 return TCG_TARGET_HAS_ctpop_i64; 1371 case INDEX_op_add2_i64: 1372 return TCG_TARGET_HAS_add2_i64; 1373 case INDEX_op_sub2_i64: 1374 return TCG_TARGET_HAS_sub2_i64; 1375 case INDEX_op_mulu2_i64: 1376 return TCG_TARGET_HAS_mulu2_i64; 1377 case INDEX_op_muls2_i64: 1378 return TCG_TARGET_HAS_muls2_i64; 1379 case INDEX_op_muluh_i64: 1380 return TCG_TARGET_HAS_muluh_i64; 1381 case INDEX_op_mulsh_i64: 1382 return TCG_TARGET_HAS_mulsh_i64; 1383 1384 case INDEX_op_mov_vec: 1385 case INDEX_op_dup_vec: 1386 case INDEX_op_dupm_vec: 1387 case INDEX_op_ld_vec: 1388 case INDEX_op_st_vec: 1389 case INDEX_op_add_vec: 1390 case INDEX_op_sub_vec: 1391 case INDEX_op_and_vec: 1392 case INDEX_op_or_vec: 1393 case INDEX_op_xor_vec: 1394 case INDEX_op_cmp_vec: 1395 return have_vec; 1396 case INDEX_op_dup2_vec: 1397 return have_vec && TCG_TARGET_REG_BITS == 32; 1398 case INDEX_op_not_vec: 1399 return have_vec && TCG_TARGET_HAS_not_vec; 1400 case INDEX_op_neg_vec: 1401 return have_vec && TCG_TARGET_HAS_neg_vec; 1402 case INDEX_op_abs_vec: 1403 return have_vec && TCG_TARGET_HAS_abs_vec; 1404 case INDEX_op_andc_vec: 1405 return have_vec && TCG_TARGET_HAS_andc_vec; 1406 case INDEX_op_orc_vec: 1407 return have_vec && TCG_TARGET_HAS_orc_vec; 1408 case INDEX_op_mul_vec: 1409 return have_vec && TCG_TARGET_HAS_mul_vec; 1410 case INDEX_op_shli_vec: 1411 case INDEX_op_shri_vec: 1412 case INDEX_op_sari_vec: 1413 return have_vec && TCG_TARGET_HAS_shi_vec; 1414 case INDEX_op_shls_vec: 1415 case INDEX_op_shrs_vec: 1416 case INDEX_op_sars_vec: 1417 return have_vec && TCG_TARGET_HAS_shs_vec; 1418 case INDEX_op_shlv_vec: 1419 case INDEX_op_shrv_vec: 1420 case INDEX_op_sarv_vec: 1421 return have_vec && TCG_TARGET_HAS_shv_vec; 1422 case INDEX_op_rotli_vec: 1423 return have_vec && TCG_TARGET_HAS_roti_vec; 1424 case INDEX_op_rotls_vec: 1425 return have_vec && TCG_TARGET_HAS_rots_vec; 1426 case INDEX_op_rotlv_vec: 1427 case INDEX_op_rotrv_vec: 1428 return have_vec && TCG_TARGET_HAS_rotv_vec; 1429 case INDEX_op_ssadd_vec: 1430 case INDEX_op_usadd_vec: 1431 case INDEX_op_sssub_vec: 1432 case INDEX_op_ussub_vec: 1433 return have_vec && TCG_TARGET_HAS_sat_vec; 1434 case INDEX_op_smin_vec: 1435 case INDEX_op_umin_vec: 1436 case INDEX_op_smax_vec: 1437 case INDEX_op_umax_vec: 1438 return have_vec && TCG_TARGET_HAS_minmax_vec; 1439 case INDEX_op_bitsel_vec: 1440 return have_vec && TCG_TARGET_HAS_bitsel_vec; 1441 case INDEX_op_cmpsel_vec: 1442 return have_vec && TCG_TARGET_HAS_cmpsel_vec; 1443 1444 default: 1445 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS); 1446 return true; 1447 } 1448 } 1449 1450 /* Note: we convert the 64 bit args to 32 bit and do some alignment 1451 and endian swap. Maybe it would be better to do the alignment 1452 and endian swap in tcg_reg_alloc_call(). */ 1453 void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) 1454 { 1455 int i, real_args, nb_rets, pi; 1456 unsigned typemask; 1457 const TCGHelperInfo *info; 1458 TCGOp *op; 1459 1460 info = g_hash_table_lookup(helper_table, (gpointer)func); 1461 typemask = info->typemask; 1462 1463 #ifdef CONFIG_PLUGIN 1464 /* detect non-plugin helpers */ 1465 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) { 1466 tcg_ctx->plugin_insn->calls_helpers = true; 1467 } 1468 #endif 1469 1470 #if defined(__sparc__) && !defined(__arch64__) \ 1471 && !defined(CONFIG_TCG_INTERPRETER) 1472 /* We have 64-bit values in one register, but need to pass as two 1473 separate parameters. Split them. */ 1474 int orig_typemask = typemask; 1475 int orig_nargs = nargs; 1476 TCGv_i64 retl, reth; 1477 TCGTemp *split_args[MAX_OPC_PARAM]; 1478 1479 retl = NULL; 1480 reth = NULL; 1481 typemask = 0; 1482 for (i = real_args = 0; i < nargs; ++i) { 1483 int argtype = extract32(orig_typemask, (i + 1) * 3, 3); 1484 bool is_64bit = (argtype & ~1) == dh_typecode_i64; 1485 1486 if (is_64bit) { 1487 TCGv_i64 orig = temp_tcgv_i64(args[i]); 1488 TCGv_i32 h = tcg_temp_new_i32(); 1489 TCGv_i32 l = tcg_temp_new_i32(); 1490 tcg_gen_extr_i64_i32(l, h, orig); 1491 split_args[real_args++] = tcgv_i32_temp(h); 1492 typemask |= dh_typecode_i32 << (real_args * 3); 1493 split_args[real_args++] = tcgv_i32_temp(l); 1494 typemask |= dh_typecode_i32 << (real_args * 3); 1495 } else { 1496 split_args[real_args++] = args[i]; 1497 typemask |= argtype << (real_args * 3); 1498 } 1499 } 1500 nargs = real_args; 1501 args = split_args; 1502 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1503 for (i = 0; i < nargs; ++i) { 1504 int argtype = extract32(typemask, (i + 1) * 3, 3); 1505 bool is_32bit = (argtype & ~1) == dh_typecode_i32; 1506 bool is_signed = argtype & 1; 1507 1508 if (is_32bit) { 1509 TCGv_i64 temp = tcg_temp_new_i64(); 1510 TCGv_i64 orig = temp_tcgv_i64(args[i]); 1511 if (is_signed) { 1512 tcg_gen_ext32s_i64(temp, orig); 1513 } else { 1514 tcg_gen_ext32u_i64(temp, orig); 1515 } 1516 args[i] = tcgv_i64_temp(temp); 1517 } 1518 } 1519 #endif /* TCG_TARGET_EXTEND_ARGS */ 1520 1521 op = tcg_emit_op(INDEX_op_call); 1522 1523 pi = 0; 1524 if (ret != NULL) { 1525 #if defined(__sparc__) && !defined(__arch64__) \ 1526 && !defined(CONFIG_TCG_INTERPRETER) 1527 if ((typemask & 6) == dh_typecode_i64) { 1528 /* The 32-bit ABI is going to return the 64-bit value in 1529 the %o0/%o1 register pair. Prepare for this by using 1530 two return temporaries, and reassemble below. */ 1531 retl = tcg_temp_new_i64(); 1532 reth = tcg_temp_new_i64(); 1533 op->args[pi++] = tcgv_i64_arg(reth); 1534 op->args[pi++] = tcgv_i64_arg(retl); 1535 nb_rets = 2; 1536 } else { 1537 op->args[pi++] = temp_arg(ret); 1538 nb_rets = 1; 1539 } 1540 #else 1541 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) { 1542 #ifdef HOST_WORDS_BIGENDIAN 1543 op->args[pi++] = temp_arg(ret + 1); 1544 op->args[pi++] = temp_arg(ret); 1545 #else 1546 op->args[pi++] = temp_arg(ret); 1547 op->args[pi++] = temp_arg(ret + 1); 1548 #endif 1549 nb_rets = 2; 1550 } else { 1551 op->args[pi++] = temp_arg(ret); 1552 nb_rets = 1; 1553 } 1554 #endif 1555 } else { 1556 nb_rets = 0; 1557 } 1558 TCGOP_CALLO(op) = nb_rets; 1559 1560 real_args = 0; 1561 for (i = 0; i < nargs; i++) { 1562 int argtype = extract32(typemask, (i + 1) * 3, 3); 1563 bool is_64bit = (argtype & ~1) == dh_typecode_i64; 1564 bool want_align = false; 1565 1566 #if defined(CONFIG_TCG_INTERPRETER) 1567 /* 1568 * Align all arguments, so that they land in predictable places 1569 * for passing off to ffi_call. 1570 */ 1571 want_align = true; 1572 #elif defined(TCG_TARGET_CALL_ALIGN_ARGS) 1573 /* Some targets want aligned 64 bit args */ 1574 want_align = is_64bit; 1575 #endif 1576 1577 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) { 1578 op->args[pi++] = TCG_CALL_DUMMY_ARG; 1579 real_args++; 1580 } 1581 1582 if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 1583 /* 1584 * If stack grows up, then we will be placing successive 1585 * arguments at lower addresses, which means we need to 1586 * reverse the order compared to how we would normally 1587 * treat either big or little-endian. For those arguments 1588 * that will wind up in registers, this still works for 1589 * HPPA (the only current STACK_GROWSUP target) since the 1590 * argument registers are *also* allocated in decreasing 1591 * order. If another such target is added, this logic may 1592 * have to get more complicated to differentiate between 1593 * stack arguments and register arguments. 1594 */ 1595 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 1596 op->args[pi++] = temp_arg(args[i] + 1); 1597 op->args[pi++] = temp_arg(args[i]); 1598 #else 1599 op->args[pi++] = temp_arg(args[i]); 1600 op->args[pi++] = temp_arg(args[i] + 1); 1601 #endif 1602 real_args += 2; 1603 continue; 1604 } 1605 1606 op->args[pi++] = temp_arg(args[i]); 1607 real_args++; 1608 } 1609 op->args[pi++] = (uintptr_t)func; 1610 op->args[pi++] = (uintptr_t)info; 1611 TCGOP_CALLI(op) = real_args; 1612 1613 /* Make sure the fields didn't overflow. */ 1614 tcg_debug_assert(TCGOP_CALLI(op) == real_args); 1615 tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); 1616 1617 #if defined(__sparc__) && !defined(__arch64__) \ 1618 && !defined(CONFIG_TCG_INTERPRETER) 1619 /* Free all of the parts we allocated above. */ 1620 for (i = real_args = 0; i < orig_nargs; ++i) { 1621 int argtype = extract32(orig_typemask, (i + 1) * 3, 3); 1622 bool is_64bit = (argtype & ~1) == dh_typecode_i64; 1623 1624 if (is_64bit) { 1625 tcg_temp_free_internal(args[real_args++]); 1626 tcg_temp_free_internal(args[real_args++]); 1627 } else { 1628 real_args++; 1629 } 1630 } 1631 if ((orig_typemask & 6) == dh_typecode_i64) { 1632 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 1633 Note that describing these as TCGv_i64 eliminates an unnecessary 1634 zero-extension that tcg_gen_concat_i32_i64 would create. */ 1635 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth); 1636 tcg_temp_free_i64(retl); 1637 tcg_temp_free_i64(reth); 1638 } 1639 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 1640 for (i = 0; i < nargs; ++i) { 1641 int argtype = extract32(typemask, (i + 1) * 3, 3); 1642 bool is_32bit = (argtype & ~1) == dh_typecode_i32; 1643 1644 if (is_32bit) { 1645 tcg_temp_free_internal(args[i]); 1646 } 1647 } 1648 #endif /* TCG_TARGET_EXTEND_ARGS */ 1649 } 1650 1651 static void tcg_reg_alloc_start(TCGContext *s) 1652 { 1653 int i, n; 1654 1655 for (i = 0, n = s->nb_temps; i < n; i++) { 1656 TCGTemp *ts = &s->temps[i]; 1657 TCGTempVal val = TEMP_VAL_MEM; 1658 1659 switch (ts->kind) { 1660 case TEMP_CONST: 1661 val = TEMP_VAL_CONST; 1662 break; 1663 case TEMP_FIXED: 1664 val = TEMP_VAL_REG; 1665 break; 1666 case TEMP_GLOBAL: 1667 break; 1668 case TEMP_NORMAL: 1669 val = TEMP_VAL_DEAD; 1670 /* fall through */ 1671 case TEMP_LOCAL: 1672 ts->mem_allocated = 0; 1673 break; 1674 default: 1675 g_assert_not_reached(); 1676 } 1677 ts->val_type = val; 1678 } 1679 1680 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 1681 } 1682 1683 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 1684 TCGTemp *ts) 1685 { 1686 int idx = temp_idx(ts); 1687 1688 switch (ts->kind) { 1689 case TEMP_FIXED: 1690 case TEMP_GLOBAL: 1691 pstrcpy(buf, buf_size, ts->name); 1692 break; 1693 case TEMP_LOCAL: 1694 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1695 break; 1696 case TEMP_NORMAL: 1697 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1698 break; 1699 case TEMP_CONST: 1700 switch (ts->type) { 1701 case TCG_TYPE_I32: 1702 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 1703 break; 1704 #if TCG_TARGET_REG_BITS > 32 1705 case TCG_TYPE_I64: 1706 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 1707 break; 1708 #endif 1709 case TCG_TYPE_V64: 1710 case TCG_TYPE_V128: 1711 case TCG_TYPE_V256: 1712 snprintf(buf, buf_size, "v%d$0x%" PRIx64, 1713 64 << (ts->type - TCG_TYPE_V64), ts->val); 1714 break; 1715 default: 1716 g_assert_not_reached(); 1717 } 1718 break; 1719 } 1720 return buf; 1721 } 1722 1723 static char *tcg_get_arg_str(TCGContext *s, char *buf, 1724 int buf_size, TCGArg arg) 1725 { 1726 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 1727 } 1728 1729 static const char * const cond_name[] = 1730 { 1731 [TCG_COND_NEVER] = "never", 1732 [TCG_COND_ALWAYS] = "always", 1733 [TCG_COND_EQ] = "eq", 1734 [TCG_COND_NE] = "ne", 1735 [TCG_COND_LT] = "lt", 1736 [TCG_COND_GE] = "ge", 1737 [TCG_COND_LE] = "le", 1738 [TCG_COND_GT] = "gt", 1739 [TCG_COND_LTU] = "ltu", 1740 [TCG_COND_GEU] = "geu", 1741 [TCG_COND_LEU] = "leu", 1742 [TCG_COND_GTU] = "gtu" 1743 }; 1744 1745 static const char * const ldst_name[] = 1746 { 1747 [MO_UB] = "ub", 1748 [MO_SB] = "sb", 1749 [MO_LEUW] = "leuw", 1750 [MO_LESW] = "lesw", 1751 [MO_LEUL] = "leul", 1752 [MO_LESL] = "lesl", 1753 [MO_LEQ] = "leq", 1754 [MO_BEUW] = "beuw", 1755 [MO_BESW] = "besw", 1756 [MO_BEUL] = "beul", 1757 [MO_BESL] = "besl", 1758 [MO_BEQ] = "beq", 1759 }; 1760 1761 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1762 #ifdef TARGET_ALIGNED_ONLY 1763 [MO_UNALN >> MO_ASHIFT] = "un+", 1764 [MO_ALIGN >> MO_ASHIFT] = "", 1765 #else 1766 [MO_UNALN >> MO_ASHIFT] = "", 1767 [MO_ALIGN >> MO_ASHIFT] = "al+", 1768 #endif 1769 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1770 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1771 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1772 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1773 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1774 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1775 }; 1776 1777 static const char bswap_flag_name[][6] = { 1778 [TCG_BSWAP_IZ] = "iz", 1779 [TCG_BSWAP_OZ] = "oz", 1780 [TCG_BSWAP_OS] = "os", 1781 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 1782 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 1783 }; 1784 1785 static inline bool tcg_regset_single(TCGRegSet d) 1786 { 1787 return (d & (d - 1)) == 0; 1788 } 1789 1790 static inline TCGReg tcg_regset_first(TCGRegSet d) 1791 { 1792 if (TCG_TARGET_NB_REGS <= 32) { 1793 return ctz32(d); 1794 } else { 1795 return ctz64(d); 1796 } 1797 } 1798 1799 static void tcg_dump_ops(TCGContext *s, bool have_prefs) 1800 { 1801 char buf[128]; 1802 TCGOp *op; 1803 1804 QTAILQ_FOREACH(op, &s->ops, link) { 1805 int i, k, nb_oargs, nb_iargs, nb_cargs; 1806 const TCGOpDef *def; 1807 TCGOpcode c; 1808 int col = 0; 1809 1810 c = op->opc; 1811 def = &tcg_op_defs[c]; 1812 1813 if (c == INDEX_op_insn_start) { 1814 nb_oargs = 0; 1815 col += qemu_log("\n ----"); 1816 1817 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 1818 target_ulong a; 1819 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1820 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 1821 #else 1822 a = op->args[i]; 1823 #endif 1824 col += qemu_log(" " TARGET_FMT_lx, a); 1825 } 1826 } else if (c == INDEX_op_call) { 1827 const TCGHelperInfo *info = tcg_call_info(op); 1828 void *func = tcg_call_func(op); 1829 1830 /* variable number of arguments */ 1831 nb_oargs = TCGOP_CALLO(op); 1832 nb_iargs = TCGOP_CALLI(op); 1833 nb_cargs = def->nb_cargs; 1834 1835 col += qemu_log(" %s ", def->name); 1836 1837 /* 1838 * Print the function name from TCGHelperInfo, if available. 1839 * Note that plugins have a template function for the info, 1840 * but the actual function pointer comes from the plugin. 1841 */ 1842 if (func == info->func) { 1843 col += qemu_log("%s", info->name); 1844 } else { 1845 col += qemu_log("plugin(%p)", func); 1846 } 1847 1848 col += qemu_log(",$0x%x,$%d", info->flags, nb_oargs); 1849 for (i = 0; i < nb_oargs; i++) { 1850 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf), 1851 op->args[i])); 1852 } 1853 for (i = 0; i < nb_iargs; i++) { 1854 TCGArg arg = op->args[nb_oargs + i]; 1855 const char *t = "<dummy>"; 1856 if (arg != TCG_CALL_DUMMY_ARG) { 1857 t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 1858 } 1859 col += qemu_log(",%s", t); 1860 } 1861 } else { 1862 col += qemu_log(" %s ", def->name); 1863 1864 nb_oargs = def->nb_oargs; 1865 nb_iargs = def->nb_iargs; 1866 nb_cargs = def->nb_cargs; 1867 1868 if (def->flags & TCG_OPF_VECTOR) { 1869 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op), 1870 8 << TCGOP_VECE(op)); 1871 } 1872 1873 k = 0; 1874 for (i = 0; i < nb_oargs; i++) { 1875 if (k != 0) { 1876 col += qemu_log(","); 1877 } 1878 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1879 op->args[k++])); 1880 } 1881 for (i = 0; i < nb_iargs; i++) { 1882 if (k != 0) { 1883 col += qemu_log(","); 1884 } 1885 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf), 1886 op->args[k++])); 1887 } 1888 switch (c) { 1889 case INDEX_op_brcond_i32: 1890 case INDEX_op_setcond_i32: 1891 case INDEX_op_movcond_i32: 1892 case INDEX_op_brcond2_i32: 1893 case INDEX_op_setcond2_i32: 1894 case INDEX_op_brcond_i64: 1895 case INDEX_op_setcond_i64: 1896 case INDEX_op_movcond_i64: 1897 case INDEX_op_cmp_vec: 1898 case INDEX_op_cmpsel_vec: 1899 if (op->args[k] < ARRAY_SIZE(cond_name) 1900 && cond_name[op->args[k]]) { 1901 col += qemu_log(",%s", cond_name[op->args[k++]]); 1902 } else { 1903 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]); 1904 } 1905 i = 1; 1906 break; 1907 case INDEX_op_qemu_ld_i32: 1908 case INDEX_op_qemu_st_i32: 1909 case INDEX_op_qemu_st8_i32: 1910 case INDEX_op_qemu_ld_i64: 1911 case INDEX_op_qemu_st_i64: 1912 { 1913 MemOpIdx oi = op->args[k++]; 1914 MemOp op = get_memop(oi); 1915 unsigned ix = get_mmuidx(oi); 1916 1917 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1918 col += qemu_log(",$0x%x,%u", op, ix); 1919 } else { 1920 const char *s_al, *s_op; 1921 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 1922 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1923 col += qemu_log(",%s%s,%u", s_al, s_op, ix); 1924 } 1925 i = 1; 1926 } 1927 break; 1928 case INDEX_op_bswap16_i32: 1929 case INDEX_op_bswap16_i64: 1930 case INDEX_op_bswap32_i32: 1931 case INDEX_op_bswap32_i64: 1932 case INDEX_op_bswap64_i64: 1933 { 1934 TCGArg flags = op->args[k]; 1935 const char *name = NULL; 1936 1937 if (flags < ARRAY_SIZE(bswap_flag_name)) { 1938 name = bswap_flag_name[flags]; 1939 } 1940 if (name) { 1941 col += qemu_log(",%s", name); 1942 } else { 1943 col += qemu_log(",$0x%" TCG_PRIlx, flags); 1944 } 1945 i = k = 1; 1946 } 1947 break; 1948 default: 1949 i = 0; 1950 break; 1951 } 1952 switch (c) { 1953 case INDEX_op_set_label: 1954 case INDEX_op_br: 1955 case INDEX_op_brcond_i32: 1956 case INDEX_op_brcond_i64: 1957 case INDEX_op_brcond2_i32: 1958 col += qemu_log("%s$L%d", k ? "," : "", 1959 arg_label(op->args[k])->id); 1960 i++, k++; 1961 break; 1962 default: 1963 break; 1964 } 1965 for (; i < nb_cargs; i++, k++) { 1966 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]); 1967 } 1968 } 1969 1970 if (have_prefs || op->life) { 1971 1972 QemuLogFile *logfile; 1973 1974 rcu_read_lock(); 1975 logfile = qatomic_rcu_read(&qemu_logfile); 1976 if (logfile) { 1977 for (; col < 40; ++col) { 1978 putc(' ', logfile->fd); 1979 } 1980 } 1981 rcu_read_unlock(); 1982 } 1983 1984 if (op->life) { 1985 unsigned life = op->life; 1986 1987 if (life & (SYNC_ARG * 3)) { 1988 qemu_log(" sync:"); 1989 for (i = 0; i < 2; ++i) { 1990 if (life & (SYNC_ARG << i)) { 1991 qemu_log(" %d", i); 1992 } 1993 } 1994 } 1995 life /= DEAD_ARG; 1996 if (life) { 1997 qemu_log(" dead:"); 1998 for (i = 0; life; ++i, life >>= 1) { 1999 if (life & 1) { 2000 qemu_log(" %d", i); 2001 } 2002 } 2003 } 2004 } 2005 2006 if (have_prefs) { 2007 for (i = 0; i < nb_oargs; ++i) { 2008 TCGRegSet set = op->output_pref[i]; 2009 2010 if (i == 0) { 2011 qemu_log(" pref="); 2012 } else { 2013 qemu_log(","); 2014 } 2015 if (set == 0) { 2016 qemu_log("none"); 2017 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 2018 qemu_log("all"); 2019 #ifdef CONFIG_DEBUG_TCG 2020 } else if (tcg_regset_single(set)) { 2021 TCGReg reg = tcg_regset_first(set); 2022 qemu_log("%s", tcg_target_reg_names[reg]); 2023 #endif 2024 } else if (TCG_TARGET_NB_REGS <= 32) { 2025 qemu_log("%#x", (uint32_t)set); 2026 } else { 2027 qemu_log("%#" PRIx64, (uint64_t)set); 2028 } 2029 } 2030 } 2031 2032 qemu_log("\n"); 2033 } 2034 } 2035 2036 /* we give more priority to constraints with less registers */ 2037 static int get_constraint_priority(const TCGOpDef *def, int k) 2038 { 2039 const TCGArgConstraint *arg_ct = &def->args_ct[k]; 2040 int n; 2041 2042 if (arg_ct->oalias) { 2043 /* an alias is equivalent to a single register */ 2044 n = 1; 2045 } else { 2046 n = ctpop64(arg_ct->regs); 2047 } 2048 return TCG_TARGET_NB_REGS - n + 1; 2049 } 2050 2051 /* sort from highest priority to lowest */ 2052 static void sort_constraints(TCGOpDef *def, int start, int n) 2053 { 2054 int i, j; 2055 TCGArgConstraint *a = def->args_ct; 2056 2057 for (i = 0; i < n; i++) { 2058 a[start + i].sort_index = start + i; 2059 } 2060 if (n <= 1) { 2061 return; 2062 } 2063 for (i = 0; i < n - 1; i++) { 2064 for (j = i + 1; j < n; j++) { 2065 int p1 = get_constraint_priority(def, a[start + i].sort_index); 2066 int p2 = get_constraint_priority(def, a[start + j].sort_index); 2067 if (p1 < p2) { 2068 int tmp = a[start + i].sort_index; 2069 a[start + i].sort_index = a[start + j].sort_index; 2070 a[start + j].sort_index = tmp; 2071 } 2072 } 2073 } 2074 } 2075 2076 static void process_op_defs(TCGContext *s) 2077 { 2078 TCGOpcode op; 2079 2080 for (op = 0; op < NB_OPS; op++) { 2081 TCGOpDef *def = &tcg_op_defs[op]; 2082 const TCGTargetOpDef *tdefs; 2083 int i, nb_args; 2084 2085 if (def->flags & TCG_OPF_NOT_PRESENT) { 2086 continue; 2087 } 2088 2089 nb_args = def->nb_iargs + def->nb_oargs; 2090 if (nb_args == 0) { 2091 continue; 2092 } 2093 2094 /* 2095 * Macro magic should make it impossible, but double-check that 2096 * the array index is in range. Since the signness of an enum 2097 * is implementation defined, force the result to unsigned. 2098 */ 2099 unsigned con_set = tcg_target_op_def(op); 2100 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 2101 tdefs = &constraint_sets[con_set]; 2102 2103 for (i = 0; i < nb_args; i++) { 2104 const char *ct_str = tdefs->args_ct_str[i]; 2105 /* Incomplete TCGTargetOpDef entry. */ 2106 tcg_debug_assert(ct_str != NULL); 2107 2108 while (*ct_str != '\0') { 2109 switch(*ct_str) { 2110 case '0' ... '9': 2111 { 2112 int oarg = *ct_str - '0'; 2113 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]); 2114 tcg_debug_assert(oarg < def->nb_oargs); 2115 tcg_debug_assert(def->args_ct[oarg].regs != 0); 2116 def->args_ct[i] = def->args_ct[oarg]; 2117 /* The output sets oalias. */ 2118 def->args_ct[oarg].oalias = true; 2119 def->args_ct[oarg].alias_index = i; 2120 /* The input sets ialias. */ 2121 def->args_ct[i].ialias = true; 2122 def->args_ct[i].alias_index = oarg; 2123 } 2124 ct_str++; 2125 break; 2126 case '&': 2127 def->args_ct[i].newreg = true; 2128 ct_str++; 2129 break; 2130 case 'i': 2131 def->args_ct[i].ct |= TCG_CT_CONST; 2132 ct_str++; 2133 break; 2134 2135 /* Include all of the target-specific constraints. */ 2136 2137 #undef CONST 2138 #define CONST(CASE, MASK) \ 2139 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break; 2140 #define REGS(CASE, MASK) \ 2141 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break; 2142 2143 #include "tcg-target-con-str.h" 2144 2145 #undef REGS 2146 #undef CONST 2147 default: 2148 /* Typo in TCGTargetOpDef constraint. */ 2149 g_assert_not_reached(); 2150 } 2151 } 2152 } 2153 2154 /* TCGTargetOpDef entry with too much information? */ 2155 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 2156 2157 /* sort the constraints (XXX: this is just an heuristic) */ 2158 sort_constraints(def, 0, def->nb_oargs); 2159 sort_constraints(def, def->nb_oargs, def->nb_iargs); 2160 } 2161 } 2162 2163 void tcg_op_remove(TCGContext *s, TCGOp *op) 2164 { 2165 TCGLabel *label; 2166 2167 switch (op->opc) { 2168 case INDEX_op_br: 2169 label = arg_label(op->args[0]); 2170 label->refs--; 2171 break; 2172 case INDEX_op_brcond_i32: 2173 case INDEX_op_brcond_i64: 2174 label = arg_label(op->args[3]); 2175 label->refs--; 2176 break; 2177 case INDEX_op_brcond2_i32: 2178 label = arg_label(op->args[5]); 2179 label->refs--; 2180 break; 2181 default: 2182 break; 2183 } 2184 2185 QTAILQ_REMOVE(&s->ops, op, link); 2186 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 2187 s->nb_ops--; 2188 2189 #ifdef CONFIG_PROFILER 2190 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1); 2191 #endif 2192 } 2193 2194 void tcg_remove_ops_after(TCGOp *op) 2195 { 2196 TCGContext *s = tcg_ctx; 2197 2198 while (true) { 2199 TCGOp *last = tcg_last_op(); 2200 if (last == op) { 2201 return; 2202 } 2203 tcg_op_remove(s, last); 2204 } 2205 } 2206 2207 static TCGOp *tcg_op_alloc(TCGOpcode opc) 2208 { 2209 TCGContext *s = tcg_ctx; 2210 TCGOp *op; 2211 2212 if (likely(QTAILQ_EMPTY(&s->free_ops))) { 2213 op = tcg_malloc(sizeof(TCGOp)); 2214 } else { 2215 op = QTAILQ_FIRST(&s->free_ops); 2216 QTAILQ_REMOVE(&s->free_ops, op, link); 2217 } 2218 memset(op, 0, offsetof(TCGOp, link)); 2219 op->opc = opc; 2220 s->nb_ops++; 2221 2222 return op; 2223 } 2224 2225 TCGOp *tcg_emit_op(TCGOpcode opc) 2226 { 2227 TCGOp *op = tcg_op_alloc(opc); 2228 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 2229 return op; 2230 } 2231 2232 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc) 2233 { 2234 TCGOp *new_op = tcg_op_alloc(opc); 2235 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 2236 return new_op; 2237 } 2238 2239 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc) 2240 { 2241 TCGOp *new_op = tcg_op_alloc(opc); 2242 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 2243 return new_op; 2244 } 2245 2246 /* Reachable analysis : remove unreachable code. */ 2247 static void reachable_code_pass(TCGContext *s) 2248 { 2249 TCGOp *op, *op_next; 2250 bool dead = false; 2251 2252 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2253 bool remove = dead; 2254 TCGLabel *label; 2255 2256 switch (op->opc) { 2257 case INDEX_op_set_label: 2258 label = arg_label(op->args[0]); 2259 if (label->refs == 0) { 2260 /* 2261 * While there is an occasional backward branch, virtually 2262 * all branches generated by the translators are forward. 2263 * Which means that generally we will have already removed 2264 * all references to the label that will be, and there is 2265 * little to be gained by iterating. 2266 */ 2267 remove = true; 2268 } else { 2269 /* Once we see a label, insns become live again. */ 2270 dead = false; 2271 remove = false; 2272 2273 /* 2274 * Optimization can fold conditional branches to unconditional. 2275 * If we find a label with one reference which is preceded by 2276 * an unconditional branch to it, remove both. This needed to 2277 * wait until the dead code in between them was removed. 2278 */ 2279 if (label->refs == 1) { 2280 TCGOp *op_prev = QTAILQ_PREV(op, link); 2281 if (op_prev->opc == INDEX_op_br && 2282 label == arg_label(op_prev->args[0])) { 2283 tcg_op_remove(s, op_prev); 2284 remove = true; 2285 } 2286 } 2287 } 2288 break; 2289 2290 case INDEX_op_br: 2291 case INDEX_op_exit_tb: 2292 case INDEX_op_goto_ptr: 2293 /* Unconditional branches; everything following is dead. */ 2294 dead = true; 2295 break; 2296 2297 case INDEX_op_call: 2298 /* Notice noreturn helper calls, raising exceptions. */ 2299 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 2300 dead = true; 2301 } 2302 break; 2303 2304 case INDEX_op_insn_start: 2305 /* Never remove -- we need to keep these for unwind. */ 2306 remove = false; 2307 break; 2308 2309 default: 2310 break; 2311 } 2312 2313 if (remove) { 2314 tcg_op_remove(s, op); 2315 } 2316 } 2317 } 2318 2319 #define TS_DEAD 1 2320 #define TS_MEM 2 2321 2322 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 2323 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 2324 2325 /* For liveness_pass_1, the register preferences for a given temp. */ 2326 static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 2327 { 2328 return ts->state_ptr; 2329 } 2330 2331 /* For liveness_pass_1, reset the preferences for a given temp to the 2332 * maximal regset for its type. 2333 */ 2334 static inline void la_reset_pref(TCGTemp *ts) 2335 { 2336 *la_temp_pref(ts) 2337 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 2338 } 2339 2340 /* liveness analysis: end of function: all temps are dead, and globals 2341 should be in memory. */ 2342 static void la_func_end(TCGContext *s, int ng, int nt) 2343 { 2344 int i; 2345 2346 for (i = 0; i < ng; ++i) { 2347 s->temps[i].state = TS_DEAD | TS_MEM; 2348 la_reset_pref(&s->temps[i]); 2349 } 2350 for (i = ng; i < nt; ++i) { 2351 s->temps[i].state = TS_DEAD; 2352 la_reset_pref(&s->temps[i]); 2353 } 2354 } 2355 2356 /* liveness analysis: end of basic block: all temps are dead, globals 2357 and local temps should be in memory. */ 2358 static void la_bb_end(TCGContext *s, int ng, int nt) 2359 { 2360 int i; 2361 2362 for (i = 0; i < nt; ++i) { 2363 TCGTemp *ts = &s->temps[i]; 2364 int state; 2365 2366 switch (ts->kind) { 2367 case TEMP_FIXED: 2368 case TEMP_GLOBAL: 2369 case TEMP_LOCAL: 2370 state = TS_DEAD | TS_MEM; 2371 break; 2372 case TEMP_NORMAL: 2373 case TEMP_CONST: 2374 state = TS_DEAD; 2375 break; 2376 default: 2377 g_assert_not_reached(); 2378 } 2379 ts->state = state; 2380 la_reset_pref(ts); 2381 } 2382 } 2383 2384 /* liveness analysis: sync globals back to memory. */ 2385 static void la_global_sync(TCGContext *s, int ng) 2386 { 2387 int i; 2388 2389 for (i = 0; i < ng; ++i) { 2390 int state = s->temps[i].state; 2391 s->temps[i].state = state | TS_MEM; 2392 if (state == TS_DEAD) { 2393 /* If the global was previously dead, reset prefs. */ 2394 la_reset_pref(&s->temps[i]); 2395 } 2396 } 2397 } 2398 2399 /* 2400 * liveness analysis: conditional branch: all temps are dead, 2401 * globals and local temps should be synced. 2402 */ 2403 static void la_bb_sync(TCGContext *s, int ng, int nt) 2404 { 2405 la_global_sync(s, ng); 2406 2407 for (int i = ng; i < nt; ++i) { 2408 TCGTemp *ts = &s->temps[i]; 2409 int state; 2410 2411 switch (ts->kind) { 2412 case TEMP_LOCAL: 2413 state = ts->state; 2414 ts->state = state | TS_MEM; 2415 if (state != TS_DEAD) { 2416 continue; 2417 } 2418 break; 2419 case TEMP_NORMAL: 2420 s->temps[i].state = TS_DEAD; 2421 break; 2422 case TEMP_CONST: 2423 continue; 2424 default: 2425 g_assert_not_reached(); 2426 } 2427 la_reset_pref(&s->temps[i]); 2428 } 2429 } 2430 2431 /* liveness analysis: sync globals back to memory and kill. */ 2432 static void la_global_kill(TCGContext *s, int ng) 2433 { 2434 int i; 2435 2436 for (i = 0; i < ng; i++) { 2437 s->temps[i].state = TS_DEAD | TS_MEM; 2438 la_reset_pref(&s->temps[i]); 2439 } 2440 } 2441 2442 /* liveness analysis: note live globals crossing calls. */ 2443 static void la_cross_call(TCGContext *s, int nt) 2444 { 2445 TCGRegSet mask = ~tcg_target_call_clobber_regs; 2446 int i; 2447 2448 for (i = 0; i < nt; i++) { 2449 TCGTemp *ts = &s->temps[i]; 2450 if (!(ts->state & TS_DEAD)) { 2451 TCGRegSet *pset = la_temp_pref(ts); 2452 TCGRegSet set = *pset; 2453 2454 set &= mask; 2455 /* If the combination is not possible, restart. */ 2456 if (set == 0) { 2457 set = tcg_target_available_regs[ts->type] & mask; 2458 } 2459 *pset = set; 2460 } 2461 } 2462 } 2463 2464 /* Liveness analysis : update the opc_arg_life array to tell if a 2465 given input arguments is dead. Instructions updating dead 2466 temporaries are removed. */ 2467 static void liveness_pass_1(TCGContext *s) 2468 { 2469 int nb_globals = s->nb_globals; 2470 int nb_temps = s->nb_temps; 2471 TCGOp *op, *op_prev; 2472 TCGRegSet *prefs; 2473 int i; 2474 2475 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 2476 for (i = 0; i < nb_temps; ++i) { 2477 s->temps[i].state_ptr = prefs + i; 2478 } 2479 2480 /* ??? Should be redundant with the exit_tb that ends the TB. */ 2481 la_func_end(s, nb_globals, nb_temps); 2482 2483 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 2484 int nb_iargs, nb_oargs; 2485 TCGOpcode opc_new, opc_new2; 2486 bool have_opc_new2; 2487 TCGLifeData arg_life = 0; 2488 TCGTemp *ts; 2489 TCGOpcode opc = op->opc; 2490 const TCGOpDef *def = &tcg_op_defs[opc]; 2491 2492 switch (opc) { 2493 case INDEX_op_call: 2494 { 2495 int call_flags; 2496 int nb_call_regs; 2497 2498 nb_oargs = TCGOP_CALLO(op); 2499 nb_iargs = TCGOP_CALLI(op); 2500 call_flags = tcg_call_flags(op); 2501 2502 /* pure functions can be removed if their result is unused */ 2503 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 2504 for (i = 0; i < nb_oargs; i++) { 2505 ts = arg_temp(op->args[i]); 2506 if (ts->state != TS_DEAD) { 2507 goto do_not_remove_call; 2508 } 2509 } 2510 goto do_remove; 2511 } 2512 do_not_remove_call: 2513 2514 /* Output args are dead. */ 2515 for (i = 0; i < nb_oargs; i++) { 2516 ts = arg_temp(op->args[i]); 2517 if (ts->state & TS_DEAD) { 2518 arg_life |= DEAD_ARG << i; 2519 } 2520 if (ts->state & TS_MEM) { 2521 arg_life |= SYNC_ARG << i; 2522 } 2523 ts->state = TS_DEAD; 2524 la_reset_pref(ts); 2525 2526 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */ 2527 op->output_pref[i] = 0; 2528 } 2529 2530 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 2531 TCG_CALL_NO_READ_GLOBALS))) { 2532 la_global_kill(s, nb_globals); 2533 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 2534 la_global_sync(s, nb_globals); 2535 } 2536 2537 /* Record arguments that die in this helper. */ 2538 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2539 ts = arg_temp(op->args[i]); 2540 if (ts && ts->state & TS_DEAD) { 2541 arg_life |= DEAD_ARG << i; 2542 } 2543 } 2544 2545 /* For all live registers, remove call-clobbered prefs. */ 2546 la_cross_call(s, nb_temps); 2547 2548 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2549 2550 /* Input arguments are live for preceding opcodes. */ 2551 for (i = 0; i < nb_iargs; i++) { 2552 ts = arg_temp(op->args[i + nb_oargs]); 2553 if (ts && ts->state & TS_DEAD) { 2554 /* For those arguments that die, and will be allocated 2555 * in registers, clear the register set for that arg, 2556 * to be filled in below. For args that will be on 2557 * the stack, reset to any available reg. 2558 */ 2559 *la_temp_pref(ts) 2560 = (i < nb_call_regs ? 0 : 2561 tcg_target_available_regs[ts->type]); 2562 ts->state &= ~TS_DEAD; 2563 } 2564 } 2565 2566 /* For each input argument, add its input register to prefs. 2567 If a temp is used once, this produces a single set bit. */ 2568 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) { 2569 ts = arg_temp(op->args[i + nb_oargs]); 2570 if (ts) { 2571 tcg_regset_set_reg(*la_temp_pref(ts), 2572 tcg_target_call_iarg_regs[i]); 2573 } 2574 } 2575 } 2576 break; 2577 case INDEX_op_insn_start: 2578 break; 2579 case INDEX_op_discard: 2580 /* mark the temporary as dead */ 2581 ts = arg_temp(op->args[0]); 2582 ts->state = TS_DEAD; 2583 la_reset_pref(ts); 2584 break; 2585 2586 case INDEX_op_add2_i32: 2587 opc_new = INDEX_op_add_i32; 2588 goto do_addsub2; 2589 case INDEX_op_sub2_i32: 2590 opc_new = INDEX_op_sub_i32; 2591 goto do_addsub2; 2592 case INDEX_op_add2_i64: 2593 opc_new = INDEX_op_add_i64; 2594 goto do_addsub2; 2595 case INDEX_op_sub2_i64: 2596 opc_new = INDEX_op_sub_i64; 2597 do_addsub2: 2598 nb_iargs = 4; 2599 nb_oargs = 2; 2600 /* Test if the high part of the operation is dead, but not 2601 the low part. The result can be optimized to a simple 2602 add or sub. This happens often for x86_64 guest when the 2603 cpu mode is set to 32 bit. */ 2604 if (arg_temp(op->args[1])->state == TS_DEAD) { 2605 if (arg_temp(op->args[0])->state == TS_DEAD) { 2606 goto do_remove; 2607 } 2608 /* Replace the opcode and adjust the args in place, 2609 leaving 3 unused args at the end. */ 2610 op->opc = opc = opc_new; 2611 op->args[1] = op->args[2]; 2612 op->args[2] = op->args[4]; 2613 /* Fall through and mark the single-word operation live. */ 2614 nb_iargs = 2; 2615 nb_oargs = 1; 2616 } 2617 goto do_not_remove; 2618 2619 case INDEX_op_mulu2_i32: 2620 opc_new = INDEX_op_mul_i32; 2621 opc_new2 = INDEX_op_muluh_i32; 2622 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 2623 goto do_mul2; 2624 case INDEX_op_muls2_i32: 2625 opc_new = INDEX_op_mul_i32; 2626 opc_new2 = INDEX_op_mulsh_i32; 2627 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 2628 goto do_mul2; 2629 case INDEX_op_mulu2_i64: 2630 opc_new = INDEX_op_mul_i64; 2631 opc_new2 = INDEX_op_muluh_i64; 2632 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 2633 goto do_mul2; 2634 case INDEX_op_muls2_i64: 2635 opc_new = INDEX_op_mul_i64; 2636 opc_new2 = INDEX_op_mulsh_i64; 2637 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 2638 goto do_mul2; 2639 do_mul2: 2640 nb_iargs = 2; 2641 nb_oargs = 2; 2642 if (arg_temp(op->args[1])->state == TS_DEAD) { 2643 if (arg_temp(op->args[0])->state == TS_DEAD) { 2644 /* Both parts of the operation are dead. */ 2645 goto do_remove; 2646 } 2647 /* The high part of the operation is dead; generate the low. */ 2648 op->opc = opc = opc_new; 2649 op->args[1] = op->args[2]; 2650 op->args[2] = op->args[3]; 2651 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) { 2652 /* The low part of the operation is dead; generate the high. */ 2653 op->opc = opc = opc_new2; 2654 op->args[0] = op->args[1]; 2655 op->args[1] = op->args[2]; 2656 op->args[2] = op->args[3]; 2657 } else { 2658 goto do_not_remove; 2659 } 2660 /* Mark the single-word operation live. */ 2661 nb_oargs = 1; 2662 goto do_not_remove; 2663 2664 default: 2665 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 2666 nb_iargs = def->nb_iargs; 2667 nb_oargs = def->nb_oargs; 2668 2669 /* Test if the operation can be removed because all 2670 its outputs are dead. We assume that nb_oargs == 0 2671 implies side effects */ 2672 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 2673 for (i = 0; i < nb_oargs; i++) { 2674 if (arg_temp(op->args[i])->state != TS_DEAD) { 2675 goto do_not_remove; 2676 } 2677 } 2678 goto do_remove; 2679 } 2680 goto do_not_remove; 2681 2682 do_remove: 2683 tcg_op_remove(s, op); 2684 break; 2685 2686 do_not_remove: 2687 for (i = 0; i < nb_oargs; i++) { 2688 ts = arg_temp(op->args[i]); 2689 2690 /* Remember the preference of the uses that followed. */ 2691 op->output_pref[i] = *la_temp_pref(ts); 2692 2693 /* Output args are dead. */ 2694 if (ts->state & TS_DEAD) { 2695 arg_life |= DEAD_ARG << i; 2696 } 2697 if (ts->state & TS_MEM) { 2698 arg_life |= SYNC_ARG << i; 2699 } 2700 ts->state = TS_DEAD; 2701 la_reset_pref(ts); 2702 } 2703 2704 /* If end of basic block, update. */ 2705 if (def->flags & TCG_OPF_BB_EXIT) { 2706 la_func_end(s, nb_globals, nb_temps); 2707 } else if (def->flags & TCG_OPF_COND_BRANCH) { 2708 la_bb_sync(s, nb_globals, nb_temps); 2709 } else if (def->flags & TCG_OPF_BB_END) { 2710 la_bb_end(s, nb_globals, nb_temps); 2711 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2712 la_global_sync(s, nb_globals); 2713 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2714 la_cross_call(s, nb_temps); 2715 } 2716 } 2717 2718 /* Record arguments that die in this opcode. */ 2719 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2720 ts = arg_temp(op->args[i]); 2721 if (ts->state & TS_DEAD) { 2722 arg_life |= DEAD_ARG << i; 2723 } 2724 } 2725 2726 /* Input arguments are live for preceding opcodes. */ 2727 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2728 ts = arg_temp(op->args[i]); 2729 if (ts->state & TS_DEAD) { 2730 /* For operands that were dead, initially allow 2731 all regs for the type. */ 2732 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 2733 ts->state &= ~TS_DEAD; 2734 } 2735 } 2736 2737 /* Incorporate constraints for this operand. */ 2738 switch (opc) { 2739 case INDEX_op_mov_i32: 2740 case INDEX_op_mov_i64: 2741 /* Note that these are TCG_OPF_NOT_PRESENT and do not 2742 have proper constraints. That said, special case 2743 moves to propagate preferences backward. */ 2744 if (IS_DEAD_ARG(1)) { 2745 *la_temp_pref(arg_temp(op->args[0])) 2746 = *la_temp_pref(arg_temp(op->args[1])); 2747 } 2748 break; 2749 2750 default: 2751 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2752 const TCGArgConstraint *ct = &def->args_ct[i]; 2753 TCGRegSet set, *pset; 2754 2755 ts = arg_temp(op->args[i]); 2756 pset = la_temp_pref(ts); 2757 set = *pset; 2758 2759 set &= ct->regs; 2760 if (ct->ialias) { 2761 set &= op->output_pref[ct->alias_index]; 2762 } 2763 /* If the combination is not possible, restart. */ 2764 if (set == 0) { 2765 set = ct->regs; 2766 } 2767 *pset = set; 2768 } 2769 break; 2770 } 2771 break; 2772 } 2773 op->life = arg_life; 2774 } 2775 } 2776 2777 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 2778 static bool liveness_pass_2(TCGContext *s) 2779 { 2780 int nb_globals = s->nb_globals; 2781 int nb_temps, i; 2782 bool changes = false; 2783 TCGOp *op, *op_next; 2784 2785 /* Create a temporary for each indirect global. */ 2786 for (i = 0; i < nb_globals; ++i) { 2787 TCGTemp *its = &s->temps[i]; 2788 if (its->indirect_reg) { 2789 TCGTemp *dts = tcg_temp_alloc(s); 2790 dts->type = its->type; 2791 dts->base_type = its->base_type; 2792 its->state_ptr = dts; 2793 } else { 2794 its->state_ptr = NULL; 2795 } 2796 /* All globals begin dead. */ 2797 its->state = TS_DEAD; 2798 } 2799 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 2800 TCGTemp *its = &s->temps[i]; 2801 its->state_ptr = NULL; 2802 its->state = TS_DEAD; 2803 } 2804 2805 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 2806 TCGOpcode opc = op->opc; 2807 const TCGOpDef *def = &tcg_op_defs[opc]; 2808 TCGLifeData arg_life = op->life; 2809 int nb_iargs, nb_oargs, call_flags; 2810 TCGTemp *arg_ts, *dir_ts; 2811 2812 if (opc == INDEX_op_call) { 2813 nb_oargs = TCGOP_CALLO(op); 2814 nb_iargs = TCGOP_CALLI(op); 2815 call_flags = tcg_call_flags(op); 2816 } else { 2817 nb_iargs = def->nb_iargs; 2818 nb_oargs = def->nb_oargs; 2819 2820 /* Set flags similar to how calls require. */ 2821 if (def->flags & TCG_OPF_COND_BRANCH) { 2822 /* Like reading globals: sync_globals */ 2823 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2824 } else if (def->flags & TCG_OPF_BB_END) { 2825 /* Like writing globals: save_globals */ 2826 call_flags = 0; 2827 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2828 /* Like reading globals: sync_globals */ 2829 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 2830 } else { 2831 /* No effect on globals. */ 2832 call_flags = (TCG_CALL_NO_READ_GLOBALS | 2833 TCG_CALL_NO_WRITE_GLOBALS); 2834 } 2835 } 2836 2837 /* Make sure that input arguments are available. */ 2838 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2839 arg_ts = arg_temp(op->args[i]); 2840 if (arg_ts) { 2841 dir_ts = arg_ts->state_ptr; 2842 if (dir_ts && arg_ts->state == TS_DEAD) { 2843 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 2844 ? INDEX_op_ld_i32 2845 : INDEX_op_ld_i64); 2846 TCGOp *lop = tcg_op_insert_before(s, op, lopc); 2847 2848 lop->args[0] = temp_arg(dir_ts); 2849 lop->args[1] = temp_arg(arg_ts->mem_base); 2850 lop->args[2] = arg_ts->mem_offset; 2851 2852 /* Loaded, but synced with memory. */ 2853 arg_ts->state = TS_MEM; 2854 } 2855 } 2856 } 2857 2858 /* Perform input replacement, and mark inputs that became dead. 2859 No action is required except keeping temp_state up to date 2860 so that we reload when needed. */ 2861 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2862 arg_ts = arg_temp(op->args[i]); 2863 if (arg_ts) { 2864 dir_ts = arg_ts->state_ptr; 2865 if (dir_ts) { 2866 op->args[i] = temp_arg(dir_ts); 2867 changes = true; 2868 if (IS_DEAD_ARG(i)) { 2869 arg_ts->state = TS_DEAD; 2870 } 2871 } 2872 } 2873 } 2874 2875 /* Liveness analysis should ensure that the following are 2876 all correct, for call sites and basic block end points. */ 2877 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 2878 /* Nothing to do */ 2879 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 2880 for (i = 0; i < nb_globals; ++i) { 2881 /* Liveness should see that globals are synced back, 2882 that is, either TS_DEAD or TS_MEM. */ 2883 arg_ts = &s->temps[i]; 2884 tcg_debug_assert(arg_ts->state_ptr == 0 2885 || arg_ts->state != 0); 2886 } 2887 } else { 2888 for (i = 0; i < nb_globals; ++i) { 2889 /* Liveness should see that globals are saved back, 2890 that is, TS_DEAD, waiting to be reloaded. */ 2891 arg_ts = &s->temps[i]; 2892 tcg_debug_assert(arg_ts->state_ptr == 0 2893 || arg_ts->state == TS_DEAD); 2894 } 2895 } 2896 2897 /* Outputs become available. */ 2898 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) { 2899 arg_ts = arg_temp(op->args[0]); 2900 dir_ts = arg_ts->state_ptr; 2901 if (dir_ts) { 2902 op->args[0] = temp_arg(dir_ts); 2903 changes = true; 2904 2905 /* The output is now live and modified. */ 2906 arg_ts->state = 0; 2907 2908 if (NEED_SYNC_ARG(0)) { 2909 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 2910 ? INDEX_op_st_i32 2911 : INDEX_op_st_i64); 2912 TCGOp *sop = tcg_op_insert_after(s, op, sopc); 2913 TCGTemp *out_ts = dir_ts; 2914 2915 if (IS_DEAD_ARG(0)) { 2916 out_ts = arg_temp(op->args[1]); 2917 arg_ts->state = TS_DEAD; 2918 tcg_op_remove(s, op); 2919 } else { 2920 arg_ts->state = TS_MEM; 2921 } 2922 2923 sop->args[0] = temp_arg(out_ts); 2924 sop->args[1] = temp_arg(arg_ts->mem_base); 2925 sop->args[2] = arg_ts->mem_offset; 2926 } else { 2927 tcg_debug_assert(!IS_DEAD_ARG(0)); 2928 } 2929 } 2930 } else { 2931 for (i = 0; i < nb_oargs; i++) { 2932 arg_ts = arg_temp(op->args[i]); 2933 dir_ts = arg_ts->state_ptr; 2934 if (!dir_ts) { 2935 continue; 2936 } 2937 op->args[i] = temp_arg(dir_ts); 2938 changes = true; 2939 2940 /* The output is now live and modified. */ 2941 arg_ts->state = 0; 2942 2943 /* Sync outputs upon their last write. */ 2944 if (NEED_SYNC_ARG(i)) { 2945 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 2946 ? INDEX_op_st_i32 2947 : INDEX_op_st_i64); 2948 TCGOp *sop = tcg_op_insert_after(s, op, sopc); 2949 2950 sop->args[0] = temp_arg(dir_ts); 2951 sop->args[1] = temp_arg(arg_ts->mem_base); 2952 sop->args[2] = arg_ts->mem_offset; 2953 2954 arg_ts->state = TS_MEM; 2955 } 2956 /* Drop outputs that are dead. */ 2957 if (IS_DEAD_ARG(i)) { 2958 arg_ts->state = TS_DEAD; 2959 } 2960 } 2961 } 2962 } 2963 2964 return changes; 2965 } 2966 2967 #ifdef CONFIG_DEBUG_TCG 2968 static void dump_regs(TCGContext *s) 2969 { 2970 TCGTemp *ts; 2971 int i; 2972 char buf[64]; 2973 2974 for(i = 0; i < s->nb_temps; i++) { 2975 ts = &s->temps[i]; 2976 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 2977 switch(ts->val_type) { 2978 case TEMP_VAL_REG: 2979 printf("%s", tcg_target_reg_names[ts->reg]); 2980 break; 2981 case TEMP_VAL_MEM: 2982 printf("%d(%s)", (int)ts->mem_offset, 2983 tcg_target_reg_names[ts->mem_base->reg]); 2984 break; 2985 case TEMP_VAL_CONST: 2986 printf("$0x%" PRIx64, ts->val); 2987 break; 2988 case TEMP_VAL_DEAD: 2989 printf("D"); 2990 break; 2991 default: 2992 printf("???"); 2993 break; 2994 } 2995 printf("\n"); 2996 } 2997 2998 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 2999 if (s->reg_to_temp[i] != NULL) { 3000 printf("%s: %s\n", 3001 tcg_target_reg_names[i], 3002 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); 3003 } 3004 } 3005 } 3006 3007 static void check_regs(TCGContext *s) 3008 { 3009 int reg; 3010 int k; 3011 TCGTemp *ts; 3012 char buf[64]; 3013 3014 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 3015 ts = s->reg_to_temp[reg]; 3016 if (ts != NULL) { 3017 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { 3018 printf("Inconsistency for register %s:\n", 3019 tcg_target_reg_names[reg]); 3020 goto fail; 3021 } 3022 } 3023 } 3024 for (k = 0; k < s->nb_temps; k++) { 3025 ts = &s->temps[k]; 3026 if (ts->val_type == TEMP_VAL_REG 3027 && ts->kind != TEMP_FIXED 3028 && s->reg_to_temp[ts->reg] != ts) { 3029 printf("Inconsistency for temp %s:\n", 3030 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 3031 fail: 3032 printf("reg state:\n"); 3033 dump_regs(s); 3034 tcg_abort(); 3035 } 3036 } 3037 } 3038 #endif 3039 3040 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 3041 { 3042 intptr_t off, size, align; 3043 3044 switch (ts->type) { 3045 case TCG_TYPE_I32: 3046 size = align = 4; 3047 break; 3048 case TCG_TYPE_I64: 3049 case TCG_TYPE_V64: 3050 size = align = 8; 3051 break; 3052 case TCG_TYPE_V128: 3053 size = align = 16; 3054 break; 3055 case TCG_TYPE_V256: 3056 /* Note that we do not require aligned storage for V256. */ 3057 size = 32, align = 16; 3058 break; 3059 default: 3060 g_assert_not_reached(); 3061 } 3062 3063 assert(align <= TCG_TARGET_STACK_ALIGN); 3064 off = ROUND_UP(s->current_frame_offset, align); 3065 3066 /* If we've exhausted the stack frame, restart with a smaller TB. */ 3067 if (off + size > s->frame_end) { 3068 tcg_raise_tb_overflow(s); 3069 } 3070 s->current_frame_offset = off + size; 3071 3072 ts->mem_offset = off; 3073 #if defined(__sparc__) 3074 ts->mem_offset += TCG_TARGET_STACK_BIAS; 3075 #endif 3076 ts->mem_base = s->frame_temp; 3077 ts->mem_allocated = 1; 3078 } 3079 3080 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 3081 3082 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 3083 mark it free; otherwise mark it dead. */ 3084 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 3085 { 3086 TCGTempVal new_type; 3087 3088 switch (ts->kind) { 3089 case TEMP_FIXED: 3090 return; 3091 case TEMP_GLOBAL: 3092 case TEMP_LOCAL: 3093 new_type = TEMP_VAL_MEM; 3094 break; 3095 case TEMP_NORMAL: 3096 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 3097 break; 3098 case TEMP_CONST: 3099 new_type = TEMP_VAL_CONST; 3100 break; 3101 default: 3102 g_assert_not_reached(); 3103 } 3104 if (ts->val_type == TEMP_VAL_REG) { 3105 s->reg_to_temp[ts->reg] = NULL; 3106 } 3107 ts->val_type = new_type; 3108 } 3109 3110 /* Mark a temporary as dead. */ 3111 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 3112 { 3113 temp_free_or_dead(s, ts, 1); 3114 } 3115 3116 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 3117 registers needs to be allocated to store a constant. If 'free_or_dead' 3118 is non-zero, subsequently release the temporary; if it is positive, the 3119 temp is dead; if it is negative, the temp is free. */ 3120 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 3121 TCGRegSet preferred_regs, int free_or_dead) 3122 { 3123 if (!temp_readonly(ts) && !ts->mem_coherent) { 3124 if (!ts->mem_allocated) { 3125 temp_allocate_frame(s, ts); 3126 } 3127 switch (ts->val_type) { 3128 case TEMP_VAL_CONST: 3129 /* If we're going to free the temp immediately, then we won't 3130 require it later in a register, so attempt to store the 3131 constant to memory directly. */ 3132 if (free_or_dead 3133 && tcg_out_sti(s, ts->type, ts->val, 3134 ts->mem_base->reg, ts->mem_offset)) { 3135 break; 3136 } 3137 temp_load(s, ts, tcg_target_available_regs[ts->type], 3138 allocated_regs, preferred_regs); 3139 /* fallthrough */ 3140 3141 case TEMP_VAL_REG: 3142 tcg_out_st(s, ts->type, ts->reg, 3143 ts->mem_base->reg, ts->mem_offset); 3144 break; 3145 3146 case TEMP_VAL_MEM: 3147 break; 3148 3149 case TEMP_VAL_DEAD: 3150 default: 3151 tcg_abort(); 3152 } 3153 ts->mem_coherent = 1; 3154 } 3155 if (free_or_dead) { 3156 temp_free_or_dead(s, ts, free_or_dead); 3157 } 3158 } 3159 3160 /* free register 'reg' by spilling the corresponding temporary if necessary */ 3161 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 3162 { 3163 TCGTemp *ts = s->reg_to_temp[reg]; 3164 if (ts != NULL) { 3165 temp_sync(s, ts, allocated_regs, 0, -1); 3166 } 3167 } 3168 3169 /** 3170 * tcg_reg_alloc: 3171 * @required_regs: Set of registers in which we must allocate. 3172 * @allocated_regs: Set of registers which must be avoided. 3173 * @preferred_regs: Set of registers we should prefer. 3174 * @rev: True if we search the registers in "indirect" order. 3175 * 3176 * The allocated register must be in @required_regs & ~@allocated_regs, 3177 * but if we can put it in @preferred_regs we may save a move later. 3178 */ 3179 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 3180 TCGRegSet allocated_regs, 3181 TCGRegSet preferred_regs, bool rev) 3182 { 3183 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 3184 TCGRegSet reg_ct[2]; 3185 const int *order; 3186 3187 reg_ct[1] = required_regs & ~allocated_regs; 3188 tcg_debug_assert(reg_ct[1] != 0); 3189 reg_ct[0] = reg_ct[1] & preferred_regs; 3190 3191 /* Skip the preferred_regs option if it cannot be satisfied, 3192 or if the preference made no difference. */ 3193 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 3194 3195 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 3196 3197 /* Try free registers, preferences first. */ 3198 for (j = f; j < 2; j++) { 3199 TCGRegSet set = reg_ct[j]; 3200 3201 if (tcg_regset_single(set)) { 3202 /* One register in the set. */ 3203 TCGReg reg = tcg_regset_first(set); 3204 if (s->reg_to_temp[reg] == NULL) { 3205 return reg; 3206 } 3207 } else { 3208 for (i = 0; i < n; i++) { 3209 TCGReg reg = order[i]; 3210 if (s->reg_to_temp[reg] == NULL && 3211 tcg_regset_test_reg(set, reg)) { 3212 return reg; 3213 } 3214 } 3215 } 3216 } 3217 3218 /* We must spill something. */ 3219 for (j = f; j < 2; j++) { 3220 TCGRegSet set = reg_ct[j]; 3221 3222 if (tcg_regset_single(set)) { 3223 /* One register in the set. */ 3224 TCGReg reg = tcg_regset_first(set); 3225 tcg_reg_free(s, reg, allocated_regs); 3226 return reg; 3227 } else { 3228 for (i = 0; i < n; i++) { 3229 TCGReg reg = order[i]; 3230 if (tcg_regset_test_reg(set, reg)) { 3231 tcg_reg_free(s, reg, allocated_regs); 3232 return reg; 3233 } 3234 } 3235 } 3236 } 3237 3238 tcg_abort(); 3239 } 3240 3241 /* Make sure the temporary is in a register. If needed, allocate the register 3242 from DESIRED while avoiding ALLOCATED. */ 3243 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 3244 TCGRegSet allocated_regs, TCGRegSet preferred_regs) 3245 { 3246 TCGReg reg; 3247 3248 switch (ts->val_type) { 3249 case TEMP_VAL_REG: 3250 return; 3251 case TEMP_VAL_CONST: 3252 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3253 preferred_regs, ts->indirect_base); 3254 if (ts->type <= TCG_TYPE_I64) { 3255 tcg_out_movi(s, ts->type, reg, ts->val); 3256 } else { 3257 uint64_t val = ts->val; 3258 MemOp vece = MO_64; 3259 3260 /* 3261 * Find the minimal vector element that matches the constant. 3262 * The targets will, in general, have to do this search anyway, 3263 * do this generically. 3264 */ 3265 if (val == dup_const(MO_8, val)) { 3266 vece = MO_8; 3267 } else if (val == dup_const(MO_16, val)) { 3268 vece = MO_16; 3269 } else if (val == dup_const(MO_32, val)) { 3270 vece = MO_32; 3271 } 3272 3273 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 3274 } 3275 ts->mem_coherent = 0; 3276 break; 3277 case TEMP_VAL_MEM: 3278 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 3279 preferred_regs, ts->indirect_base); 3280 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 3281 ts->mem_coherent = 1; 3282 break; 3283 case TEMP_VAL_DEAD: 3284 default: 3285 tcg_abort(); 3286 } 3287 ts->reg = reg; 3288 ts->val_type = TEMP_VAL_REG; 3289 s->reg_to_temp[reg] = ts; 3290 } 3291 3292 /* Save a temporary to memory. 'allocated_regs' is used in case a 3293 temporary registers needs to be allocated to store a constant. */ 3294 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 3295 { 3296 /* The liveness analysis already ensures that globals are back 3297 in memory. Keep an tcg_debug_assert for safety. */ 3298 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 3299 } 3300 3301 /* save globals to their canonical location and assume they can be 3302 modified be the following code. 'allocated_regs' is used in case a 3303 temporary registers needs to be allocated to store a constant. */ 3304 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 3305 { 3306 int i, n; 3307 3308 for (i = 0, n = s->nb_globals; i < n; i++) { 3309 temp_save(s, &s->temps[i], allocated_regs); 3310 } 3311 } 3312 3313 /* sync globals to their canonical location and assume they can be 3314 read by the following code. 'allocated_regs' is used in case a 3315 temporary registers needs to be allocated to store a constant. */ 3316 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 3317 { 3318 int i, n; 3319 3320 for (i = 0, n = s->nb_globals; i < n; i++) { 3321 TCGTemp *ts = &s->temps[i]; 3322 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 3323 || ts->kind == TEMP_FIXED 3324 || ts->mem_coherent); 3325 } 3326 } 3327 3328 /* at the end of a basic block, we assume all temporaries are dead and 3329 all globals are stored at their canonical location. */ 3330 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 3331 { 3332 int i; 3333 3334 for (i = s->nb_globals; i < s->nb_temps; i++) { 3335 TCGTemp *ts = &s->temps[i]; 3336 3337 switch (ts->kind) { 3338 case TEMP_LOCAL: 3339 temp_save(s, ts, allocated_regs); 3340 break; 3341 case TEMP_NORMAL: 3342 /* The liveness analysis already ensures that temps are dead. 3343 Keep an tcg_debug_assert for safety. */ 3344 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3345 break; 3346 case TEMP_CONST: 3347 /* Similarly, we should have freed any allocated register. */ 3348 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 3349 break; 3350 default: 3351 g_assert_not_reached(); 3352 } 3353 } 3354 3355 save_globals(s, allocated_regs); 3356 } 3357 3358 /* 3359 * At a conditional branch, we assume all temporaries are dead and 3360 * all globals and local temps are synced to their location. 3361 */ 3362 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 3363 { 3364 sync_globals(s, allocated_regs); 3365 3366 for (int i = s->nb_globals; i < s->nb_temps; i++) { 3367 TCGTemp *ts = &s->temps[i]; 3368 /* 3369 * The liveness analysis already ensures that temps are dead. 3370 * Keep tcg_debug_asserts for safety. 3371 */ 3372 switch (ts->kind) { 3373 case TEMP_LOCAL: 3374 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 3375 break; 3376 case TEMP_NORMAL: 3377 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 3378 break; 3379 case TEMP_CONST: 3380 break; 3381 default: 3382 g_assert_not_reached(); 3383 } 3384 } 3385 } 3386 3387 /* 3388 * Specialized code generation for INDEX_op_mov_* with a constant. 3389 */ 3390 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 3391 tcg_target_ulong val, TCGLifeData arg_life, 3392 TCGRegSet preferred_regs) 3393 { 3394 /* ENV should not be modified. */ 3395 tcg_debug_assert(!temp_readonly(ots)); 3396 3397 /* The movi is not explicitly generated here. */ 3398 if (ots->val_type == TEMP_VAL_REG) { 3399 s->reg_to_temp[ots->reg] = NULL; 3400 } 3401 ots->val_type = TEMP_VAL_CONST; 3402 ots->val = val; 3403 ots->mem_coherent = 0; 3404 if (NEED_SYNC_ARG(0)) { 3405 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 3406 } else if (IS_DEAD_ARG(0)) { 3407 temp_dead(s, ots); 3408 } 3409 } 3410 3411 /* 3412 * Specialized code generation for INDEX_op_mov_*. 3413 */ 3414 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 3415 { 3416 const TCGLifeData arg_life = op->life; 3417 TCGRegSet allocated_regs, preferred_regs; 3418 TCGTemp *ts, *ots; 3419 TCGType otype, itype; 3420 3421 allocated_regs = s->reserved_regs; 3422 preferred_regs = op->output_pref[0]; 3423 ots = arg_temp(op->args[0]); 3424 ts = arg_temp(op->args[1]); 3425 3426 /* ENV should not be modified. */ 3427 tcg_debug_assert(!temp_readonly(ots)); 3428 3429 /* Note that otype != itype for no-op truncation. */ 3430 otype = ots->type; 3431 itype = ts->type; 3432 3433 if (ts->val_type == TEMP_VAL_CONST) { 3434 /* propagate constant or generate sti */ 3435 tcg_target_ulong val = ts->val; 3436 if (IS_DEAD_ARG(1)) { 3437 temp_dead(s, ts); 3438 } 3439 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 3440 return; 3441 } 3442 3443 /* If the source value is in memory we're going to be forced 3444 to have it in a register in order to perform the copy. Copy 3445 the SOURCE value into its own register first, that way we 3446 don't have to reload SOURCE the next time it is used. */ 3447 if (ts->val_type == TEMP_VAL_MEM) { 3448 temp_load(s, ts, tcg_target_available_regs[itype], 3449 allocated_regs, preferred_regs); 3450 } 3451 3452 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 3453 if (IS_DEAD_ARG(0)) { 3454 /* mov to a non-saved dead register makes no sense (even with 3455 liveness analysis disabled). */ 3456 tcg_debug_assert(NEED_SYNC_ARG(0)); 3457 if (!ots->mem_allocated) { 3458 temp_allocate_frame(s, ots); 3459 } 3460 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); 3461 if (IS_DEAD_ARG(1)) { 3462 temp_dead(s, ts); 3463 } 3464 temp_dead(s, ots); 3465 } else { 3466 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 3467 /* the mov can be suppressed */ 3468 if (ots->val_type == TEMP_VAL_REG) { 3469 s->reg_to_temp[ots->reg] = NULL; 3470 } 3471 ots->reg = ts->reg; 3472 temp_dead(s, ts); 3473 } else { 3474 if (ots->val_type != TEMP_VAL_REG) { 3475 /* When allocating a new register, make sure to not spill the 3476 input one. */ 3477 tcg_regset_set_reg(allocated_regs, ts->reg); 3478 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 3479 allocated_regs, preferred_regs, 3480 ots->indirect_base); 3481 } 3482 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) { 3483 /* 3484 * Cross register class move not supported. 3485 * Store the source register into the destination slot 3486 * and leave the destination temp as TEMP_VAL_MEM. 3487 */ 3488 assert(!temp_readonly(ots)); 3489 if (!ts->mem_allocated) { 3490 temp_allocate_frame(s, ots); 3491 } 3492 tcg_out_st(s, ts->type, ts->reg, 3493 ots->mem_base->reg, ots->mem_offset); 3494 ots->mem_coherent = 1; 3495 temp_free_or_dead(s, ots, -1); 3496 return; 3497 } 3498 } 3499 ots->val_type = TEMP_VAL_REG; 3500 ots->mem_coherent = 0; 3501 s->reg_to_temp[ots->reg] = ots; 3502 if (NEED_SYNC_ARG(0)) { 3503 temp_sync(s, ots, allocated_regs, 0, 0); 3504 } 3505 } 3506 } 3507 3508 /* 3509 * Specialized code generation for INDEX_op_dup_vec. 3510 */ 3511 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 3512 { 3513 const TCGLifeData arg_life = op->life; 3514 TCGRegSet dup_out_regs, dup_in_regs; 3515 TCGTemp *its, *ots; 3516 TCGType itype, vtype; 3517 intptr_t endian_fixup; 3518 unsigned vece; 3519 bool ok; 3520 3521 ots = arg_temp(op->args[0]); 3522 its = arg_temp(op->args[1]); 3523 3524 /* ENV should not be modified. */ 3525 tcg_debug_assert(!temp_readonly(ots)); 3526 3527 itype = its->type; 3528 vece = TCGOP_VECE(op); 3529 vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 3530 3531 if (its->val_type == TEMP_VAL_CONST) { 3532 /* Propagate constant via movi -> dupi. */ 3533 tcg_target_ulong val = its->val; 3534 if (IS_DEAD_ARG(1)) { 3535 temp_dead(s, its); 3536 } 3537 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]); 3538 return; 3539 } 3540 3541 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 3542 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs; 3543 3544 /* Allocate the output register now. */ 3545 if (ots->val_type != TEMP_VAL_REG) { 3546 TCGRegSet allocated_regs = s->reserved_regs; 3547 3548 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 3549 /* Make sure to not spill the input register. */ 3550 tcg_regset_set_reg(allocated_regs, its->reg); 3551 } 3552 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 3553 op->output_pref[0], ots->indirect_base); 3554 ots->val_type = TEMP_VAL_REG; 3555 ots->mem_coherent = 0; 3556 s->reg_to_temp[ots->reg] = ots; 3557 } 3558 3559 switch (its->val_type) { 3560 case TEMP_VAL_REG: 3561 /* 3562 * The dup constriaints must be broad, covering all possible VECE. 3563 * However, tcg_op_dup_vec() gets to see the VECE and we allow it 3564 * to fail, indicating that extra moves are required for that case. 3565 */ 3566 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 3567 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 3568 goto done; 3569 } 3570 /* Try again from memory or a vector input register. */ 3571 } 3572 if (!its->mem_coherent) { 3573 /* 3574 * The input register is not synced, and so an extra store 3575 * would be required to use memory. Attempt an integer-vector 3576 * register move first. We do not have a TCGRegSet for this. 3577 */ 3578 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 3579 break; 3580 } 3581 /* Sync the temp back to its slot and load from there. */ 3582 temp_sync(s, its, s->reserved_regs, 0, 0); 3583 } 3584 /* fall through */ 3585 3586 case TEMP_VAL_MEM: 3587 #ifdef HOST_WORDS_BIGENDIAN 3588 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8; 3589 endian_fixup -= 1 << vece; 3590 #else 3591 endian_fixup = 0; 3592 #endif 3593 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 3594 its->mem_offset + endian_fixup)) { 3595 goto done; 3596 } 3597 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 3598 break; 3599 3600 default: 3601 g_assert_not_reached(); 3602 } 3603 3604 /* We now have a vector input register, so dup must succeed. */ 3605 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 3606 tcg_debug_assert(ok); 3607 3608 done: 3609 if (IS_DEAD_ARG(1)) { 3610 temp_dead(s, its); 3611 } 3612 if (NEED_SYNC_ARG(0)) { 3613 temp_sync(s, ots, s->reserved_regs, 0, 0); 3614 } 3615 if (IS_DEAD_ARG(0)) { 3616 temp_dead(s, ots); 3617 } 3618 } 3619 3620 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 3621 { 3622 const TCGLifeData arg_life = op->life; 3623 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 3624 TCGRegSet i_allocated_regs; 3625 TCGRegSet o_allocated_regs; 3626 int i, k, nb_iargs, nb_oargs; 3627 TCGReg reg; 3628 TCGArg arg; 3629 const TCGArgConstraint *arg_ct; 3630 TCGTemp *ts; 3631 TCGArg new_args[TCG_MAX_OP_ARGS]; 3632 int const_args[TCG_MAX_OP_ARGS]; 3633 3634 nb_oargs = def->nb_oargs; 3635 nb_iargs = def->nb_iargs; 3636 3637 /* copy constants */ 3638 memcpy(new_args + nb_oargs + nb_iargs, 3639 op->args + nb_oargs + nb_iargs, 3640 sizeof(TCGArg) * def->nb_cargs); 3641 3642 i_allocated_regs = s->reserved_regs; 3643 o_allocated_regs = s->reserved_regs; 3644 3645 /* satisfy input constraints */ 3646 for (k = 0; k < nb_iargs; k++) { 3647 TCGRegSet i_preferred_regs, o_preferred_regs; 3648 3649 i = def->args_ct[nb_oargs + k].sort_index; 3650 arg = op->args[i]; 3651 arg_ct = &def->args_ct[i]; 3652 ts = arg_temp(arg); 3653 3654 if (ts->val_type == TEMP_VAL_CONST 3655 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) { 3656 /* constant is OK for instruction */ 3657 const_args[i] = 1; 3658 new_args[i] = ts->val; 3659 continue; 3660 } 3661 3662 i_preferred_regs = o_preferred_regs = 0; 3663 if (arg_ct->ialias) { 3664 o_preferred_regs = op->output_pref[arg_ct->alias_index]; 3665 3666 /* 3667 * If the input is readonly, then it cannot also be an 3668 * output and aliased to itself. If the input is not 3669 * dead after the instruction, we must allocate a new 3670 * register and move it. 3671 */ 3672 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { 3673 goto allocate_in_reg; 3674 } 3675 3676 /* 3677 * Check if the current register has already been allocated 3678 * for another input aliased to an output. 3679 */ 3680 if (ts->val_type == TEMP_VAL_REG) { 3681 reg = ts->reg; 3682 for (int k2 = 0; k2 < k; k2++) { 3683 int i2 = def->args_ct[nb_oargs + k2].sort_index; 3684 if (def->args_ct[i2].ialias && reg == new_args[i2]) { 3685 goto allocate_in_reg; 3686 } 3687 } 3688 } 3689 i_preferred_regs = o_preferred_regs; 3690 } 3691 3692 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs); 3693 reg = ts->reg; 3694 3695 if (!tcg_regset_test_reg(arg_ct->regs, reg)) { 3696 allocate_in_reg: 3697 /* 3698 * Allocate a new register matching the constraint 3699 * and move the temporary register into it. 3700 */ 3701 temp_load(s, ts, tcg_target_available_regs[ts->type], 3702 i_allocated_regs, 0); 3703 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs, 3704 o_preferred_regs, ts->indirect_base); 3705 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 3706 /* 3707 * Cross register class move not supported. Sync the 3708 * temp back to its slot and load from there. 3709 */ 3710 temp_sync(s, ts, i_allocated_regs, 0, 0); 3711 tcg_out_ld(s, ts->type, reg, 3712 ts->mem_base->reg, ts->mem_offset); 3713 } 3714 } 3715 new_args[i] = reg; 3716 const_args[i] = 0; 3717 tcg_regset_set_reg(i_allocated_regs, reg); 3718 } 3719 3720 /* mark dead temporaries and free the associated registers */ 3721 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 3722 if (IS_DEAD_ARG(i)) { 3723 temp_dead(s, arg_temp(op->args[i])); 3724 } 3725 } 3726 3727 if (def->flags & TCG_OPF_COND_BRANCH) { 3728 tcg_reg_alloc_cbranch(s, i_allocated_regs); 3729 } else if (def->flags & TCG_OPF_BB_END) { 3730 tcg_reg_alloc_bb_end(s, i_allocated_regs); 3731 } else { 3732 if (def->flags & TCG_OPF_CALL_CLOBBER) { 3733 /* XXX: permit generic clobber register list ? */ 3734 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3735 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 3736 tcg_reg_free(s, i, i_allocated_regs); 3737 } 3738 } 3739 } 3740 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 3741 /* sync globals if the op has side effects and might trigger 3742 an exception. */ 3743 sync_globals(s, i_allocated_regs); 3744 } 3745 3746 /* satisfy the output constraints */ 3747 for(k = 0; k < nb_oargs; k++) { 3748 i = def->args_ct[k].sort_index; 3749 arg = op->args[i]; 3750 arg_ct = &def->args_ct[i]; 3751 ts = arg_temp(arg); 3752 3753 /* ENV should not be modified. */ 3754 tcg_debug_assert(!temp_readonly(ts)); 3755 3756 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 3757 reg = new_args[arg_ct->alias_index]; 3758 } else if (arg_ct->newreg) { 3759 reg = tcg_reg_alloc(s, arg_ct->regs, 3760 i_allocated_regs | o_allocated_regs, 3761 op->output_pref[k], ts->indirect_base); 3762 } else { 3763 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 3764 op->output_pref[k], ts->indirect_base); 3765 } 3766 tcg_regset_set_reg(o_allocated_regs, reg); 3767 if (ts->val_type == TEMP_VAL_REG) { 3768 s->reg_to_temp[ts->reg] = NULL; 3769 } 3770 ts->val_type = TEMP_VAL_REG; 3771 ts->reg = reg; 3772 /* 3773 * Temp value is modified, so the value kept in memory is 3774 * potentially not the same. 3775 */ 3776 ts->mem_coherent = 0; 3777 s->reg_to_temp[reg] = ts; 3778 new_args[i] = reg; 3779 } 3780 } 3781 3782 /* emit instruction */ 3783 if (def->flags & TCG_OPF_VECTOR) { 3784 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op), 3785 new_args, const_args); 3786 } else { 3787 tcg_out_op(s, op->opc, new_args, const_args); 3788 } 3789 3790 /* move the outputs in the correct register if needed */ 3791 for(i = 0; i < nb_oargs; i++) { 3792 ts = arg_temp(op->args[i]); 3793 3794 /* ENV should not be modified. */ 3795 tcg_debug_assert(!temp_readonly(ts)); 3796 3797 if (NEED_SYNC_ARG(i)) { 3798 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 3799 } else if (IS_DEAD_ARG(i)) { 3800 temp_dead(s, ts); 3801 } 3802 } 3803 } 3804 3805 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 3806 { 3807 const TCGLifeData arg_life = op->life; 3808 TCGTemp *ots, *itsl, *itsh; 3809 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64; 3810 3811 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 3812 tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 3813 tcg_debug_assert(TCGOP_VECE(op) == MO_64); 3814 3815 ots = arg_temp(op->args[0]); 3816 itsl = arg_temp(op->args[1]); 3817 itsh = arg_temp(op->args[2]); 3818 3819 /* ENV should not be modified. */ 3820 tcg_debug_assert(!temp_readonly(ots)); 3821 3822 /* Allocate the output register now. */ 3823 if (ots->val_type != TEMP_VAL_REG) { 3824 TCGRegSet allocated_regs = s->reserved_regs; 3825 TCGRegSet dup_out_regs = 3826 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; 3827 3828 /* Make sure to not spill the input registers. */ 3829 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 3830 tcg_regset_set_reg(allocated_regs, itsl->reg); 3831 } 3832 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 3833 tcg_regset_set_reg(allocated_regs, itsh->reg); 3834 } 3835 3836 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 3837 op->output_pref[0], ots->indirect_base); 3838 ots->val_type = TEMP_VAL_REG; 3839 ots->mem_coherent = 0; 3840 s->reg_to_temp[ots->reg] = ots; 3841 } 3842 3843 /* Promote dup2 of immediates to dupi_vec. */ 3844 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 3845 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 3846 MemOp vece = MO_64; 3847 3848 if (val == dup_const(MO_8, val)) { 3849 vece = MO_8; 3850 } else if (val == dup_const(MO_16, val)) { 3851 vece = MO_16; 3852 } else if (val == dup_const(MO_32, val)) { 3853 vece = MO_32; 3854 } 3855 3856 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 3857 goto done; 3858 } 3859 3860 /* If the two inputs form one 64-bit value, try dupm_vec. */ 3861 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) { 3862 if (!itsl->mem_coherent) { 3863 temp_sync(s, itsl, s->reserved_regs, 0, 0); 3864 } 3865 if (!itsh->mem_coherent) { 3866 temp_sync(s, itsh, s->reserved_regs, 0, 0); 3867 } 3868 #ifdef HOST_WORDS_BIGENDIAN 3869 TCGTemp *its = itsh; 3870 #else 3871 TCGTemp *its = itsl; 3872 #endif 3873 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 3874 its->mem_base->reg, its->mem_offset)) { 3875 goto done; 3876 } 3877 } 3878 3879 /* Fall back to generic expansion. */ 3880 return false; 3881 3882 done: 3883 if (IS_DEAD_ARG(1)) { 3884 temp_dead(s, itsl); 3885 } 3886 if (IS_DEAD_ARG(2)) { 3887 temp_dead(s, itsh); 3888 } 3889 if (NEED_SYNC_ARG(0)) { 3890 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 3891 } else if (IS_DEAD_ARG(0)) { 3892 temp_dead(s, ots); 3893 } 3894 return true; 3895 } 3896 3897 #ifdef TCG_TARGET_STACK_GROWSUP 3898 #define STACK_DIR(x) (-(x)) 3899 #else 3900 #define STACK_DIR(x) (x) 3901 #endif 3902 3903 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 3904 { 3905 const int nb_oargs = TCGOP_CALLO(op); 3906 const int nb_iargs = TCGOP_CALLI(op); 3907 const TCGLifeData arg_life = op->life; 3908 const TCGHelperInfo *info; 3909 int flags, nb_regs, i; 3910 TCGReg reg; 3911 TCGArg arg; 3912 TCGTemp *ts; 3913 intptr_t stack_offset; 3914 size_t call_stack_size; 3915 tcg_insn_unit *func_addr; 3916 int allocate_args; 3917 TCGRegSet allocated_regs; 3918 3919 func_addr = tcg_call_func(op); 3920 info = tcg_call_info(op); 3921 flags = info->flags; 3922 3923 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 3924 if (nb_regs > nb_iargs) { 3925 nb_regs = nb_iargs; 3926 } 3927 3928 /* assign stack slots first */ 3929 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 3930 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 3931 ~(TCG_TARGET_STACK_ALIGN - 1); 3932 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 3933 if (allocate_args) { 3934 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 3935 preallocate call stack */ 3936 tcg_abort(); 3937 } 3938 3939 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 3940 for (i = nb_regs; i < nb_iargs; i++) { 3941 arg = op->args[nb_oargs + i]; 3942 #ifdef TCG_TARGET_STACK_GROWSUP 3943 stack_offset -= sizeof(tcg_target_long); 3944 #endif 3945 if (arg != TCG_CALL_DUMMY_ARG) { 3946 ts = arg_temp(arg); 3947 temp_load(s, ts, tcg_target_available_regs[ts->type], 3948 s->reserved_regs, 0); 3949 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 3950 } 3951 #ifndef TCG_TARGET_STACK_GROWSUP 3952 stack_offset += sizeof(tcg_target_long); 3953 #endif 3954 } 3955 3956 /* assign input registers */ 3957 allocated_regs = s->reserved_regs; 3958 for (i = 0; i < nb_regs; i++) { 3959 arg = op->args[nb_oargs + i]; 3960 if (arg != TCG_CALL_DUMMY_ARG) { 3961 ts = arg_temp(arg); 3962 reg = tcg_target_call_iarg_regs[i]; 3963 3964 if (ts->val_type == TEMP_VAL_REG) { 3965 if (ts->reg != reg) { 3966 tcg_reg_free(s, reg, allocated_regs); 3967 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 3968 /* 3969 * Cross register class move not supported. Sync the 3970 * temp back to its slot and load from there. 3971 */ 3972 temp_sync(s, ts, allocated_regs, 0, 0); 3973 tcg_out_ld(s, ts->type, reg, 3974 ts->mem_base->reg, ts->mem_offset); 3975 } 3976 } 3977 } else { 3978 TCGRegSet arg_set = 0; 3979 3980 tcg_reg_free(s, reg, allocated_regs); 3981 tcg_regset_set_reg(arg_set, reg); 3982 temp_load(s, ts, arg_set, allocated_regs, 0); 3983 } 3984 3985 tcg_regset_set_reg(allocated_regs, reg); 3986 } 3987 } 3988 3989 /* mark dead temporaries and free the associated registers */ 3990 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 3991 if (IS_DEAD_ARG(i)) { 3992 temp_dead(s, arg_temp(op->args[i])); 3993 } 3994 } 3995 3996 /* clobber call registers */ 3997 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 3998 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 3999 tcg_reg_free(s, i, allocated_regs); 4000 } 4001 } 4002 4003 /* Save globals if they might be written by the helper, sync them if 4004 they might be read. */ 4005 if (flags & TCG_CALL_NO_READ_GLOBALS) { 4006 /* Nothing to do */ 4007 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 4008 sync_globals(s, allocated_regs); 4009 } else { 4010 save_globals(s, allocated_regs); 4011 } 4012 4013 #ifdef CONFIG_TCG_INTERPRETER 4014 { 4015 gpointer hash = (gpointer)(uintptr_t)info->typemask; 4016 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash); 4017 assert(cif != NULL); 4018 tcg_out_call(s, func_addr, cif); 4019 } 4020 #else 4021 tcg_out_call(s, func_addr); 4022 #endif 4023 4024 /* assign output registers and emit moves if needed */ 4025 for(i = 0; i < nb_oargs; i++) { 4026 arg = op->args[i]; 4027 ts = arg_temp(arg); 4028 4029 /* ENV should not be modified. */ 4030 tcg_debug_assert(!temp_readonly(ts)); 4031 4032 reg = tcg_target_call_oarg_regs[i]; 4033 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 4034 if (ts->val_type == TEMP_VAL_REG) { 4035 s->reg_to_temp[ts->reg] = NULL; 4036 } 4037 ts->val_type = TEMP_VAL_REG; 4038 ts->reg = reg; 4039 ts->mem_coherent = 0; 4040 s->reg_to_temp[reg] = ts; 4041 if (NEED_SYNC_ARG(i)) { 4042 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i)); 4043 } else if (IS_DEAD_ARG(i)) { 4044 temp_dead(s, ts); 4045 } 4046 } 4047 } 4048 4049 #ifdef CONFIG_PROFILER 4050 4051 /* avoid copy/paste errors */ 4052 #define PROF_ADD(to, from, field) \ 4053 do { \ 4054 (to)->field += qatomic_read(&((from)->field)); \ 4055 } while (0) 4056 4057 #define PROF_MAX(to, from, field) \ 4058 do { \ 4059 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \ 4060 if (val__ > (to)->field) { \ 4061 (to)->field = val__; \ 4062 } \ 4063 } while (0) 4064 4065 /* Pass in a zero'ed @prof */ 4066 static inline 4067 void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table) 4068 { 4069 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4070 unsigned int i; 4071 4072 for (i = 0; i < n_ctxs; i++) { 4073 TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4074 const TCGProfile *orig = &s->prof; 4075 4076 if (counters) { 4077 PROF_ADD(prof, orig, cpu_exec_time); 4078 PROF_ADD(prof, orig, tb_count1); 4079 PROF_ADD(prof, orig, tb_count); 4080 PROF_ADD(prof, orig, op_count); 4081 PROF_MAX(prof, orig, op_count_max); 4082 PROF_ADD(prof, orig, temp_count); 4083 PROF_MAX(prof, orig, temp_count_max); 4084 PROF_ADD(prof, orig, del_op_count); 4085 PROF_ADD(prof, orig, code_in_len); 4086 PROF_ADD(prof, orig, code_out_len); 4087 PROF_ADD(prof, orig, search_out_len); 4088 PROF_ADD(prof, orig, interm_time); 4089 PROF_ADD(prof, orig, code_time); 4090 PROF_ADD(prof, orig, la_time); 4091 PROF_ADD(prof, orig, opt_time); 4092 PROF_ADD(prof, orig, restore_count); 4093 PROF_ADD(prof, orig, restore_time); 4094 } 4095 if (table) { 4096 int i; 4097 4098 for (i = 0; i < NB_OPS; i++) { 4099 PROF_ADD(prof, orig, table_op_count[i]); 4100 } 4101 } 4102 } 4103 } 4104 4105 #undef PROF_ADD 4106 #undef PROF_MAX 4107 4108 static void tcg_profile_snapshot_counters(TCGProfile *prof) 4109 { 4110 tcg_profile_snapshot(prof, true, false); 4111 } 4112 4113 static void tcg_profile_snapshot_table(TCGProfile *prof) 4114 { 4115 tcg_profile_snapshot(prof, false, true); 4116 } 4117 4118 void tcg_dump_op_count(void) 4119 { 4120 TCGProfile prof = {}; 4121 int i; 4122 4123 tcg_profile_snapshot_table(&prof); 4124 for (i = 0; i < NB_OPS; i++) { 4125 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name, 4126 prof.table_op_count[i]); 4127 } 4128 } 4129 4130 int64_t tcg_cpu_exec_time(void) 4131 { 4132 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs); 4133 unsigned int i; 4134 int64_t ret = 0; 4135 4136 for (i = 0; i < n_ctxs; i++) { 4137 const TCGContext *s = qatomic_read(&tcg_ctxs[i]); 4138 const TCGProfile *prof = &s->prof; 4139 4140 ret += qatomic_read(&prof->cpu_exec_time); 4141 } 4142 return ret; 4143 } 4144 #else 4145 void tcg_dump_op_count(void) 4146 { 4147 qemu_printf("[TCG profiler not compiled]\n"); 4148 } 4149 4150 int64_t tcg_cpu_exec_time(void) 4151 { 4152 error_report("%s: TCG profiler not compiled", __func__); 4153 exit(EXIT_FAILURE); 4154 } 4155 #endif 4156 4157 4158 int tcg_gen_code(TCGContext *s, TranslationBlock *tb) 4159 { 4160 #ifdef CONFIG_PROFILER 4161 TCGProfile *prof = &s->prof; 4162 #endif 4163 int i, num_insns; 4164 TCGOp *op; 4165 4166 #ifdef CONFIG_PROFILER 4167 { 4168 int n = 0; 4169 4170 QTAILQ_FOREACH(op, &s->ops, link) { 4171 n++; 4172 } 4173 qatomic_set(&prof->op_count, prof->op_count + n); 4174 if (n > prof->op_count_max) { 4175 qatomic_set(&prof->op_count_max, n); 4176 } 4177 4178 n = s->nb_temps; 4179 qatomic_set(&prof->temp_count, prof->temp_count + n); 4180 if (n > prof->temp_count_max) { 4181 qatomic_set(&prof->temp_count_max, n); 4182 } 4183 } 4184 #endif 4185 4186 #ifdef DEBUG_DISAS 4187 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 4188 && qemu_log_in_addr_range(tb->pc))) { 4189 FILE *logfile = qemu_log_lock(); 4190 qemu_log("OP:\n"); 4191 tcg_dump_ops(s, false); 4192 qemu_log("\n"); 4193 qemu_log_unlock(logfile); 4194 } 4195 #endif 4196 4197 #ifdef CONFIG_DEBUG_TCG 4198 /* Ensure all labels referenced have been emitted. */ 4199 { 4200 TCGLabel *l; 4201 bool error = false; 4202 4203 QSIMPLEQ_FOREACH(l, &s->labels, next) { 4204 if (unlikely(!l->present) && l->refs) { 4205 qemu_log_mask(CPU_LOG_TB_OP, 4206 "$L%d referenced but not present.\n", l->id); 4207 error = true; 4208 } 4209 } 4210 assert(!error); 4211 } 4212 #endif 4213 4214 #ifdef CONFIG_PROFILER 4215 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock()); 4216 #endif 4217 4218 #ifdef USE_TCG_OPTIMIZATIONS 4219 tcg_optimize(s); 4220 #endif 4221 4222 #ifdef CONFIG_PROFILER 4223 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock()); 4224 qatomic_set(&prof->la_time, prof->la_time - profile_getclock()); 4225 #endif 4226 4227 reachable_code_pass(s); 4228 liveness_pass_1(s); 4229 4230 if (s->nb_indirects > 0) { 4231 #ifdef DEBUG_DISAS 4232 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 4233 && qemu_log_in_addr_range(tb->pc))) { 4234 FILE *logfile = qemu_log_lock(); 4235 qemu_log("OP before indirect lowering:\n"); 4236 tcg_dump_ops(s, false); 4237 qemu_log("\n"); 4238 qemu_log_unlock(logfile); 4239 } 4240 #endif 4241 /* Replace indirect temps with direct temps. */ 4242 if (liveness_pass_2(s)) { 4243 /* If changes were made, re-run liveness. */ 4244 liveness_pass_1(s); 4245 } 4246 } 4247 4248 #ifdef CONFIG_PROFILER 4249 qatomic_set(&prof->la_time, prof->la_time + profile_getclock()); 4250 #endif 4251 4252 #ifdef DEBUG_DISAS 4253 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 4254 && qemu_log_in_addr_range(tb->pc))) { 4255 FILE *logfile = qemu_log_lock(); 4256 qemu_log("OP after optimization and liveness analysis:\n"); 4257 tcg_dump_ops(s, true); 4258 qemu_log("\n"); 4259 qemu_log_unlock(logfile); 4260 } 4261 #endif 4262 4263 tcg_reg_alloc_start(s); 4264 4265 /* 4266 * Reset the buffer pointers when restarting after overflow. 4267 * TODO: Move this into translate-all.c with the rest of the 4268 * buffer management. Having only this done here is confusing. 4269 */ 4270 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 4271 s->code_ptr = s->code_buf; 4272 4273 #ifdef TCG_TARGET_NEED_LDST_LABELS 4274 QSIMPLEQ_INIT(&s->ldst_labels); 4275 #endif 4276 #ifdef TCG_TARGET_NEED_POOL_LABELS 4277 s->pool_labels = NULL; 4278 #endif 4279 4280 num_insns = -1; 4281 QTAILQ_FOREACH(op, &s->ops, link) { 4282 TCGOpcode opc = op->opc; 4283 4284 #ifdef CONFIG_PROFILER 4285 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1); 4286 #endif 4287 4288 switch (opc) { 4289 case INDEX_op_mov_i32: 4290 case INDEX_op_mov_i64: 4291 case INDEX_op_mov_vec: 4292 tcg_reg_alloc_mov(s, op); 4293 break; 4294 case INDEX_op_dup_vec: 4295 tcg_reg_alloc_dup(s, op); 4296 break; 4297 case INDEX_op_insn_start: 4298 if (num_insns >= 0) { 4299 size_t off = tcg_current_code_size(s); 4300 s->gen_insn_end_off[num_insns] = off; 4301 /* Assert that we do not overflow our stored offset. */ 4302 assert(s->gen_insn_end_off[num_insns] == off); 4303 } 4304 num_insns++; 4305 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 4306 target_ulong a; 4307 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 4308 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]); 4309 #else 4310 a = op->args[i]; 4311 #endif 4312 s->gen_insn_data[num_insns][i] = a; 4313 } 4314 break; 4315 case INDEX_op_discard: 4316 temp_dead(s, arg_temp(op->args[0])); 4317 break; 4318 case INDEX_op_set_label: 4319 tcg_reg_alloc_bb_end(s, s->reserved_regs); 4320 tcg_out_label(s, arg_label(op->args[0])); 4321 break; 4322 case INDEX_op_call: 4323 tcg_reg_alloc_call(s, op); 4324 break; 4325 case INDEX_op_dup2_vec: 4326 if (tcg_reg_alloc_dup2(s, op)) { 4327 break; 4328 } 4329 /* fall through */ 4330 default: 4331 /* Sanity check that we've not introduced any unhandled opcodes. */ 4332 tcg_debug_assert(tcg_op_supported(opc)); 4333 /* Note: in order to speed up the code, it would be much 4334 faster to have specialized register allocator functions for 4335 some common argument patterns */ 4336 tcg_reg_alloc_op(s, op); 4337 break; 4338 } 4339 #ifdef CONFIG_DEBUG_TCG 4340 check_regs(s); 4341 #endif 4342 /* Test for (pending) buffer overflow. The assumption is that any 4343 one operation beginning below the high water mark cannot overrun 4344 the buffer completely. Thus we can test for overflow after 4345 generating code without having to check during generation. */ 4346 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 4347 return -1; 4348 } 4349 /* Test for TB overflow, as seen by gen_insn_end_off. */ 4350 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 4351 return -2; 4352 } 4353 } 4354 tcg_debug_assert(num_insns >= 0); 4355 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 4356 4357 /* Generate TB finalization at the end of block */ 4358 #ifdef TCG_TARGET_NEED_LDST_LABELS 4359 i = tcg_out_ldst_finalize(s); 4360 if (i < 0) { 4361 return i; 4362 } 4363 #endif 4364 #ifdef TCG_TARGET_NEED_POOL_LABELS 4365 i = tcg_out_pool_finalize(s); 4366 if (i < 0) { 4367 return i; 4368 } 4369 #endif 4370 if (!tcg_resolve_relocs(s)) { 4371 return -2; 4372 } 4373 4374 #ifndef CONFIG_TCG_INTERPRETER 4375 /* flush instruction cache */ 4376 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 4377 (uintptr_t)s->code_buf, 4378 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 4379 #endif 4380 4381 return tcg_current_code_size(s); 4382 } 4383 4384 #ifdef CONFIG_PROFILER 4385 void tcg_dump_info(void) 4386 { 4387 TCGProfile prof = {}; 4388 const TCGProfile *s; 4389 int64_t tb_count; 4390 int64_t tb_div_count; 4391 int64_t tot; 4392 4393 tcg_profile_snapshot_counters(&prof); 4394 s = &prof; 4395 tb_count = s->tb_count; 4396 tb_div_count = tb_count ? tb_count : 1; 4397 tot = s->interm_time + s->code_time; 4398 4399 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 4400 tot, tot / 2.4e9); 4401 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64 4402 " %0.1f%%)\n", 4403 tb_count, s->tb_count1 - tb_count, 4404 (double)(s->tb_count1 - s->tb_count) 4405 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 4406 qemu_printf("avg ops/TB %0.1f max=%d\n", 4407 (double)s->op_count / tb_div_count, s->op_count_max); 4408 qemu_printf("deleted ops/TB %0.2f\n", 4409 (double)s->del_op_count / tb_div_count); 4410 qemu_printf("avg temps/TB %0.2f max=%d\n", 4411 (double)s->temp_count / tb_div_count, s->temp_count_max); 4412 qemu_printf("avg host code/TB %0.1f\n", 4413 (double)s->code_out_len / tb_div_count); 4414 qemu_printf("avg search data/TB %0.1f\n", 4415 (double)s->search_out_len / tb_div_count); 4416 4417 qemu_printf("cycles/op %0.1f\n", 4418 s->op_count ? (double)tot / s->op_count : 0); 4419 qemu_printf("cycles/in byte %0.1f\n", 4420 s->code_in_len ? (double)tot / s->code_in_len : 0); 4421 qemu_printf("cycles/out byte %0.1f\n", 4422 s->code_out_len ? (double)tot / s->code_out_len : 0); 4423 qemu_printf("cycles/search byte %0.1f\n", 4424 s->search_out_len ? (double)tot / s->search_out_len : 0); 4425 if (tot == 0) { 4426 tot = 1; 4427 } 4428 qemu_printf(" gen_interm time %0.1f%%\n", 4429 (double)s->interm_time / tot * 100.0); 4430 qemu_printf(" gen_code time %0.1f%%\n", 4431 (double)s->code_time / tot * 100.0); 4432 qemu_printf("optim./code time %0.1f%%\n", 4433 (double)s->opt_time / (s->code_time ? s->code_time : 1) 4434 * 100.0); 4435 qemu_printf("liveness/code time %0.1f%%\n", 4436 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 4437 qemu_printf("cpu_restore count %" PRId64 "\n", 4438 s->restore_count); 4439 qemu_printf(" avg cycles %0.1f\n", 4440 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 4441 } 4442 #else 4443 void tcg_dump_info(void) 4444 { 4445 qemu_printf("[TCG profiler not compiled]\n"); 4446 } 4447 #endif 4448 4449 #ifdef ELF_HOST_MACHINE 4450 /* In order to use this feature, the backend needs to do three things: 4451 4452 (1) Define ELF_HOST_MACHINE to indicate both what value to 4453 put into the ELF image and to indicate support for the feature. 4454 4455 (2) Define tcg_register_jit. This should create a buffer containing 4456 the contents of a .debug_frame section that describes the post- 4457 prologue unwind info for the tcg machine. 4458 4459 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 4460 */ 4461 4462 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 4463 typedef enum { 4464 JIT_NOACTION = 0, 4465 JIT_REGISTER_FN, 4466 JIT_UNREGISTER_FN 4467 } jit_actions_t; 4468 4469 struct jit_code_entry { 4470 struct jit_code_entry *next_entry; 4471 struct jit_code_entry *prev_entry; 4472 const void *symfile_addr; 4473 uint64_t symfile_size; 4474 }; 4475 4476 struct jit_descriptor { 4477 uint32_t version; 4478 uint32_t action_flag; 4479 struct jit_code_entry *relevant_entry; 4480 struct jit_code_entry *first_entry; 4481 }; 4482 4483 void __jit_debug_register_code(void) __attribute__((noinline)); 4484 void __jit_debug_register_code(void) 4485 { 4486 asm(""); 4487 } 4488 4489 /* Must statically initialize the version, because GDB may check 4490 the version before we can set it. */ 4491 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 4492 4493 /* End GDB interface. */ 4494 4495 static int find_string(const char *strtab, const char *str) 4496 { 4497 const char *p = strtab + 1; 4498 4499 while (1) { 4500 if (strcmp(p, str) == 0) { 4501 return p - strtab; 4502 } 4503 p += strlen(p) + 1; 4504 } 4505 } 4506 4507 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 4508 const void *debug_frame, 4509 size_t debug_frame_size) 4510 { 4511 struct __attribute__((packed)) DebugInfo { 4512 uint32_t len; 4513 uint16_t version; 4514 uint32_t abbrev; 4515 uint8_t ptr_size; 4516 uint8_t cu_die; 4517 uint16_t cu_lang; 4518 uintptr_t cu_low_pc; 4519 uintptr_t cu_high_pc; 4520 uint8_t fn_die; 4521 char fn_name[16]; 4522 uintptr_t fn_low_pc; 4523 uintptr_t fn_high_pc; 4524 uint8_t cu_eoc; 4525 }; 4526 4527 struct ElfImage { 4528 ElfW(Ehdr) ehdr; 4529 ElfW(Phdr) phdr; 4530 ElfW(Shdr) shdr[7]; 4531 ElfW(Sym) sym[2]; 4532 struct DebugInfo di; 4533 uint8_t da[24]; 4534 char str[80]; 4535 }; 4536 4537 struct ElfImage *img; 4538 4539 static const struct ElfImage img_template = { 4540 .ehdr = { 4541 .e_ident[EI_MAG0] = ELFMAG0, 4542 .e_ident[EI_MAG1] = ELFMAG1, 4543 .e_ident[EI_MAG2] = ELFMAG2, 4544 .e_ident[EI_MAG3] = ELFMAG3, 4545 .e_ident[EI_CLASS] = ELF_CLASS, 4546 .e_ident[EI_DATA] = ELF_DATA, 4547 .e_ident[EI_VERSION] = EV_CURRENT, 4548 .e_type = ET_EXEC, 4549 .e_machine = ELF_HOST_MACHINE, 4550 .e_version = EV_CURRENT, 4551 .e_phoff = offsetof(struct ElfImage, phdr), 4552 .e_shoff = offsetof(struct ElfImage, shdr), 4553 .e_ehsize = sizeof(ElfW(Shdr)), 4554 .e_phentsize = sizeof(ElfW(Phdr)), 4555 .e_phnum = 1, 4556 .e_shentsize = sizeof(ElfW(Shdr)), 4557 .e_shnum = ARRAY_SIZE(img->shdr), 4558 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 4559 #ifdef ELF_HOST_FLAGS 4560 .e_flags = ELF_HOST_FLAGS, 4561 #endif 4562 #ifdef ELF_OSABI 4563 .e_ident[EI_OSABI] = ELF_OSABI, 4564 #endif 4565 }, 4566 .phdr = { 4567 .p_type = PT_LOAD, 4568 .p_flags = PF_X, 4569 }, 4570 .shdr = { 4571 [0] = { .sh_type = SHT_NULL }, 4572 /* Trick: The contents of code_gen_buffer are not present in 4573 this fake ELF file; that got allocated elsewhere. Therefore 4574 we mark .text as SHT_NOBITS (similar to .bss) so that readers 4575 will not look for contents. We can record any address. */ 4576 [1] = { /* .text */ 4577 .sh_type = SHT_NOBITS, 4578 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 4579 }, 4580 [2] = { /* .debug_info */ 4581 .sh_type = SHT_PROGBITS, 4582 .sh_offset = offsetof(struct ElfImage, di), 4583 .sh_size = sizeof(struct DebugInfo), 4584 }, 4585 [3] = { /* .debug_abbrev */ 4586 .sh_type = SHT_PROGBITS, 4587 .sh_offset = offsetof(struct ElfImage, da), 4588 .sh_size = sizeof(img->da), 4589 }, 4590 [4] = { /* .debug_frame */ 4591 .sh_type = SHT_PROGBITS, 4592 .sh_offset = sizeof(struct ElfImage), 4593 }, 4594 [5] = { /* .symtab */ 4595 .sh_type = SHT_SYMTAB, 4596 .sh_offset = offsetof(struct ElfImage, sym), 4597 .sh_size = sizeof(img->sym), 4598 .sh_info = 1, 4599 .sh_link = ARRAY_SIZE(img->shdr) - 1, 4600 .sh_entsize = sizeof(ElfW(Sym)), 4601 }, 4602 [6] = { /* .strtab */ 4603 .sh_type = SHT_STRTAB, 4604 .sh_offset = offsetof(struct ElfImage, str), 4605 .sh_size = sizeof(img->str), 4606 } 4607 }, 4608 .sym = { 4609 [1] = { /* code_gen_buffer */ 4610 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 4611 .st_shndx = 1, 4612 } 4613 }, 4614 .di = { 4615 .len = sizeof(struct DebugInfo) - 4, 4616 .version = 2, 4617 .ptr_size = sizeof(void *), 4618 .cu_die = 1, 4619 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 4620 .fn_die = 2, 4621 .fn_name = "code_gen_buffer" 4622 }, 4623 .da = { 4624 1, /* abbrev number (the cu) */ 4625 0x11, 1, /* DW_TAG_compile_unit, has children */ 4626 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 4627 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 4628 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 4629 0, 0, /* end of abbrev */ 4630 2, /* abbrev number (the fn) */ 4631 0x2e, 0, /* DW_TAG_subprogram, no children */ 4632 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 4633 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 4634 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 4635 0, 0, /* end of abbrev */ 4636 0 /* no more abbrev */ 4637 }, 4638 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 4639 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 4640 }; 4641 4642 /* We only need a single jit entry; statically allocate it. */ 4643 static struct jit_code_entry one_entry; 4644 4645 uintptr_t buf = (uintptr_t)buf_ptr; 4646 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 4647 DebugFrameHeader *dfh; 4648 4649 img = g_malloc(img_size); 4650 *img = img_template; 4651 4652 img->phdr.p_vaddr = buf; 4653 img->phdr.p_paddr = buf; 4654 img->phdr.p_memsz = buf_size; 4655 4656 img->shdr[1].sh_name = find_string(img->str, ".text"); 4657 img->shdr[1].sh_addr = buf; 4658 img->shdr[1].sh_size = buf_size; 4659 4660 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 4661 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 4662 4663 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 4664 img->shdr[4].sh_size = debug_frame_size; 4665 4666 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 4667 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 4668 4669 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 4670 img->sym[1].st_value = buf; 4671 img->sym[1].st_size = buf_size; 4672 4673 img->di.cu_low_pc = buf; 4674 img->di.cu_high_pc = buf + buf_size; 4675 img->di.fn_low_pc = buf; 4676 img->di.fn_high_pc = buf + buf_size; 4677 4678 dfh = (DebugFrameHeader *)(img + 1); 4679 memcpy(dfh, debug_frame, debug_frame_size); 4680 dfh->fde.func_start = buf; 4681 dfh->fde.func_len = buf_size; 4682 4683 #ifdef DEBUG_JIT 4684 /* Enable this block to be able to debug the ELF image file creation. 4685 One can use readelf, objdump, or other inspection utilities. */ 4686 { 4687 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 4688 if (f) { 4689 if (fwrite(img, img_size, 1, f) != img_size) { 4690 /* Avoid stupid unused return value warning for fwrite. */ 4691 } 4692 fclose(f); 4693 } 4694 } 4695 #endif 4696 4697 one_entry.symfile_addr = img; 4698 one_entry.symfile_size = img_size; 4699 4700 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 4701 __jit_debug_descriptor.relevant_entry = &one_entry; 4702 __jit_debug_descriptor.first_entry = &one_entry; 4703 __jit_debug_register_code(); 4704 } 4705 #else 4706 /* No support for the feature. Provide the entry point expected by exec.c, 4707 and implement the internal function we declared earlier. */ 4708 4709 static void tcg_register_jit_int(const void *buf, size_t size, 4710 const void *debug_frame, 4711 size_t debug_frame_size) 4712 { 4713 } 4714 4715 void tcg_register_jit(const void *buf, size_t buf_size) 4716 { 4717 } 4718 #endif /* ELF_HOST_MACHINE */ 4719 4720 #if !TCG_TARGET_MAYBE_vec 4721 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 4722 { 4723 g_assert_not_reached(); 4724 } 4725 #endif 4726