1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_LIVENESS_ANALYSIS 27 #define USE_TCG_OPTIMIZATIONS 28 29 #include "qemu/osdep.h" 30 31 /* Define to jump the ELF file used to communicate with GDB. */ 32 #undef DEBUG_JIT 33 34 #include "qemu/cutils.h" 35 #include "qemu/host-utils.h" 36 #include "qemu/timer.h" 37 38 /* Note: the long term plan is to reduce the dependencies on the QEMU 39 CPU definitions. Currently they are used for qemu_ld/st 40 instructions */ 41 #define NO_CPU_IO_DEFS 42 #include "cpu.h" 43 44 #include "qemu/host-utils.h" 45 #include "qemu/timer.h" 46 #include "exec/cpu-common.h" 47 #include "exec/exec-all.h" 48 49 #include "tcg-op.h" 50 51 #if UINTPTR_MAX == UINT32_MAX 52 # define ELF_CLASS ELFCLASS32 53 #else 54 # define ELF_CLASS ELFCLASS64 55 #endif 56 #ifdef HOST_WORDS_BIGENDIAN 57 # define ELF_DATA ELFDATA2MSB 58 #else 59 # define ELF_DATA ELFDATA2LSB 60 #endif 61 62 #include "elf.h" 63 #include "exec/log.h" 64 65 /* Forward declarations for functions declared in tcg-target.inc.c and 66 used here. */ 67 static void tcg_target_init(TCGContext *s); 68 static void tcg_target_qemu_prologue(TCGContext *s); 69 static void patch_reloc(tcg_insn_unit *code_ptr, int type, 70 intptr_t value, intptr_t addend); 71 72 /* The CIE and FDE header definitions will be common to all hosts. */ 73 typedef struct { 74 uint32_t len __attribute__((aligned((sizeof(void *))))); 75 uint32_t id; 76 uint8_t version; 77 char augmentation[1]; 78 uint8_t code_align; 79 uint8_t data_align; 80 uint8_t return_column; 81 } DebugFrameCIE; 82 83 typedef struct QEMU_PACKED { 84 uint32_t len __attribute__((aligned((sizeof(void *))))); 85 uint32_t cie_offset; 86 uintptr_t func_start; 87 uintptr_t func_len; 88 } DebugFrameFDEHeader; 89 90 typedef struct QEMU_PACKED { 91 DebugFrameCIE cie; 92 DebugFrameFDEHeader fde; 93 } DebugFrameHeader; 94 95 static void tcg_register_jit_int(void *buf, size_t size, 96 const void *debug_frame, 97 size_t debug_frame_size) 98 __attribute__((unused)); 99 100 /* Forward declarations for functions declared and used in tcg-target.inc.c. */ 101 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str); 102 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 103 intptr_t arg2); 104 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 105 static void tcg_out_movi(TCGContext *s, TCGType type, 106 TCGReg ret, tcg_target_long arg); 107 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 108 const int *const_args); 109 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 110 intptr_t arg2); 111 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 112 TCGReg base, intptr_t ofs); 113 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); 114 static int tcg_target_const_match(tcg_target_long val, TCGType type, 115 const TCGArgConstraint *arg_ct); 116 static void tcg_out_tb_init(TCGContext *s); 117 static bool tcg_out_tb_finalize(TCGContext *s); 118 119 120 121 static TCGRegSet tcg_target_available_regs[2]; 122 static TCGRegSet tcg_target_call_clobber_regs; 123 124 #if TCG_TARGET_INSN_UNIT_SIZE == 1 125 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 126 { 127 *s->code_ptr++ = v; 128 } 129 130 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 131 uint8_t v) 132 { 133 *p = v; 134 } 135 #endif 136 137 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 138 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 139 { 140 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 141 *s->code_ptr++ = v; 142 } else { 143 tcg_insn_unit *p = s->code_ptr; 144 memcpy(p, &v, sizeof(v)); 145 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 146 } 147 } 148 149 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 150 uint16_t v) 151 { 152 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 153 *p = v; 154 } else { 155 memcpy(p, &v, sizeof(v)); 156 } 157 } 158 #endif 159 160 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 161 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 162 { 163 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 164 *s->code_ptr++ = v; 165 } else { 166 tcg_insn_unit *p = s->code_ptr; 167 memcpy(p, &v, sizeof(v)); 168 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 169 } 170 } 171 172 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 173 uint32_t v) 174 { 175 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 176 *p = v; 177 } else { 178 memcpy(p, &v, sizeof(v)); 179 } 180 } 181 #endif 182 183 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 184 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 185 { 186 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 187 *s->code_ptr++ = v; 188 } else { 189 tcg_insn_unit *p = s->code_ptr; 190 memcpy(p, &v, sizeof(v)); 191 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 192 } 193 } 194 195 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 196 uint64_t v) 197 { 198 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 199 *p = v; 200 } else { 201 memcpy(p, &v, sizeof(v)); 202 } 203 } 204 #endif 205 206 /* label relocation processing */ 207 208 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 209 TCGLabel *l, intptr_t addend) 210 { 211 TCGRelocation *r; 212 213 if (l->has_value) { 214 /* FIXME: This may break relocations on RISC targets that 215 modify instruction fields in place. The caller may not have 216 written the initial value. */ 217 patch_reloc(code_ptr, type, l->u.value, addend); 218 } else { 219 /* add a new relocation entry */ 220 r = tcg_malloc(sizeof(TCGRelocation)); 221 r->type = type; 222 r->ptr = code_ptr; 223 r->addend = addend; 224 r->next = l->u.first_reloc; 225 l->u.first_reloc = r; 226 } 227 } 228 229 static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr) 230 { 231 intptr_t value = (intptr_t)ptr; 232 TCGRelocation *r; 233 234 tcg_debug_assert(!l->has_value); 235 236 for (r = l->u.first_reloc; r != NULL; r = r->next) { 237 patch_reloc(r->ptr, r->type, value, r->addend); 238 } 239 240 l->has_value = 1; 241 l->u.value_ptr = ptr; 242 } 243 244 TCGLabel *gen_new_label(void) 245 { 246 TCGContext *s = &tcg_ctx; 247 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 248 249 *l = (TCGLabel){ 250 .id = s->nb_labels++ 251 }; 252 253 return l; 254 } 255 256 #include "tcg-target.inc.c" 257 258 /* pool based memory allocation */ 259 void *tcg_malloc_internal(TCGContext *s, int size) 260 { 261 TCGPool *p; 262 int pool_size; 263 264 if (size > TCG_POOL_CHUNK_SIZE) { 265 /* big malloc: insert a new pool (XXX: could optimize) */ 266 p = g_malloc(sizeof(TCGPool) + size); 267 p->size = size; 268 p->next = s->pool_first_large; 269 s->pool_first_large = p; 270 return p->data; 271 } else { 272 p = s->pool_current; 273 if (!p) { 274 p = s->pool_first; 275 if (!p) 276 goto new_pool; 277 } else { 278 if (!p->next) { 279 new_pool: 280 pool_size = TCG_POOL_CHUNK_SIZE; 281 p = g_malloc(sizeof(TCGPool) + pool_size); 282 p->size = pool_size; 283 p->next = NULL; 284 if (s->pool_current) 285 s->pool_current->next = p; 286 else 287 s->pool_first = p; 288 } else { 289 p = p->next; 290 } 291 } 292 } 293 s->pool_current = p; 294 s->pool_cur = p->data + size; 295 s->pool_end = p->data + p->size; 296 return p->data; 297 } 298 299 void tcg_pool_reset(TCGContext *s) 300 { 301 TCGPool *p, *t; 302 for (p = s->pool_first_large; p; p = t) { 303 t = p->next; 304 g_free(p); 305 } 306 s->pool_first_large = NULL; 307 s->pool_cur = s->pool_end = NULL; 308 s->pool_current = NULL; 309 } 310 311 typedef struct TCGHelperInfo { 312 void *func; 313 const char *name; 314 unsigned flags; 315 unsigned sizemask; 316 } TCGHelperInfo; 317 318 #include "exec/helper-proto.h" 319 320 static const TCGHelperInfo all_helpers[] = { 321 #include "exec/helper-tcg.h" 322 }; 323 324 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 325 326 void tcg_context_init(TCGContext *s) 327 { 328 int op, total_args, n, i; 329 TCGOpDef *def; 330 TCGArgConstraint *args_ct; 331 int *sorted_args; 332 GHashTable *helper_table; 333 334 memset(s, 0, sizeof(*s)); 335 s->nb_globals = 0; 336 337 /* Count total number of arguments and allocate the corresponding 338 space */ 339 total_args = 0; 340 for(op = 0; op < NB_OPS; op++) { 341 def = &tcg_op_defs[op]; 342 n = def->nb_iargs + def->nb_oargs; 343 total_args += n; 344 } 345 346 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 347 sorted_args = g_malloc(sizeof(int) * total_args); 348 349 for(op = 0; op < NB_OPS; op++) { 350 def = &tcg_op_defs[op]; 351 def->args_ct = args_ct; 352 def->sorted_args = sorted_args; 353 n = def->nb_iargs + def->nb_oargs; 354 sorted_args += n; 355 args_ct += n; 356 } 357 358 /* Register helpers. */ 359 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 360 s->helpers = helper_table = g_hash_table_new(NULL, NULL); 361 362 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 363 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 364 (gpointer)&all_helpers[i]); 365 } 366 367 tcg_target_init(s); 368 369 /* Reverse the order of the saved registers, assuming they're all at 370 the start of tcg_target_reg_alloc_order. */ 371 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 372 int r = tcg_target_reg_alloc_order[n]; 373 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 374 break; 375 } 376 } 377 for (i = 0; i < n; ++i) { 378 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 379 } 380 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 381 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 382 } 383 } 384 385 void tcg_prologue_init(TCGContext *s) 386 { 387 size_t prologue_size, total_size; 388 void *buf0, *buf1; 389 390 /* Put the prologue at the beginning of code_gen_buffer. */ 391 buf0 = s->code_gen_buffer; 392 s->code_ptr = buf0; 393 s->code_buf = buf0; 394 s->code_gen_prologue = buf0; 395 396 /* Generate the prologue. */ 397 tcg_target_qemu_prologue(s); 398 buf1 = s->code_ptr; 399 flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1); 400 401 /* Deduct the prologue from the buffer. */ 402 prologue_size = tcg_current_code_size(s); 403 s->code_gen_ptr = buf1; 404 s->code_gen_buffer = buf1; 405 s->code_buf = buf1; 406 total_size = s->code_gen_buffer_size - prologue_size; 407 s->code_gen_buffer_size = total_size; 408 409 /* Compute a high-water mark, at which we voluntarily flush the buffer 410 and start over. The size here is arbitrary, significantly larger 411 than we expect the code generation for any one opcode to require. */ 412 s->code_gen_highwater = s->code_gen_buffer + (total_size - 1024); 413 414 tcg_register_jit(s->code_gen_buffer, total_size); 415 416 #ifdef DEBUG_DISAS 417 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 418 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size); 419 log_disas(buf0, prologue_size); 420 qemu_log("\n"); 421 qemu_log_flush(); 422 } 423 #endif 424 } 425 426 void tcg_func_start(TCGContext *s) 427 { 428 tcg_pool_reset(s); 429 s->nb_temps = s->nb_globals; 430 431 /* No temps have been previously allocated for size or locality. */ 432 memset(s->free_temps, 0, sizeof(s->free_temps)); 433 434 s->nb_labels = 0; 435 s->current_frame_offset = s->frame_start; 436 437 #ifdef CONFIG_DEBUG_TCG 438 s->goto_tb_issue_mask = 0; 439 #endif 440 441 s->gen_first_op_idx = 0; 442 s->gen_last_op_idx = -1; 443 s->gen_next_op_idx = 0; 444 s->gen_next_parm_idx = 0; 445 446 s->be = tcg_malloc(sizeof(TCGBackendData)); 447 } 448 449 static inline int temp_idx(TCGContext *s, TCGTemp *ts) 450 { 451 ptrdiff_t n = ts - s->temps; 452 tcg_debug_assert(n >= 0 && n < s->nb_temps); 453 return n; 454 } 455 456 static inline TCGTemp *tcg_temp_alloc(TCGContext *s) 457 { 458 int n = s->nb_temps++; 459 tcg_debug_assert(n < TCG_MAX_TEMPS); 460 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 461 } 462 463 static inline TCGTemp *tcg_global_alloc(TCGContext *s) 464 { 465 tcg_debug_assert(s->nb_globals == s->nb_temps); 466 s->nb_globals++; 467 return tcg_temp_alloc(s); 468 } 469 470 static int tcg_global_reg_new_internal(TCGContext *s, TCGType type, 471 TCGReg reg, const char *name) 472 { 473 TCGTemp *ts; 474 475 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) { 476 tcg_abort(); 477 } 478 479 ts = tcg_global_alloc(s); 480 ts->base_type = type; 481 ts->type = type; 482 ts->fixed_reg = 1; 483 ts->reg = reg; 484 ts->name = name; 485 tcg_regset_set_reg(s->reserved_regs, reg); 486 487 return temp_idx(s, ts); 488 } 489 490 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 491 { 492 int idx; 493 s->frame_start = start; 494 s->frame_end = start + size; 495 idx = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 496 s->frame_temp = &s->temps[idx]; 497 } 498 499 TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name) 500 { 501 TCGContext *s = &tcg_ctx; 502 int idx; 503 504 if (tcg_regset_test_reg(s->reserved_regs, reg)) { 505 tcg_abort(); 506 } 507 idx = tcg_global_reg_new_internal(s, TCG_TYPE_I32, reg, name); 508 return MAKE_TCGV_I32(idx); 509 } 510 511 TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name) 512 { 513 TCGContext *s = &tcg_ctx; 514 int idx; 515 516 if (tcg_regset_test_reg(s->reserved_regs, reg)) { 517 tcg_abort(); 518 } 519 idx = tcg_global_reg_new_internal(s, TCG_TYPE_I64, reg, name); 520 return MAKE_TCGV_I64(idx); 521 } 522 523 int tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, 524 intptr_t offset, const char *name) 525 { 526 TCGContext *s = &tcg_ctx; 527 TCGTemp *base_ts = &s->temps[GET_TCGV_PTR(base)]; 528 TCGTemp *ts = tcg_global_alloc(s); 529 int indirect_reg = 0, bigendian = 0; 530 #ifdef HOST_WORDS_BIGENDIAN 531 bigendian = 1; 532 #endif 533 534 if (!base_ts->fixed_reg) { 535 indirect_reg = 1; 536 base_ts->indirect_base = 1; 537 } 538 539 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 540 TCGTemp *ts2 = tcg_global_alloc(s); 541 char buf[64]; 542 543 ts->base_type = TCG_TYPE_I64; 544 ts->type = TCG_TYPE_I32; 545 ts->indirect_reg = indirect_reg; 546 ts->mem_allocated = 1; 547 ts->mem_base = base_ts; 548 ts->mem_offset = offset + bigendian * 4; 549 pstrcpy(buf, sizeof(buf), name); 550 pstrcat(buf, sizeof(buf), "_0"); 551 ts->name = strdup(buf); 552 553 tcg_debug_assert(ts2 == ts + 1); 554 ts2->base_type = TCG_TYPE_I64; 555 ts2->type = TCG_TYPE_I32; 556 ts2->indirect_reg = indirect_reg; 557 ts2->mem_allocated = 1; 558 ts2->mem_base = base_ts; 559 ts2->mem_offset = offset + (1 - bigendian) * 4; 560 pstrcpy(buf, sizeof(buf), name); 561 pstrcat(buf, sizeof(buf), "_1"); 562 ts2->name = strdup(buf); 563 } else { 564 ts->base_type = type; 565 ts->type = type; 566 ts->indirect_reg = indirect_reg; 567 ts->mem_allocated = 1; 568 ts->mem_base = base_ts; 569 ts->mem_offset = offset; 570 ts->name = name; 571 } 572 return temp_idx(s, ts); 573 } 574 575 static int tcg_temp_new_internal(TCGType type, int temp_local) 576 { 577 TCGContext *s = &tcg_ctx; 578 TCGTemp *ts; 579 int idx, k; 580 581 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 582 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 583 if (idx < TCG_MAX_TEMPS) { 584 /* There is already an available temp with the right type. */ 585 clear_bit(idx, s->free_temps[k].l); 586 587 ts = &s->temps[idx]; 588 ts->temp_allocated = 1; 589 tcg_debug_assert(ts->base_type == type); 590 tcg_debug_assert(ts->temp_local == temp_local); 591 } else { 592 ts = tcg_temp_alloc(s); 593 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 594 TCGTemp *ts2 = tcg_temp_alloc(s); 595 596 ts->base_type = type; 597 ts->type = TCG_TYPE_I32; 598 ts->temp_allocated = 1; 599 ts->temp_local = temp_local; 600 601 tcg_debug_assert(ts2 == ts + 1); 602 ts2->base_type = TCG_TYPE_I64; 603 ts2->type = TCG_TYPE_I32; 604 ts2->temp_allocated = 1; 605 ts2->temp_local = temp_local; 606 } else { 607 ts->base_type = type; 608 ts->type = type; 609 ts->temp_allocated = 1; 610 ts->temp_local = temp_local; 611 } 612 idx = temp_idx(s, ts); 613 } 614 615 #if defined(CONFIG_DEBUG_TCG) 616 s->temps_in_use++; 617 #endif 618 return idx; 619 } 620 621 TCGv_i32 tcg_temp_new_internal_i32(int temp_local) 622 { 623 int idx; 624 625 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local); 626 return MAKE_TCGV_I32(idx); 627 } 628 629 TCGv_i64 tcg_temp_new_internal_i64(int temp_local) 630 { 631 int idx; 632 633 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local); 634 return MAKE_TCGV_I64(idx); 635 } 636 637 static void tcg_temp_free_internal(int idx) 638 { 639 TCGContext *s = &tcg_ctx; 640 TCGTemp *ts; 641 int k; 642 643 #if defined(CONFIG_DEBUG_TCG) 644 s->temps_in_use--; 645 if (s->temps_in_use < 0) { 646 fprintf(stderr, "More temporaries freed than allocated!\n"); 647 } 648 #endif 649 650 tcg_debug_assert(idx >= s->nb_globals && idx < s->nb_temps); 651 ts = &s->temps[idx]; 652 tcg_debug_assert(ts->temp_allocated != 0); 653 ts->temp_allocated = 0; 654 655 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); 656 set_bit(idx, s->free_temps[k].l); 657 } 658 659 void tcg_temp_free_i32(TCGv_i32 arg) 660 { 661 tcg_temp_free_internal(GET_TCGV_I32(arg)); 662 } 663 664 void tcg_temp_free_i64(TCGv_i64 arg) 665 { 666 tcg_temp_free_internal(GET_TCGV_I64(arg)); 667 } 668 669 TCGv_i32 tcg_const_i32(int32_t val) 670 { 671 TCGv_i32 t0; 672 t0 = tcg_temp_new_i32(); 673 tcg_gen_movi_i32(t0, val); 674 return t0; 675 } 676 677 TCGv_i64 tcg_const_i64(int64_t val) 678 { 679 TCGv_i64 t0; 680 t0 = tcg_temp_new_i64(); 681 tcg_gen_movi_i64(t0, val); 682 return t0; 683 } 684 685 TCGv_i32 tcg_const_local_i32(int32_t val) 686 { 687 TCGv_i32 t0; 688 t0 = tcg_temp_local_new_i32(); 689 tcg_gen_movi_i32(t0, val); 690 return t0; 691 } 692 693 TCGv_i64 tcg_const_local_i64(int64_t val) 694 { 695 TCGv_i64 t0; 696 t0 = tcg_temp_local_new_i64(); 697 tcg_gen_movi_i64(t0, val); 698 return t0; 699 } 700 701 #if defined(CONFIG_DEBUG_TCG) 702 void tcg_clear_temp_count(void) 703 { 704 TCGContext *s = &tcg_ctx; 705 s->temps_in_use = 0; 706 } 707 708 int tcg_check_temp_count(void) 709 { 710 TCGContext *s = &tcg_ctx; 711 if (s->temps_in_use) { 712 /* Clear the count so that we don't give another 713 * warning immediately next time around. 714 */ 715 s->temps_in_use = 0; 716 return 1; 717 } 718 return 0; 719 } 720 #endif 721 722 /* Note: we convert the 64 bit args to 32 bit and do some alignment 723 and endian swap. Maybe it would be better to do the alignment 724 and endian swap in tcg_reg_alloc_call(). */ 725 void tcg_gen_callN(TCGContext *s, void *func, TCGArg ret, 726 int nargs, TCGArg *args) 727 { 728 int i, real_args, nb_rets, pi, pi_first; 729 unsigned sizemask, flags; 730 TCGHelperInfo *info; 731 732 info = g_hash_table_lookup(s->helpers, (gpointer)func); 733 flags = info->flags; 734 sizemask = info->sizemask; 735 736 #if defined(__sparc__) && !defined(__arch64__) \ 737 && !defined(CONFIG_TCG_INTERPRETER) 738 /* We have 64-bit values in one register, but need to pass as two 739 separate parameters. Split them. */ 740 int orig_sizemask = sizemask; 741 int orig_nargs = nargs; 742 TCGv_i64 retl, reth; 743 744 TCGV_UNUSED_I64(retl); 745 TCGV_UNUSED_I64(reth); 746 if (sizemask != 0) { 747 TCGArg *split_args = __builtin_alloca(sizeof(TCGArg) * nargs * 2); 748 for (i = real_args = 0; i < nargs; ++i) { 749 int is_64bit = sizemask & (1 << (i+1)*2); 750 if (is_64bit) { 751 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 752 TCGv_i32 h = tcg_temp_new_i32(); 753 TCGv_i32 l = tcg_temp_new_i32(); 754 tcg_gen_extr_i64_i32(l, h, orig); 755 split_args[real_args++] = GET_TCGV_I32(h); 756 split_args[real_args++] = GET_TCGV_I32(l); 757 } else { 758 split_args[real_args++] = args[i]; 759 } 760 } 761 nargs = real_args; 762 args = split_args; 763 sizemask = 0; 764 } 765 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 766 for (i = 0; i < nargs; ++i) { 767 int is_64bit = sizemask & (1 << (i+1)*2); 768 int is_signed = sizemask & (2 << (i+1)*2); 769 if (!is_64bit) { 770 TCGv_i64 temp = tcg_temp_new_i64(); 771 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 772 if (is_signed) { 773 tcg_gen_ext32s_i64(temp, orig); 774 } else { 775 tcg_gen_ext32u_i64(temp, orig); 776 } 777 args[i] = GET_TCGV_I64(temp); 778 } 779 } 780 #endif /* TCG_TARGET_EXTEND_ARGS */ 781 782 pi_first = pi = s->gen_next_parm_idx; 783 if (ret != TCG_CALL_DUMMY_ARG) { 784 #if defined(__sparc__) && !defined(__arch64__) \ 785 && !defined(CONFIG_TCG_INTERPRETER) 786 if (orig_sizemask & 1) { 787 /* The 32-bit ABI is going to return the 64-bit value in 788 the %o0/%o1 register pair. Prepare for this by using 789 two return temporaries, and reassemble below. */ 790 retl = tcg_temp_new_i64(); 791 reth = tcg_temp_new_i64(); 792 s->gen_opparam_buf[pi++] = GET_TCGV_I64(reth); 793 s->gen_opparam_buf[pi++] = GET_TCGV_I64(retl); 794 nb_rets = 2; 795 } else { 796 s->gen_opparam_buf[pi++] = ret; 797 nb_rets = 1; 798 } 799 #else 800 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { 801 #ifdef HOST_WORDS_BIGENDIAN 802 s->gen_opparam_buf[pi++] = ret + 1; 803 s->gen_opparam_buf[pi++] = ret; 804 #else 805 s->gen_opparam_buf[pi++] = ret; 806 s->gen_opparam_buf[pi++] = ret + 1; 807 #endif 808 nb_rets = 2; 809 } else { 810 s->gen_opparam_buf[pi++] = ret; 811 nb_rets = 1; 812 } 813 #endif 814 } else { 815 nb_rets = 0; 816 } 817 real_args = 0; 818 for (i = 0; i < nargs; i++) { 819 int is_64bit = sizemask & (1 << (i+1)*2); 820 if (TCG_TARGET_REG_BITS < 64 && is_64bit) { 821 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 822 /* some targets want aligned 64 bit args */ 823 if (real_args & 1) { 824 s->gen_opparam_buf[pi++] = TCG_CALL_DUMMY_ARG; 825 real_args++; 826 } 827 #endif 828 /* If stack grows up, then we will be placing successive 829 arguments at lower addresses, which means we need to 830 reverse the order compared to how we would normally 831 treat either big or little-endian. For those arguments 832 that will wind up in registers, this still works for 833 HPPA (the only current STACK_GROWSUP target) since the 834 argument registers are *also* allocated in decreasing 835 order. If another such target is added, this logic may 836 have to get more complicated to differentiate between 837 stack arguments and register arguments. */ 838 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 839 s->gen_opparam_buf[pi++] = args[i] + 1; 840 s->gen_opparam_buf[pi++] = args[i]; 841 #else 842 s->gen_opparam_buf[pi++] = args[i]; 843 s->gen_opparam_buf[pi++] = args[i] + 1; 844 #endif 845 real_args += 2; 846 continue; 847 } 848 849 s->gen_opparam_buf[pi++] = args[i]; 850 real_args++; 851 } 852 s->gen_opparam_buf[pi++] = (uintptr_t)func; 853 s->gen_opparam_buf[pi++] = flags; 854 855 i = s->gen_next_op_idx; 856 tcg_debug_assert(i < OPC_BUF_SIZE); 857 tcg_debug_assert(pi <= OPPARAM_BUF_SIZE); 858 859 /* Set links for sequential allocation during translation. */ 860 s->gen_op_buf[i] = (TCGOp){ 861 .opc = INDEX_op_call, 862 .callo = nb_rets, 863 .calli = real_args, 864 .args = pi_first, 865 .prev = i - 1, 866 .next = i + 1 867 }; 868 869 /* Make sure the calli field didn't overflow. */ 870 tcg_debug_assert(s->gen_op_buf[i].calli == real_args); 871 872 s->gen_last_op_idx = i; 873 s->gen_next_op_idx = i + 1; 874 s->gen_next_parm_idx = pi; 875 876 #if defined(__sparc__) && !defined(__arch64__) \ 877 && !defined(CONFIG_TCG_INTERPRETER) 878 /* Free all of the parts we allocated above. */ 879 for (i = real_args = 0; i < orig_nargs; ++i) { 880 int is_64bit = orig_sizemask & (1 << (i+1)*2); 881 if (is_64bit) { 882 TCGv_i32 h = MAKE_TCGV_I32(args[real_args++]); 883 TCGv_i32 l = MAKE_TCGV_I32(args[real_args++]); 884 tcg_temp_free_i32(h); 885 tcg_temp_free_i32(l); 886 } else { 887 real_args++; 888 } 889 } 890 if (orig_sizemask & 1) { 891 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 892 Note that describing these as TCGv_i64 eliminates an unnecessary 893 zero-extension that tcg_gen_concat_i32_i64 would create. */ 894 tcg_gen_concat32_i64(MAKE_TCGV_I64(ret), retl, reth); 895 tcg_temp_free_i64(retl); 896 tcg_temp_free_i64(reth); 897 } 898 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 899 for (i = 0; i < nargs; ++i) { 900 int is_64bit = sizemask & (1 << (i+1)*2); 901 if (!is_64bit) { 902 TCGv_i64 temp = MAKE_TCGV_I64(args[i]); 903 tcg_temp_free_i64(temp); 904 } 905 } 906 #endif /* TCG_TARGET_EXTEND_ARGS */ 907 } 908 909 static void tcg_reg_alloc_start(TCGContext *s) 910 { 911 int i; 912 TCGTemp *ts; 913 for(i = 0; i < s->nb_globals; i++) { 914 ts = &s->temps[i]; 915 if (ts->fixed_reg) { 916 ts->val_type = TEMP_VAL_REG; 917 } else { 918 ts->val_type = TEMP_VAL_MEM; 919 } 920 } 921 for(i = s->nb_globals; i < s->nb_temps; i++) { 922 ts = &s->temps[i]; 923 if (ts->temp_local) { 924 ts->val_type = TEMP_VAL_MEM; 925 } else { 926 ts->val_type = TEMP_VAL_DEAD; 927 } 928 ts->mem_allocated = 0; 929 ts->fixed_reg = 0; 930 } 931 932 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 933 } 934 935 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 936 TCGTemp *ts) 937 { 938 int idx = temp_idx(s, ts); 939 940 if (idx < s->nb_globals) { 941 pstrcpy(buf, buf_size, ts->name); 942 } else if (ts->temp_local) { 943 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 944 } else { 945 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 946 } 947 return buf; 948 } 949 950 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, 951 int buf_size, int idx) 952 { 953 tcg_debug_assert(idx >= 0 && idx < s->nb_temps); 954 return tcg_get_arg_str_ptr(s, buf, buf_size, &s->temps[idx]); 955 } 956 957 /* Find helper name. */ 958 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) 959 { 960 const char *ret = NULL; 961 if (s->helpers) { 962 TCGHelperInfo *info = g_hash_table_lookup(s->helpers, (gpointer)val); 963 if (info) { 964 ret = info->name; 965 } 966 } 967 return ret; 968 } 969 970 static const char * const cond_name[] = 971 { 972 [TCG_COND_NEVER] = "never", 973 [TCG_COND_ALWAYS] = "always", 974 [TCG_COND_EQ] = "eq", 975 [TCG_COND_NE] = "ne", 976 [TCG_COND_LT] = "lt", 977 [TCG_COND_GE] = "ge", 978 [TCG_COND_LE] = "le", 979 [TCG_COND_GT] = "gt", 980 [TCG_COND_LTU] = "ltu", 981 [TCG_COND_GEU] = "geu", 982 [TCG_COND_LEU] = "leu", 983 [TCG_COND_GTU] = "gtu" 984 }; 985 986 static const char * const ldst_name[] = 987 { 988 [MO_UB] = "ub", 989 [MO_SB] = "sb", 990 [MO_LEUW] = "leuw", 991 [MO_LESW] = "lesw", 992 [MO_LEUL] = "leul", 993 [MO_LESL] = "lesl", 994 [MO_LEQ] = "leq", 995 [MO_BEUW] = "beuw", 996 [MO_BESW] = "besw", 997 [MO_BEUL] = "beul", 998 [MO_BESL] = "besl", 999 [MO_BEQ] = "beq", 1000 }; 1001 1002 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 1003 #ifdef ALIGNED_ONLY 1004 [MO_UNALN >> MO_ASHIFT] = "un+", 1005 [MO_ALIGN >> MO_ASHIFT] = "", 1006 #else 1007 [MO_UNALN >> MO_ASHIFT] = "", 1008 [MO_ALIGN >> MO_ASHIFT] = "al+", 1009 #endif 1010 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 1011 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 1012 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 1013 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 1014 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 1015 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 1016 }; 1017 1018 void tcg_dump_ops(TCGContext *s) 1019 { 1020 char buf[128]; 1021 TCGOp *op; 1022 int oi; 1023 1024 for (oi = s->gen_first_op_idx; oi >= 0; oi = op->next) { 1025 int i, k, nb_oargs, nb_iargs, nb_cargs; 1026 const TCGOpDef *def; 1027 const TCGArg *args; 1028 TCGOpcode c; 1029 1030 op = &s->gen_op_buf[oi]; 1031 c = op->opc; 1032 def = &tcg_op_defs[c]; 1033 args = &s->gen_opparam_buf[op->args]; 1034 1035 if (c == INDEX_op_insn_start) { 1036 qemu_log("%s ----", oi != s->gen_first_op_idx ? "\n" : ""); 1037 1038 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 1039 target_ulong a; 1040 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1041 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2]; 1042 #else 1043 a = args[i]; 1044 #endif 1045 qemu_log(" " TARGET_FMT_lx, a); 1046 } 1047 } else if (c == INDEX_op_call) { 1048 /* variable number of arguments */ 1049 nb_oargs = op->callo; 1050 nb_iargs = op->calli; 1051 nb_cargs = def->nb_cargs; 1052 1053 /* function name, flags, out args */ 1054 qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, 1055 tcg_find_helper(s, args[nb_oargs + nb_iargs]), 1056 args[nb_oargs + nb_iargs + 1], nb_oargs); 1057 for (i = 0; i < nb_oargs; i++) { 1058 qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1059 args[i])); 1060 } 1061 for (i = 0; i < nb_iargs; i++) { 1062 TCGArg arg = args[nb_oargs + i]; 1063 const char *t = "<dummy>"; 1064 if (arg != TCG_CALL_DUMMY_ARG) { 1065 t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg); 1066 } 1067 qemu_log(",%s", t); 1068 } 1069 } else { 1070 qemu_log(" %s ", def->name); 1071 1072 nb_oargs = def->nb_oargs; 1073 nb_iargs = def->nb_iargs; 1074 nb_cargs = def->nb_cargs; 1075 1076 k = 0; 1077 for (i = 0; i < nb_oargs; i++) { 1078 if (k != 0) { 1079 qemu_log(","); 1080 } 1081 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1082 args[k++])); 1083 } 1084 for (i = 0; i < nb_iargs; i++) { 1085 if (k != 0) { 1086 qemu_log(","); 1087 } 1088 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1089 args[k++])); 1090 } 1091 switch (c) { 1092 case INDEX_op_brcond_i32: 1093 case INDEX_op_setcond_i32: 1094 case INDEX_op_movcond_i32: 1095 case INDEX_op_brcond2_i32: 1096 case INDEX_op_setcond2_i32: 1097 case INDEX_op_brcond_i64: 1098 case INDEX_op_setcond_i64: 1099 case INDEX_op_movcond_i64: 1100 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) { 1101 qemu_log(",%s", cond_name[args[k++]]); 1102 } else { 1103 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1104 } 1105 i = 1; 1106 break; 1107 case INDEX_op_qemu_ld_i32: 1108 case INDEX_op_qemu_st_i32: 1109 case INDEX_op_qemu_ld_i64: 1110 case INDEX_op_qemu_st_i64: 1111 { 1112 TCGMemOpIdx oi = args[k++]; 1113 TCGMemOp op = get_memop(oi); 1114 unsigned ix = get_mmuidx(oi); 1115 1116 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) { 1117 qemu_log(",$0x%x,%u", op, ix); 1118 } else { 1119 const char *s_al, *s_op; 1120 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT]; 1121 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)]; 1122 qemu_log(",%s%s,%u", s_al, s_op, ix); 1123 } 1124 i = 1; 1125 } 1126 break; 1127 default: 1128 i = 0; 1129 break; 1130 } 1131 switch (c) { 1132 case INDEX_op_set_label: 1133 case INDEX_op_br: 1134 case INDEX_op_brcond_i32: 1135 case INDEX_op_brcond_i64: 1136 case INDEX_op_brcond2_i32: 1137 qemu_log("%s$L%d", k ? "," : "", arg_label(args[k])->id); 1138 i++, k++; 1139 break; 1140 default: 1141 break; 1142 } 1143 for (; i < nb_cargs; i++, k++) { 1144 qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", args[k]); 1145 } 1146 } 1147 qemu_log("\n"); 1148 } 1149 } 1150 1151 /* we give more priority to constraints with less registers */ 1152 static int get_constraint_priority(const TCGOpDef *def, int k) 1153 { 1154 const TCGArgConstraint *arg_ct; 1155 1156 int i, n; 1157 arg_ct = &def->args_ct[k]; 1158 if (arg_ct->ct & TCG_CT_ALIAS) { 1159 /* an alias is equivalent to a single register */ 1160 n = 1; 1161 } else { 1162 if (!(arg_ct->ct & TCG_CT_REG)) 1163 return 0; 1164 n = 0; 1165 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1166 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 1167 n++; 1168 } 1169 } 1170 return TCG_TARGET_NB_REGS - n + 1; 1171 } 1172 1173 /* sort from highest priority to lowest */ 1174 static void sort_constraints(TCGOpDef *def, int start, int n) 1175 { 1176 int i, j, p1, p2, tmp; 1177 1178 for(i = 0; i < n; i++) 1179 def->sorted_args[start + i] = start + i; 1180 if (n <= 1) 1181 return; 1182 for(i = 0; i < n - 1; i++) { 1183 for(j = i + 1; j < n; j++) { 1184 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 1185 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 1186 if (p1 < p2) { 1187 tmp = def->sorted_args[start + i]; 1188 def->sorted_args[start + i] = def->sorted_args[start + j]; 1189 def->sorted_args[start + j] = tmp; 1190 } 1191 } 1192 } 1193 } 1194 1195 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs) 1196 { 1197 TCGOpcode op; 1198 TCGOpDef *def; 1199 const char *ct_str; 1200 int i, nb_args; 1201 1202 for(;;) { 1203 if (tdefs->op == (TCGOpcode)-1) 1204 break; 1205 op = tdefs->op; 1206 tcg_debug_assert((unsigned)op < NB_OPS); 1207 def = &tcg_op_defs[op]; 1208 #if defined(CONFIG_DEBUG_TCG) 1209 /* Duplicate entry in op definitions? */ 1210 tcg_debug_assert(!def->used); 1211 def->used = 1; 1212 #endif 1213 nb_args = def->nb_iargs + def->nb_oargs; 1214 for(i = 0; i < nb_args; i++) { 1215 ct_str = tdefs->args_ct_str[i]; 1216 /* Incomplete TCGTargetOpDef entry? */ 1217 tcg_debug_assert(ct_str != NULL); 1218 tcg_regset_clear(def->args_ct[i].u.regs); 1219 def->args_ct[i].ct = 0; 1220 if (ct_str[0] >= '0' && ct_str[0] <= '9') { 1221 int oarg; 1222 oarg = ct_str[0] - '0'; 1223 tcg_debug_assert(oarg < def->nb_oargs); 1224 tcg_debug_assert(def->args_ct[oarg].ct & TCG_CT_REG); 1225 /* TCG_CT_ALIAS is for the output arguments. The input 1226 argument is tagged with TCG_CT_IALIAS. */ 1227 def->args_ct[i] = def->args_ct[oarg]; 1228 def->args_ct[oarg].ct = TCG_CT_ALIAS; 1229 def->args_ct[oarg].alias_index = i; 1230 def->args_ct[i].ct |= TCG_CT_IALIAS; 1231 def->args_ct[i].alias_index = oarg; 1232 } else { 1233 for(;;) { 1234 if (*ct_str == '\0') 1235 break; 1236 switch(*ct_str) { 1237 case 'i': 1238 def->args_ct[i].ct |= TCG_CT_CONST; 1239 ct_str++; 1240 break; 1241 default: 1242 if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) { 1243 fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n", 1244 ct_str, i, def->name); 1245 exit(1); 1246 } 1247 } 1248 } 1249 } 1250 } 1251 1252 /* TCGTargetOpDef entry with too much information? */ 1253 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 1254 1255 /* sort the constraints (XXX: this is just an heuristic) */ 1256 sort_constraints(def, 0, def->nb_oargs); 1257 sort_constraints(def, def->nb_oargs, def->nb_iargs); 1258 1259 #if 0 1260 { 1261 int i; 1262 1263 printf("%s: sorted=", def->name); 1264 for(i = 0; i < def->nb_oargs + def->nb_iargs; i++) 1265 printf(" %d", def->sorted_args[i]); 1266 printf("\n"); 1267 } 1268 #endif 1269 tdefs++; 1270 } 1271 1272 #if defined(CONFIG_DEBUG_TCG) 1273 i = 0; 1274 for (op = 0; op < tcg_op_defs_max; op++) { 1275 const TCGOpDef *def = &tcg_op_defs[op]; 1276 if (def->flags & TCG_OPF_NOT_PRESENT) { 1277 /* Wrong entry in op definitions? */ 1278 if (def->used) { 1279 fprintf(stderr, "Invalid op definition for %s\n", def->name); 1280 i = 1; 1281 } 1282 } else { 1283 /* Missing entry in op definitions? */ 1284 if (!def->used) { 1285 fprintf(stderr, "Missing op definition for %s\n", def->name); 1286 i = 1; 1287 } 1288 } 1289 } 1290 if (i == 1) { 1291 tcg_abort(); 1292 } 1293 #endif 1294 } 1295 1296 void tcg_op_remove(TCGContext *s, TCGOp *op) 1297 { 1298 int next = op->next; 1299 int prev = op->prev; 1300 1301 if (next >= 0) { 1302 s->gen_op_buf[next].prev = prev; 1303 } else { 1304 s->gen_last_op_idx = prev; 1305 } 1306 if (prev >= 0) { 1307 s->gen_op_buf[prev].next = next; 1308 } else { 1309 s->gen_first_op_idx = next; 1310 } 1311 1312 memset(op, -1, sizeof(*op)); 1313 1314 #ifdef CONFIG_PROFILER 1315 s->del_op_count++; 1316 #endif 1317 } 1318 1319 #ifdef USE_LIVENESS_ANALYSIS 1320 /* liveness analysis: end of function: all temps are dead, and globals 1321 should be in memory. */ 1322 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps, 1323 uint8_t *mem_temps) 1324 { 1325 memset(dead_temps, 1, s->nb_temps); 1326 memset(mem_temps, 1, s->nb_globals); 1327 memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals); 1328 } 1329 1330 /* liveness analysis: end of basic block: all temps are dead, globals 1331 and local temps should be in memory. */ 1332 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps, 1333 uint8_t *mem_temps) 1334 { 1335 int i; 1336 1337 memset(dead_temps, 1, s->nb_temps); 1338 memset(mem_temps, 1, s->nb_globals); 1339 for(i = s->nb_globals; i < s->nb_temps; i++) { 1340 mem_temps[i] = s->temps[i].temp_local; 1341 } 1342 } 1343 1344 /* Liveness analysis : update the opc_dead_args array to tell if a 1345 given input arguments is dead. Instructions updating dead 1346 temporaries are removed. */ 1347 static void tcg_liveness_analysis(TCGContext *s) 1348 { 1349 uint8_t *dead_temps, *mem_temps; 1350 int oi, oi_prev, nb_ops; 1351 1352 nb_ops = s->gen_next_op_idx; 1353 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1354 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1355 1356 dead_temps = tcg_malloc(s->nb_temps); 1357 mem_temps = tcg_malloc(s->nb_temps); 1358 tcg_la_func_end(s, dead_temps, mem_temps); 1359 1360 for (oi = s->gen_last_op_idx; oi >= 0; oi = oi_prev) { 1361 int i, nb_iargs, nb_oargs; 1362 TCGOpcode opc_new, opc_new2; 1363 bool have_opc_new2; 1364 uint16_t dead_args; 1365 uint8_t sync_args; 1366 TCGArg arg; 1367 1368 TCGOp * const op = &s->gen_op_buf[oi]; 1369 TCGArg * const args = &s->gen_opparam_buf[op->args]; 1370 TCGOpcode opc = op->opc; 1371 const TCGOpDef *def = &tcg_op_defs[opc]; 1372 1373 oi_prev = op->prev; 1374 1375 switch (opc) { 1376 case INDEX_op_call: 1377 { 1378 int call_flags; 1379 1380 nb_oargs = op->callo; 1381 nb_iargs = op->calli; 1382 call_flags = args[nb_oargs + nb_iargs + 1]; 1383 1384 /* pure functions can be removed if their result is unused */ 1385 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 1386 for (i = 0; i < nb_oargs; i++) { 1387 arg = args[i]; 1388 if (!dead_temps[arg] || mem_temps[arg]) { 1389 goto do_not_remove_call; 1390 } 1391 } 1392 goto do_remove; 1393 } else { 1394 do_not_remove_call: 1395 1396 /* output args are dead */ 1397 dead_args = 0; 1398 sync_args = 0; 1399 for (i = 0; i < nb_oargs; i++) { 1400 arg = args[i]; 1401 if (dead_temps[arg]) { 1402 dead_args |= (1 << i); 1403 } 1404 if (mem_temps[arg]) { 1405 sync_args |= (1 << i); 1406 } 1407 dead_temps[arg] = 1; 1408 mem_temps[arg] = 0; 1409 } 1410 1411 if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 1412 /* globals should be synced to memory */ 1413 memset(mem_temps, 1, s->nb_globals); 1414 } 1415 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 1416 TCG_CALL_NO_READ_GLOBALS))) { 1417 /* globals should go back to memory */ 1418 memset(dead_temps, 1, s->nb_globals); 1419 } 1420 1421 /* record arguments that die in this helper */ 1422 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1423 arg = args[i]; 1424 if (arg != TCG_CALL_DUMMY_ARG) { 1425 if (dead_temps[arg]) { 1426 dead_args |= (1 << i); 1427 } 1428 } 1429 } 1430 /* input arguments are live for preceding opcodes */ 1431 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1432 arg = args[i]; 1433 dead_temps[arg] = 0; 1434 } 1435 s->op_dead_args[oi] = dead_args; 1436 s->op_sync_args[oi] = sync_args; 1437 } 1438 } 1439 break; 1440 case INDEX_op_insn_start: 1441 break; 1442 case INDEX_op_discard: 1443 /* mark the temporary as dead */ 1444 dead_temps[args[0]] = 1; 1445 mem_temps[args[0]] = 0; 1446 break; 1447 1448 case INDEX_op_add2_i32: 1449 opc_new = INDEX_op_add_i32; 1450 goto do_addsub2; 1451 case INDEX_op_sub2_i32: 1452 opc_new = INDEX_op_sub_i32; 1453 goto do_addsub2; 1454 case INDEX_op_add2_i64: 1455 opc_new = INDEX_op_add_i64; 1456 goto do_addsub2; 1457 case INDEX_op_sub2_i64: 1458 opc_new = INDEX_op_sub_i64; 1459 do_addsub2: 1460 nb_iargs = 4; 1461 nb_oargs = 2; 1462 /* Test if the high part of the operation is dead, but not 1463 the low part. The result can be optimized to a simple 1464 add or sub. This happens often for x86_64 guest when the 1465 cpu mode is set to 32 bit. */ 1466 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1467 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1468 goto do_remove; 1469 } 1470 /* Replace the opcode and adjust the args in place, 1471 leaving 3 unused args at the end. */ 1472 op->opc = opc = opc_new; 1473 args[1] = args[2]; 1474 args[2] = args[4]; 1475 /* Fall through and mark the single-word operation live. */ 1476 nb_iargs = 2; 1477 nb_oargs = 1; 1478 } 1479 goto do_not_remove; 1480 1481 case INDEX_op_mulu2_i32: 1482 opc_new = INDEX_op_mul_i32; 1483 opc_new2 = INDEX_op_muluh_i32; 1484 have_opc_new2 = TCG_TARGET_HAS_muluh_i32; 1485 goto do_mul2; 1486 case INDEX_op_muls2_i32: 1487 opc_new = INDEX_op_mul_i32; 1488 opc_new2 = INDEX_op_mulsh_i32; 1489 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32; 1490 goto do_mul2; 1491 case INDEX_op_mulu2_i64: 1492 opc_new = INDEX_op_mul_i64; 1493 opc_new2 = INDEX_op_muluh_i64; 1494 have_opc_new2 = TCG_TARGET_HAS_muluh_i64; 1495 goto do_mul2; 1496 case INDEX_op_muls2_i64: 1497 opc_new = INDEX_op_mul_i64; 1498 opc_new2 = INDEX_op_mulsh_i64; 1499 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64; 1500 goto do_mul2; 1501 do_mul2: 1502 nb_iargs = 2; 1503 nb_oargs = 2; 1504 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1505 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1506 /* Both parts of the operation are dead. */ 1507 goto do_remove; 1508 } 1509 /* The high part of the operation is dead; generate the low. */ 1510 op->opc = opc = opc_new; 1511 args[1] = args[2]; 1512 args[2] = args[3]; 1513 } else if (have_opc_new2 && dead_temps[args[0]] 1514 && !mem_temps[args[0]]) { 1515 /* The low part of the operation is dead; generate the high. */ 1516 op->opc = opc = opc_new2; 1517 args[0] = args[1]; 1518 args[1] = args[2]; 1519 args[2] = args[3]; 1520 } else { 1521 goto do_not_remove; 1522 } 1523 /* Mark the single-word operation live. */ 1524 nb_oargs = 1; 1525 goto do_not_remove; 1526 1527 default: 1528 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 1529 nb_iargs = def->nb_iargs; 1530 nb_oargs = def->nb_oargs; 1531 1532 /* Test if the operation can be removed because all 1533 its outputs are dead. We assume that nb_oargs == 0 1534 implies side effects */ 1535 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 1536 for (i = 0; i < nb_oargs; i++) { 1537 arg = args[i]; 1538 if (!dead_temps[arg] || mem_temps[arg]) { 1539 goto do_not_remove; 1540 } 1541 } 1542 do_remove: 1543 tcg_op_remove(s, op); 1544 } else { 1545 do_not_remove: 1546 /* output args are dead */ 1547 dead_args = 0; 1548 sync_args = 0; 1549 for (i = 0; i < nb_oargs; i++) { 1550 arg = args[i]; 1551 if (dead_temps[arg]) { 1552 dead_args |= (1 << i); 1553 } 1554 if (mem_temps[arg]) { 1555 sync_args |= (1 << i); 1556 } 1557 dead_temps[arg] = 1; 1558 mem_temps[arg] = 0; 1559 } 1560 1561 /* if end of basic block, update */ 1562 if (def->flags & TCG_OPF_BB_END) { 1563 tcg_la_bb_end(s, dead_temps, mem_temps); 1564 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 1565 /* globals should be synced to memory */ 1566 memset(mem_temps, 1, s->nb_globals); 1567 } 1568 1569 /* record arguments that die in this opcode */ 1570 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1571 arg = args[i]; 1572 if (dead_temps[arg]) { 1573 dead_args |= (1 << i); 1574 } 1575 } 1576 /* input arguments are live for preceding opcodes */ 1577 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1578 arg = args[i]; 1579 dead_temps[arg] = 0; 1580 } 1581 s->op_dead_args[oi] = dead_args; 1582 s->op_sync_args[oi] = sync_args; 1583 } 1584 break; 1585 } 1586 } 1587 } 1588 #else 1589 /* dummy liveness analysis */ 1590 static void tcg_liveness_analysis(TCGContext *s) 1591 { 1592 int nb_ops = s->gen_next_op_idx; 1593 1594 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1595 memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t)); 1596 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1597 memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t)); 1598 } 1599 #endif 1600 1601 #ifdef CONFIG_DEBUG_TCG 1602 static void dump_regs(TCGContext *s) 1603 { 1604 TCGTemp *ts; 1605 int i; 1606 char buf[64]; 1607 1608 for(i = 0; i < s->nb_temps; i++) { 1609 ts = &s->temps[i]; 1610 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i)); 1611 switch(ts->val_type) { 1612 case TEMP_VAL_REG: 1613 printf("%s", tcg_target_reg_names[ts->reg]); 1614 break; 1615 case TEMP_VAL_MEM: 1616 printf("%d(%s)", (int)ts->mem_offset, 1617 tcg_target_reg_names[ts->mem_base->reg]); 1618 break; 1619 case TEMP_VAL_CONST: 1620 printf("$0x%" TCG_PRIlx, ts->val); 1621 break; 1622 case TEMP_VAL_DEAD: 1623 printf("D"); 1624 break; 1625 default: 1626 printf("???"); 1627 break; 1628 } 1629 printf("\n"); 1630 } 1631 1632 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1633 if (s->reg_to_temp[i] != NULL) { 1634 printf("%s: %s\n", 1635 tcg_target_reg_names[i], 1636 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); 1637 } 1638 } 1639 } 1640 1641 static void check_regs(TCGContext *s) 1642 { 1643 int reg; 1644 int k; 1645 TCGTemp *ts; 1646 char buf[64]; 1647 1648 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 1649 ts = s->reg_to_temp[reg]; 1650 if (ts != NULL) { 1651 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { 1652 printf("Inconsistency for register %s:\n", 1653 tcg_target_reg_names[reg]); 1654 goto fail; 1655 } 1656 } 1657 } 1658 for (k = 0; k < s->nb_temps; k++) { 1659 ts = &s->temps[k]; 1660 if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg 1661 && s->reg_to_temp[ts->reg] != ts) { 1662 printf("Inconsistency for temp %s:\n", 1663 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); 1664 fail: 1665 printf("reg state:\n"); 1666 dump_regs(s); 1667 tcg_abort(); 1668 } 1669 } 1670 } 1671 #endif 1672 1673 static void temp_allocate_frame(TCGContext *s, int temp) 1674 { 1675 TCGTemp *ts; 1676 ts = &s->temps[temp]; 1677 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 1678 /* Sparc64 stack is accessed with offset of 2047 */ 1679 s->current_frame_offset = (s->current_frame_offset + 1680 (tcg_target_long)sizeof(tcg_target_long) - 1) & 1681 ~(sizeof(tcg_target_long) - 1); 1682 #endif 1683 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 1684 s->frame_end) { 1685 tcg_abort(); 1686 } 1687 ts->mem_offset = s->current_frame_offset; 1688 ts->mem_base = s->frame_temp; 1689 ts->mem_allocated = 1; 1690 s->current_frame_offset += sizeof(tcg_target_long); 1691 } 1692 1693 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet); 1694 1695 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 1696 mark it free; otherwise mark it dead. */ 1697 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 1698 { 1699 if (ts->fixed_reg) { 1700 return; 1701 } 1702 if (ts->val_type == TEMP_VAL_REG) { 1703 s->reg_to_temp[ts->reg] = NULL; 1704 } 1705 ts->val_type = (free_or_dead < 0 1706 || ts->temp_local 1707 || temp_idx(s, ts) < s->nb_globals 1708 ? TEMP_VAL_MEM : TEMP_VAL_DEAD); 1709 } 1710 1711 /* Mark a temporary as dead. */ 1712 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 1713 { 1714 temp_free_or_dead(s, ts, 1); 1715 } 1716 1717 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 1718 registers needs to be allocated to store a constant. If 'free_or_dead' 1719 is non-zero, subsequently release the temporary; if it is positive, the 1720 temp is dead; if it is negative, the temp is free. */ 1721 static void temp_sync(TCGContext *s, TCGTemp *ts, 1722 TCGRegSet allocated_regs, int free_or_dead) 1723 { 1724 if (ts->fixed_reg) { 1725 return; 1726 } 1727 if (!ts->mem_coherent) { 1728 if (!ts->mem_allocated) { 1729 temp_allocate_frame(s, temp_idx(s, ts)); 1730 } 1731 if (ts->indirect_reg) { 1732 if (ts->val_type == TEMP_VAL_REG) { 1733 tcg_regset_set_reg(allocated_regs, ts->reg); 1734 } 1735 temp_load(s, ts->mem_base, 1736 tcg_target_available_regs[TCG_TYPE_PTR], 1737 allocated_regs); 1738 } 1739 switch (ts->val_type) { 1740 case TEMP_VAL_CONST: 1741 /* If we're going to free the temp immediately, then we won't 1742 require it later in a register, so attempt to store the 1743 constant to memory directly. */ 1744 if (free_or_dead 1745 && tcg_out_sti(s, ts->type, ts->val, 1746 ts->mem_base->reg, ts->mem_offset)) { 1747 break; 1748 } 1749 temp_load(s, ts, tcg_target_available_regs[ts->type], 1750 allocated_regs); 1751 /* fallthrough */ 1752 1753 case TEMP_VAL_REG: 1754 tcg_out_st(s, ts->type, ts->reg, 1755 ts->mem_base->reg, ts->mem_offset); 1756 break; 1757 1758 case TEMP_VAL_MEM: 1759 break; 1760 1761 case TEMP_VAL_DEAD: 1762 default: 1763 tcg_abort(); 1764 } 1765 ts->mem_coherent = 1; 1766 } 1767 if (free_or_dead) { 1768 temp_free_or_dead(s, ts, free_or_dead); 1769 } 1770 } 1771 1772 /* free register 'reg' by spilling the corresponding temporary if necessary */ 1773 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 1774 { 1775 TCGTemp *ts = s->reg_to_temp[reg]; 1776 if (ts != NULL) { 1777 temp_sync(s, ts, allocated_regs, -1); 1778 } 1779 } 1780 1781 /* Allocate a register belonging to reg1 & ~reg2 */ 1782 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs, 1783 TCGRegSet allocated_regs, bool rev) 1784 { 1785 int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 1786 const int *order; 1787 TCGReg reg; 1788 TCGRegSet reg_ct; 1789 1790 tcg_regset_andnot(reg_ct, desired_regs, allocated_regs); 1791 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 1792 1793 /* first try free registers */ 1794 for(i = 0; i < n; i++) { 1795 reg = order[i]; 1796 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL) 1797 return reg; 1798 } 1799 1800 /* XXX: do better spill choice */ 1801 for(i = 0; i < n; i++) { 1802 reg = order[i]; 1803 if (tcg_regset_test_reg(reg_ct, reg)) { 1804 tcg_reg_free(s, reg, allocated_regs); 1805 return reg; 1806 } 1807 } 1808 1809 tcg_abort(); 1810 } 1811 1812 /* Make sure the temporary is in a register. If needed, allocate the register 1813 from DESIRED while avoiding ALLOCATED. */ 1814 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 1815 TCGRegSet allocated_regs) 1816 { 1817 TCGReg reg; 1818 1819 switch (ts->val_type) { 1820 case TEMP_VAL_REG: 1821 return; 1822 case TEMP_VAL_CONST: 1823 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); 1824 tcg_out_movi(s, ts->type, reg, ts->val); 1825 ts->mem_coherent = 0; 1826 break; 1827 case TEMP_VAL_MEM: 1828 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base); 1829 if (ts->indirect_reg) { 1830 tcg_regset_set_reg(allocated_regs, reg); 1831 temp_load(s, ts->mem_base, 1832 tcg_target_available_regs[TCG_TYPE_PTR], 1833 allocated_regs); 1834 } 1835 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 1836 ts->mem_coherent = 1; 1837 break; 1838 case TEMP_VAL_DEAD: 1839 default: 1840 tcg_abort(); 1841 } 1842 ts->reg = reg; 1843 ts->val_type = TEMP_VAL_REG; 1844 s->reg_to_temp[reg] = ts; 1845 } 1846 1847 /* Save a temporary to memory. 'allocated_regs' is used in case a 1848 temporary registers needs to be allocated to store a constant. */ 1849 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 1850 { 1851 #ifdef USE_LIVENESS_ANALYSIS 1852 /* ??? Liveness does not yet incorporate indirect bases. */ 1853 if (!ts->indirect_base) { 1854 /* The liveness analysis already ensures that globals are back 1855 in memory. Keep an tcg_debug_assert for safety. */ 1856 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg); 1857 return; 1858 } 1859 #endif 1860 temp_sync(s, ts, allocated_regs, 1); 1861 } 1862 1863 /* save globals to their canonical location and assume they can be 1864 modified be the following code. 'allocated_regs' is used in case a 1865 temporary registers needs to be allocated to store a constant. */ 1866 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 1867 { 1868 int i; 1869 1870 for (i = 0; i < s->nb_globals; i++) { 1871 temp_save(s, &s->temps[i], allocated_regs); 1872 } 1873 } 1874 1875 /* sync globals to their canonical location and assume they can be 1876 read by the following code. 'allocated_regs' is used in case a 1877 temporary registers needs to be allocated to store a constant. */ 1878 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 1879 { 1880 int i; 1881 1882 for (i = 0; i < s->nb_globals; i++) { 1883 TCGTemp *ts = &s->temps[i]; 1884 #ifdef USE_LIVENESS_ANALYSIS 1885 /* ??? Liveness does not yet incorporate indirect bases. */ 1886 if (!ts->indirect_base) { 1887 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 1888 || ts->fixed_reg 1889 || ts->mem_coherent); 1890 continue; 1891 } 1892 #endif 1893 temp_sync(s, ts, allocated_regs, 0); 1894 } 1895 } 1896 1897 /* at the end of a basic block, we assume all temporaries are dead and 1898 all globals are stored at their canonical location. */ 1899 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 1900 { 1901 int i; 1902 1903 for (i = s->nb_globals; i < s->nb_temps; i++) { 1904 TCGTemp *ts = &s->temps[i]; 1905 if (ts->temp_local) { 1906 temp_save(s, ts, allocated_regs); 1907 } else { 1908 #ifdef USE_LIVENESS_ANALYSIS 1909 /* ??? Liveness does not yet incorporate indirect bases. */ 1910 if (!ts->indirect_base) { 1911 /* The liveness analysis already ensures that temps are dead. 1912 Keep an tcg_debug_assert for safety. */ 1913 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 1914 continue; 1915 } 1916 #endif 1917 temp_dead(s, ts); 1918 } 1919 } 1920 1921 save_globals(s, allocated_regs); 1922 } 1923 1924 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1) 1925 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1) 1926 1927 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args, 1928 uint16_t dead_args, uint8_t sync_args) 1929 { 1930 TCGTemp *ots; 1931 tcg_target_ulong val; 1932 1933 ots = &s->temps[args[0]]; 1934 val = args[1]; 1935 1936 if (ots->fixed_reg) { 1937 /* For fixed registers, we do not do any constant propagation. */ 1938 tcg_out_movi(s, ots->type, ots->reg, val); 1939 return; 1940 } 1941 1942 /* The movi is not explicitly generated here. */ 1943 if (ots->val_type == TEMP_VAL_REG) { 1944 s->reg_to_temp[ots->reg] = NULL; 1945 } 1946 ots->val_type = TEMP_VAL_CONST; 1947 ots->val = val; 1948 ots->mem_coherent = 0; 1949 if (NEED_SYNC_ARG(0)) { 1950 temp_sync(s, ots, s->reserved_regs, IS_DEAD_ARG(0)); 1951 } else if (IS_DEAD_ARG(0)) { 1952 temp_dead(s, ots); 1953 } 1954 } 1955 1956 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, 1957 const TCGArg *args, uint16_t dead_args, 1958 uint8_t sync_args) 1959 { 1960 TCGRegSet allocated_regs; 1961 TCGTemp *ts, *ots; 1962 TCGType otype, itype; 1963 1964 tcg_regset_set(allocated_regs, s->reserved_regs); 1965 ots = &s->temps[args[0]]; 1966 ts = &s->temps[args[1]]; 1967 1968 /* Note that otype != itype for no-op truncation. */ 1969 otype = ots->type; 1970 itype = ts->type; 1971 1972 /* If the source value is not in a register, and we're going to be 1973 forced to have it in a register in order to perform the copy, 1974 then copy the SOURCE value into its own register first. That way 1975 we don't have to reload SOURCE the next time it is used. */ 1976 if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG) 1977 || ts->val_type == TEMP_VAL_MEM) { 1978 temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs); 1979 } 1980 1981 if (IS_DEAD_ARG(0) && !ots->fixed_reg) { 1982 /* mov to a non-saved dead register makes no sense (even with 1983 liveness analysis disabled). */ 1984 tcg_debug_assert(NEED_SYNC_ARG(0)); 1985 /* The code above should have moved the temp to a register. */ 1986 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 1987 if (!ots->mem_allocated) { 1988 temp_allocate_frame(s, args[0]); 1989 } 1990 if (ots->indirect_reg) { 1991 tcg_regset_set_reg(allocated_regs, ts->reg); 1992 temp_load(s, ots->mem_base, 1993 tcg_target_available_regs[TCG_TYPE_PTR], 1994 allocated_regs); 1995 } 1996 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); 1997 if (IS_DEAD_ARG(1)) { 1998 temp_dead(s, ts); 1999 } 2000 temp_dead(s, ots); 2001 } else if (ts->val_type == TEMP_VAL_CONST) { 2002 /* propagate constant */ 2003 if (ots->val_type == TEMP_VAL_REG) { 2004 s->reg_to_temp[ots->reg] = NULL; 2005 } 2006 ots->val_type = TEMP_VAL_CONST; 2007 ots->val = ts->val; 2008 if (IS_DEAD_ARG(1)) { 2009 temp_dead(s, ts); 2010 } 2011 } else { 2012 /* The code in the first if block should have moved the 2013 temp to a register. */ 2014 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 2015 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 2016 /* the mov can be suppressed */ 2017 if (ots->val_type == TEMP_VAL_REG) { 2018 s->reg_to_temp[ots->reg] = NULL; 2019 } 2020 ots->reg = ts->reg; 2021 temp_dead(s, ts); 2022 } else { 2023 if (ots->val_type != TEMP_VAL_REG) { 2024 /* When allocating a new register, make sure to not spill the 2025 input one. */ 2026 tcg_regset_set_reg(allocated_regs, ts->reg); 2027 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 2028 allocated_regs, ots->indirect_base); 2029 } 2030 tcg_out_mov(s, otype, ots->reg, ts->reg); 2031 } 2032 ots->val_type = TEMP_VAL_REG; 2033 ots->mem_coherent = 0; 2034 s->reg_to_temp[ots->reg] = ots; 2035 if (NEED_SYNC_ARG(0)) { 2036 temp_sync(s, ots, allocated_regs, 0); 2037 } 2038 } 2039 } 2040 2041 static void tcg_reg_alloc_op(TCGContext *s, 2042 const TCGOpDef *def, TCGOpcode opc, 2043 const TCGArg *args, uint16_t dead_args, 2044 uint8_t sync_args) 2045 { 2046 TCGRegSet allocated_regs; 2047 int i, k, nb_iargs, nb_oargs; 2048 TCGReg reg; 2049 TCGArg arg; 2050 const TCGArgConstraint *arg_ct; 2051 TCGTemp *ts; 2052 TCGArg new_args[TCG_MAX_OP_ARGS]; 2053 int const_args[TCG_MAX_OP_ARGS]; 2054 2055 nb_oargs = def->nb_oargs; 2056 nb_iargs = def->nb_iargs; 2057 2058 /* copy constants */ 2059 memcpy(new_args + nb_oargs + nb_iargs, 2060 args + nb_oargs + nb_iargs, 2061 sizeof(TCGArg) * def->nb_cargs); 2062 2063 /* satisfy input constraints */ 2064 tcg_regset_set(allocated_regs, s->reserved_regs); 2065 for(k = 0; k < nb_iargs; k++) { 2066 i = def->sorted_args[nb_oargs + k]; 2067 arg = args[i]; 2068 arg_ct = &def->args_ct[i]; 2069 ts = &s->temps[arg]; 2070 2071 if (ts->val_type == TEMP_VAL_CONST 2072 && tcg_target_const_match(ts->val, ts->type, arg_ct)) { 2073 /* constant is OK for instruction */ 2074 const_args[i] = 1; 2075 new_args[i] = ts->val; 2076 goto iarg_end; 2077 } 2078 2079 temp_load(s, ts, arg_ct->u.regs, allocated_regs); 2080 2081 if (arg_ct->ct & TCG_CT_IALIAS) { 2082 if (ts->fixed_reg) { 2083 /* if fixed register, we must allocate a new register 2084 if the alias is not the same register */ 2085 if (arg != args[arg_ct->alias_index]) 2086 goto allocate_in_reg; 2087 } else { 2088 /* if the input is aliased to an output and if it is 2089 not dead after the instruction, we must allocate 2090 a new register and move it */ 2091 if (!IS_DEAD_ARG(i)) { 2092 goto allocate_in_reg; 2093 } 2094 /* check if the current register has already been allocated 2095 for another input aliased to an output */ 2096 int k2, i2; 2097 for (k2 = 0 ; k2 < k ; k2++) { 2098 i2 = def->sorted_args[nb_oargs + k2]; 2099 if ((def->args_ct[i2].ct & TCG_CT_IALIAS) && 2100 (new_args[i2] == ts->reg)) { 2101 goto allocate_in_reg; 2102 } 2103 } 2104 } 2105 } 2106 reg = ts->reg; 2107 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2108 /* nothing to do : the constraint is satisfied */ 2109 } else { 2110 allocate_in_reg: 2111 /* allocate a new register matching the constraint 2112 and move the temporary register into it */ 2113 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs, 2114 ts->indirect_base); 2115 tcg_out_mov(s, ts->type, reg, ts->reg); 2116 } 2117 new_args[i] = reg; 2118 const_args[i] = 0; 2119 tcg_regset_set_reg(allocated_regs, reg); 2120 iarg_end: ; 2121 } 2122 2123 /* mark dead temporaries and free the associated registers */ 2124 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2125 if (IS_DEAD_ARG(i)) { 2126 temp_dead(s, &s->temps[args[i]]); 2127 } 2128 } 2129 2130 if (def->flags & TCG_OPF_BB_END) { 2131 tcg_reg_alloc_bb_end(s, allocated_regs); 2132 } else { 2133 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2134 /* XXX: permit generic clobber register list ? */ 2135 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 2136 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 2137 tcg_reg_free(s, i, allocated_regs); 2138 } 2139 } 2140 } 2141 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2142 /* sync globals if the op has side effects and might trigger 2143 an exception. */ 2144 sync_globals(s, allocated_regs); 2145 } 2146 2147 /* satisfy the output constraints */ 2148 tcg_regset_set(allocated_regs, s->reserved_regs); 2149 for(k = 0; k < nb_oargs; k++) { 2150 i = def->sorted_args[k]; 2151 arg = args[i]; 2152 arg_ct = &def->args_ct[i]; 2153 ts = &s->temps[arg]; 2154 if (arg_ct->ct & TCG_CT_ALIAS) { 2155 reg = new_args[arg_ct->alias_index]; 2156 } else { 2157 /* if fixed register, we try to use it */ 2158 reg = ts->reg; 2159 if (ts->fixed_reg && 2160 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2161 goto oarg_end; 2162 } 2163 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs, 2164 ts->indirect_base); 2165 } 2166 tcg_regset_set_reg(allocated_regs, reg); 2167 /* if a fixed register is used, then a move will be done afterwards */ 2168 if (!ts->fixed_reg) { 2169 if (ts->val_type == TEMP_VAL_REG) { 2170 s->reg_to_temp[ts->reg] = NULL; 2171 } 2172 ts->val_type = TEMP_VAL_REG; 2173 ts->reg = reg; 2174 /* temp value is modified, so the value kept in memory is 2175 potentially not the same */ 2176 ts->mem_coherent = 0; 2177 s->reg_to_temp[reg] = ts; 2178 } 2179 oarg_end: 2180 new_args[i] = reg; 2181 } 2182 } 2183 2184 /* emit instruction */ 2185 tcg_out_op(s, opc, new_args, const_args); 2186 2187 /* move the outputs in the correct register if needed */ 2188 for(i = 0; i < nb_oargs; i++) { 2189 ts = &s->temps[args[i]]; 2190 reg = new_args[i]; 2191 if (ts->fixed_reg && ts->reg != reg) { 2192 tcg_out_mov(s, ts->type, ts->reg, reg); 2193 } 2194 if (NEED_SYNC_ARG(i)) { 2195 temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i)); 2196 } else if (IS_DEAD_ARG(i)) { 2197 temp_dead(s, ts); 2198 } 2199 } 2200 } 2201 2202 #ifdef TCG_TARGET_STACK_GROWSUP 2203 #define STACK_DIR(x) (-(x)) 2204 #else 2205 #define STACK_DIR(x) (x) 2206 #endif 2207 2208 static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs, 2209 const TCGArg * const args, uint16_t dead_args, 2210 uint8_t sync_args) 2211 { 2212 int flags, nb_regs, i; 2213 TCGReg reg; 2214 TCGArg arg; 2215 TCGTemp *ts; 2216 intptr_t stack_offset; 2217 size_t call_stack_size; 2218 tcg_insn_unit *func_addr; 2219 int allocate_args; 2220 TCGRegSet allocated_regs; 2221 2222 func_addr = (tcg_insn_unit *)(intptr_t)args[nb_oargs + nb_iargs]; 2223 flags = args[nb_oargs + nb_iargs + 1]; 2224 2225 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2226 if (nb_regs > nb_iargs) { 2227 nb_regs = nb_iargs; 2228 } 2229 2230 /* assign stack slots first */ 2231 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); 2232 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 2233 ~(TCG_TARGET_STACK_ALIGN - 1); 2234 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 2235 if (allocate_args) { 2236 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 2237 preallocate call stack */ 2238 tcg_abort(); 2239 } 2240 2241 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 2242 for(i = nb_regs; i < nb_iargs; i++) { 2243 arg = args[nb_oargs + i]; 2244 #ifdef TCG_TARGET_STACK_GROWSUP 2245 stack_offset -= sizeof(tcg_target_long); 2246 #endif 2247 if (arg != TCG_CALL_DUMMY_ARG) { 2248 ts = &s->temps[arg]; 2249 temp_load(s, ts, tcg_target_available_regs[ts->type], 2250 s->reserved_regs); 2251 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 2252 } 2253 #ifndef TCG_TARGET_STACK_GROWSUP 2254 stack_offset += sizeof(tcg_target_long); 2255 #endif 2256 } 2257 2258 /* assign input registers */ 2259 tcg_regset_set(allocated_regs, s->reserved_regs); 2260 for(i = 0; i < nb_regs; i++) { 2261 arg = args[nb_oargs + i]; 2262 if (arg != TCG_CALL_DUMMY_ARG) { 2263 ts = &s->temps[arg]; 2264 reg = tcg_target_call_iarg_regs[i]; 2265 tcg_reg_free(s, reg, allocated_regs); 2266 2267 if (ts->val_type == TEMP_VAL_REG) { 2268 if (ts->reg != reg) { 2269 tcg_out_mov(s, ts->type, reg, ts->reg); 2270 } 2271 } else { 2272 TCGRegSet arg_set; 2273 2274 tcg_regset_clear(arg_set); 2275 tcg_regset_set_reg(arg_set, reg); 2276 temp_load(s, ts, arg_set, allocated_regs); 2277 } 2278 2279 tcg_regset_set_reg(allocated_regs, reg); 2280 } 2281 } 2282 2283 /* mark dead temporaries and free the associated registers */ 2284 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2285 if (IS_DEAD_ARG(i)) { 2286 temp_dead(s, &s->temps[args[i]]); 2287 } 2288 } 2289 2290 /* clobber call registers */ 2291 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 2292 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 2293 tcg_reg_free(s, i, allocated_regs); 2294 } 2295 } 2296 2297 /* Save globals if they might be written by the helper, sync them if 2298 they might be read. */ 2299 if (flags & TCG_CALL_NO_READ_GLOBALS) { 2300 /* Nothing to do */ 2301 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 2302 sync_globals(s, allocated_regs); 2303 } else { 2304 save_globals(s, allocated_regs); 2305 } 2306 2307 tcg_out_call(s, func_addr); 2308 2309 /* assign output registers and emit moves if needed */ 2310 for(i = 0; i < nb_oargs; i++) { 2311 arg = args[i]; 2312 ts = &s->temps[arg]; 2313 reg = tcg_target_call_oarg_regs[i]; 2314 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 2315 2316 if (ts->fixed_reg) { 2317 if (ts->reg != reg) { 2318 tcg_out_mov(s, ts->type, ts->reg, reg); 2319 } 2320 } else { 2321 if (ts->val_type == TEMP_VAL_REG) { 2322 s->reg_to_temp[ts->reg] = NULL; 2323 } 2324 ts->val_type = TEMP_VAL_REG; 2325 ts->reg = reg; 2326 ts->mem_coherent = 0; 2327 s->reg_to_temp[reg] = ts; 2328 if (NEED_SYNC_ARG(i)) { 2329 temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i)); 2330 } else if (IS_DEAD_ARG(i)) { 2331 temp_dead(s, ts); 2332 } 2333 } 2334 } 2335 } 2336 2337 #ifdef CONFIG_PROFILER 2338 2339 static int64_t tcg_table_op_count[NB_OPS]; 2340 2341 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 2342 { 2343 int i; 2344 2345 for (i = 0; i < NB_OPS; i++) { 2346 cpu_fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, 2347 tcg_table_op_count[i]); 2348 } 2349 } 2350 #else 2351 void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf) 2352 { 2353 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2354 } 2355 #endif 2356 2357 2358 int tcg_gen_code(TCGContext *s, TranslationBlock *tb) 2359 { 2360 int i, oi, oi_next, num_insns; 2361 2362 #ifdef CONFIG_PROFILER 2363 { 2364 int n; 2365 2366 n = s->gen_last_op_idx + 1; 2367 s->op_count += n; 2368 if (n > s->op_count_max) { 2369 s->op_count_max = n; 2370 } 2371 2372 n = s->nb_temps; 2373 s->temp_count += n; 2374 if (n > s->temp_count_max) { 2375 s->temp_count_max = n; 2376 } 2377 } 2378 #endif 2379 2380 #ifdef DEBUG_DISAS 2381 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 2382 && qemu_log_in_addr_range(tb->pc))) { 2383 qemu_log("OP:\n"); 2384 tcg_dump_ops(s); 2385 qemu_log("\n"); 2386 } 2387 #endif 2388 2389 #ifdef CONFIG_PROFILER 2390 s->opt_time -= profile_getclock(); 2391 #endif 2392 2393 #ifdef USE_TCG_OPTIMIZATIONS 2394 tcg_optimize(s); 2395 #endif 2396 2397 #ifdef CONFIG_PROFILER 2398 s->opt_time += profile_getclock(); 2399 s->la_time -= profile_getclock(); 2400 #endif 2401 2402 tcg_liveness_analysis(s); 2403 2404 #ifdef CONFIG_PROFILER 2405 s->la_time += profile_getclock(); 2406 #endif 2407 2408 #ifdef DEBUG_DISAS 2409 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 2410 && qemu_log_in_addr_range(tb->pc))) { 2411 qemu_log("OP after optimization and liveness analysis:\n"); 2412 tcg_dump_ops(s); 2413 qemu_log("\n"); 2414 } 2415 #endif 2416 2417 tcg_reg_alloc_start(s); 2418 2419 s->code_buf = tb->tc_ptr; 2420 s->code_ptr = tb->tc_ptr; 2421 2422 tcg_out_tb_init(s); 2423 2424 num_insns = -1; 2425 for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) { 2426 TCGOp * const op = &s->gen_op_buf[oi]; 2427 TCGArg * const args = &s->gen_opparam_buf[op->args]; 2428 TCGOpcode opc = op->opc; 2429 const TCGOpDef *def = &tcg_op_defs[opc]; 2430 uint16_t dead_args = s->op_dead_args[oi]; 2431 uint8_t sync_args = s->op_sync_args[oi]; 2432 2433 oi_next = op->next; 2434 #ifdef CONFIG_PROFILER 2435 tcg_table_op_count[opc]++; 2436 #endif 2437 2438 switch (opc) { 2439 case INDEX_op_mov_i32: 2440 case INDEX_op_mov_i64: 2441 tcg_reg_alloc_mov(s, def, args, dead_args, sync_args); 2442 break; 2443 case INDEX_op_movi_i32: 2444 case INDEX_op_movi_i64: 2445 tcg_reg_alloc_movi(s, args, dead_args, sync_args); 2446 break; 2447 case INDEX_op_insn_start: 2448 if (num_insns >= 0) { 2449 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 2450 } 2451 num_insns++; 2452 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) { 2453 target_ulong a; 2454 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 2455 a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2]; 2456 #else 2457 a = args[i]; 2458 #endif 2459 s->gen_insn_data[num_insns][i] = a; 2460 } 2461 break; 2462 case INDEX_op_discard: 2463 temp_dead(s, &s->temps[args[0]]); 2464 break; 2465 case INDEX_op_set_label: 2466 tcg_reg_alloc_bb_end(s, s->reserved_regs); 2467 tcg_out_label(s, arg_label(args[0]), s->code_ptr); 2468 break; 2469 case INDEX_op_call: 2470 tcg_reg_alloc_call(s, op->callo, op->calli, args, 2471 dead_args, sync_args); 2472 break; 2473 default: 2474 /* Sanity check that we've not introduced any unhandled opcodes. */ 2475 if (def->flags & TCG_OPF_NOT_PRESENT) { 2476 tcg_abort(); 2477 } 2478 /* Note: in order to speed up the code, it would be much 2479 faster to have specialized register allocator functions for 2480 some common argument patterns */ 2481 tcg_reg_alloc_op(s, def, opc, args, dead_args, sync_args); 2482 break; 2483 } 2484 #ifdef CONFIG_DEBUG_TCG 2485 check_regs(s); 2486 #endif 2487 /* Test for (pending) buffer overflow. The assumption is that any 2488 one operation beginning below the high water mark cannot overrun 2489 the buffer completely. Thus we can test for overflow after 2490 generating code without having to check during generation. */ 2491 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 2492 return -1; 2493 } 2494 } 2495 tcg_debug_assert(num_insns >= 0); 2496 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 2497 2498 /* Generate TB finalization at the end of block */ 2499 if (!tcg_out_tb_finalize(s)) { 2500 return -1; 2501 } 2502 2503 /* flush instruction cache */ 2504 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 2505 2506 return tcg_current_code_size(s); 2507 } 2508 2509 #ifdef CONFIG_PROFILER 2510 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2511 { 2512 TCGContext *s = &tcg_ctx; 2513 int64_t tb_count = s->tb_count; 2514 int64_t tb_div_count = tb_count ? tb_count : 1; 2515 int64_t tot = s->interm_time + s->code_time; 2516 2517 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 2518 tot, tot / 2.4e9); 2519 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 2520 tb_count, s->tb_count1 - tb_count, 2521 (double)(s->tb_count1 - s->tb_count) 2522 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0); 2523 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 2524 (double)s->op_count / tb_div_count, s->op_count_max); 2525 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 2526 (double)s->del_op_count / tb_div_count); 2527 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 2528 (double)s->temp_count / tb_div_count, s->temp_count_max); 2529 cpu_fprintf(f, "avg host code/TB %0.1f\n", 2530 (double)s->code_out_len / tb_div_count); 2531 cpu_fprintf(f, "avg search data/TB %0.1f\n", 2532 (double)s->search_out_len / tb_div_count); 2533 2534 cpu_fprintf(f, "cycles/op %0.1f\n", 2535 s->op_count ? (double)tot / s->op_count : 0); 2536 cpu_fprintf(f, "cycles/in byte %0.1f\n", 2537 s->code_in_len ? (double)tot / s->code_in_len : 0); 2538 cpu_fprintf(f, "cycles/out byte %0.1f\n", 2539 s->code_out_len ? (double)tot / s->code_out_len : 0); 2540 cpu_fprintf(f, "cycles/search byte %0.1f\n", 2541 s->search_out_len ? (double)tot / s->search_out_len : 0); 2542 if (tot == 0) { 2543 tot = 1; 2544 } 2545 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 2546 (double)s->interm_time / tot * 100.0); 2547 cpu_fprintf(f, " gen_code time %0.1f%%\n", 2548 (double)s->code_time / tot * 100.0); 2549 cpu_fprintf(f, "optim./code time %0.1f%%\n", 2550 (double)s->opt_time / (s->code_time ? s->code_time : 1) 2551 * 100.0); 2552 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 2553 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 2554 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 2555 s->restore_count); 2556 cpu_fprintf(f, " avg cycles %0.1f\n", 2557 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 2558 } 2559 #else 2560 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2561 { 2562 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2563 } 2564 #endif 2565 2566 #ifdef ELF_HOST_MACHINE 2567 /* In order to use this feature, the backend needs to do three things: 2568 2569 (1) Define ELF_HOST_MACHINE to indicate both what value to 2570 put into the ELF image and to indicate support for the feature. 2571 2572 (2) Define tcg_register_jit. This should create a buffer containing 2573 the contents of a .debug_frame section that describes the post- 2574 prologue unwind info for the tcg machine. 2575 2576 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 2577 */ 2578 2579 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 2580 typedef enum { 2581 JIT_NOACTION = 0, 2582 JIT_REGISTER_FN, 2583 JIT_UNREGISTER_FN 2584 } jit_actions_t; 2585 2586 struct jit_code_entry { 2587 struct jit_code_entry *next_entry; 2588 struct jit_code_entry *prev_entry; 2589 const void *symfile_addr; 2590 uint64_t symfile_size; 2591 }; 2592 2593 struct jit_descriptor { 2594 uint32_t version; 2595 uint32_t action_flag; 2596 struct jit_code_entry *relevant_entry; 2597 struct jit_code_entry *first_entry; 2598 }; 2599 2600 void __jit_debug_register_code(void) __attribute__((noinline)); 2601 void __jit_debug_register_code(void) 2602 { 2603 asm(""); 2604 } 2605 2606 /* Must statically initialize the version, because GDB may check 2607 the version before we can set it. */ 2608 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 2609 2610 /* End GDB interface. */ 2611 2612 static int find_string(const char *strtab, const char *str) 2613 { 2614 const char *p = strtab + 1; 2615 2616 while (1) { 2617 if (strcmp(p, str) == 0) { 2618 return p - strtab; 2619 } 2620 p += strlen(p) + 1; 2621 } 2622 } 2623 2624 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 2625 const void *debug_frame, 2626 size_t debug_frame_size) 2627 { 2628 struct __attribute__((packed)) DebugInfo { 2629 uint32_t len; 2630 uint16_t version; 2631 uint32_t abbrev; 2632 uint8_t ptr_size; 2633 uint8_t cu_die; 2634 uint16_t cu_lang; 2635 uintptr_t cu_low_pc; 2636 uintptr_t cu_high_pc; 2637 uint8_t fn_die; 2638 char fn_name[16]; 2639 uintptr_t fn_low_pc; 2640 uintptr_t fn_high_pc; 2641 uint8_t cu_eoc; 2642 }; 2643 2644 struct ElfImage { 2645 ElfW(Ehdr) ehdr; 2646 ElfW(Phdr) phdr; 2647 ElfW(Shdr) shdr[7]; 2648 ElfW(Sym) sym[2]; 2649 struct DebugInfo di; 2650 uint8_t da[24]; 2651 char str[80]; 2652 }; 2653 2654 struct ElfImage *img; 2655 2656 static const struct ElfImage img_template = { 2657 .ehdr = { 2658 .e_ident[EI_MAG0] = ELFMAG0, 2659 .e_ident[EI_MAG1] = ELFMAG1, 2660 .e_ident[EI_MAG2] = ELFMAG2, 2661 .e_ident[EI_MAG3] = ELFMAG3, 2662 .e_ident[EI_CLASS] = ELF_CLASS, 2663 .e_ident[EI_DATA] = ELF_DATA, 2664 .e_ident[EI_VERSION] = EV_CURRENT, 2665 .e_type = ET_EXEC, 2666 .e_machine = ELF_HOST_MACHINE, 2667 .e_version = EV_CURRENT, 2668 .e_phoff = offsetof(struct ElfImage, phdr), 2669 .e_shoff = offsetof(struct ElfImage, shdr), 2670 .e_ehsize = sizeof(ElfW(Shdr)), 2671 .e_phentsize = sizeof(ElfW(Phdr)), 2672 .e_phnum = 1, 2673 .e_shentsize = sizeof(ElfW(Shdr)), 2674 .e_shnum = ARRAY_SIZE(img->shdr), 2675 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 2676 #ifdef ELF_HOST_FLAGS 2677 .e_flags = ELF_HOST_FLAGS, 2678 #endif 2679 #ifdef ELF_OSABI 2680 .e_ident[EI_OSABI] = ELF_OSABI, 2681 #endif 2682 }, 2683 .phdr = { 2684 .p_type = PT_LOAD, 2685 .p_flags = PF_X, 2686 }, 2687 .shdr = { 2688 [0] = { .sh_type = SHT_NULL }, 2689 /* Trick: The contents of code_gen_buffer are not present in 2690 this fake ELF file; that got allocated elsewhere. Therefore 2691 we mark .text as SHT_NOBITS (similar to .bss) so that readers 2692 will not look for contents. We can record any address. */ 2693 [1] = { /* .text */ 2694 .sh_type = SHT_NOBITS, 2695 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 2696 }, 2697 [2] = { /* .debug_info */ 2698 .sh_type = SHT_PROGBITS, 2699 .sh_offset = offsetof(struct ElfImage, di), 2700 .sh_size = sizeof(struct DebugInfo), 2701 }, 2702 [3] = { /* .debug_abbrev */ 2703 .sh_type = SHT_PROGBITS, 2704 .sh_offset = offsetof(struct ElfImage, da), 2705 .sh_size = sizeof(img->da), 2706 }, 2707 [4] = { /* .debug_frame */ 2708 .sh_type = SHT_PROGBITS, 2709 .sh_offset = sizeof(struct ElfImage), 2710 }, 2711 [5] = { /* .symtab */ 2712 .sh_type = SHT_SYMTAB, 2713 .sh_offset = offsetof(struct ElfImage, sym), 2714 .sh_size = sizeof(img->sym), 2715 .sh_info = 1, 2716 .sh_link = ARRAY_SIZE(img->shdr) - 1, 2717 .sh_entsize = sizeof(ElfW(Sym)), 2718 }, 2719 [6] = { /* .strtab */ 2720 .sh_type = SHT_STRTAB, 2721 .sh_offset = offsetof(struct ElfImage, str), 2722 .sh_size = sizeof(img->str), 2723 } 2724 }, 2725 .sym = { 2726 [1] = { /* code_gen_buffer */ 2727 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 2728 .st_shndx = 1, 2729 } 2730 }, 2731 .di = { 2732 .len = sizeof(struct DebugInfo) - 4, 2733 .version = 2, 2734 .ptr_size = sizeof(void *), 2735 .cu_die = 1, 2736 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 2737 .fn_die = 2, 2738 .fn_name = "code_gen_buffer" 2739 }, 2740 .da = { 2741 1, /* abbrev number (the cu) */ 2742 0x11, 1, /* DW_TAG_compile_unit, has children */ 2743 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 2744 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2745 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2746 0, 0, /* end of abbrev */ 2747 2, /* abbrev number (the fn) */ 2748 0x2e, 0, /* DW_TAG_subprogram, no children */ 2749 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 2750 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2751 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2752 0, 0, /* end of abbrev */ 2753 0 /* no more abbrev */ 2754 }, 2755 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 2756 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 2757 }; 2758 2759 /* We only need a single jit entry; statically allocate it. */ 2760 static struct jit_code_entry one_entry; 2761 2762 uintptr_t buf = (uintptr_t)buf_ptr; 2763 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 2764 DebugFrameHeader *dfh; 2765 2766 img = g_malloc(img_size); 2767 *img = img_template; 2768 2769 img->phdr.p_vaddr = buf; 2770 img->phdr.p_paddr = buf; 2771 img->phdr.p_memsz = buf_size; 2772 2773 img->shdr[1].sh_name = find_string(img->str, ".text"); 2774 img->shdr[1].sh_addr = buf; 2775 img->shdr[1].sh_size = buf_size; 2776 2777 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 2778 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 2779 2780 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 2781 img->shdr[4].sh_size = debug_frame_size; 2782 2783 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 2784 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 2785 2786 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 2787 img->sym[1].st_value = buf; 2788 img->sym[1].st_size = buf_size; 2789 2790 img->di.cu_low_pc = buf; 2791 img->di.cu_high_pc = buf + buf_size; 2792 img->di.fn_low_pc = buf; 2793 img->di.fn_high_pc = buf + buf_size; 2794 2795 dfh = (DebugFrameHeader *)(img + 1); 2796 memcpy(dfh, debug_frame, debug_frame_size); 2797 dfh->fde.func_start = buf; 2798 dfh->fde.func_len = buf_size; 2799 2800 #ifdef DEBUG_JIT 2801 /* Enable this block to be able to debug the ELF image file creation. 2802 One can use readelf, objdump, or other inspection utilities. */ 2803 { 2804 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 2805 if (f) { 2806 if (fwrite(img, img_size, 1, f) != img_size) { 2807 /* Avoid stupid unused return value warning for fwrite. */ 2808 } 2809 fclose(f); 2810 } 2811 } 2812 #endif 2813 2814 one_entry.symfile_addr = img; 2815 one_entry.symfile_size = img_size; 2816 2817 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 2818 __jit_debug_descriptor.relevant_entry = &one_entry; 2819 __jit_debug_descriptor.first_entry = &one_entry; 2820 __jit_debug_register_code(); 2821 } 2822 #else 2823 /* No support for the feature. Provide the entry point expected by exec.c, 2824 and implement the internal function we declared earlier. */ 2825 2826 static void tcg_register_jit_int(void *buf, size_t size, 2827 const void *debug_frame, 2828 size_t debug_frame_size) 2829 { 2830 } 2831 2832 void tcg_register_jit(void *buf, size_t buf_size) 2833 { 2834 } 2835 #endif /* ELF_HOST_MACHINE */ 2836