1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/units.h" 22 #include "qemu-common.h" 23 24 #define NO_CPU_IO_DEFS 25 #include "cpu.h" 26 #include "trace.h" 27 #include "disas/disas.h" 28 #include "exec/exec-all.h" 29 #include "tcg/tcg.h" 30 #if defined(CONFIG_USER_ONLY) 31 #include "qemu.h" 32 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) 33 #include <sys/param.h> 34 #if __FreeBSD_version >= 700104 35 #define HAVE_KINFO_GETVMMAP 36 #define sigqueue sigqueue_freebsd /* avoid redefinition */ 37 #include <sys/proc.h> 38 #include <machine/profile.h> 39 #define _KERNEL 40 #include <sys/user.h> 41 #undef _KERNEL 42 #undef sigqueue 43 #include <libutil.h> 44 #endif 45 #endif 46 #else 47 #include "exec/ram_addr.h" 48 #endif 49 50 #include "exec/cputlb.h" 51 #include "exec/tb-hash.h" 52 #include "exec/translate-all.h" 53 #include "qemu/bitmap.h" 54 #include "qemu/error-report.h" 55 #include "qemu/qemu-print.h" 56 #include "qemu/timer.h" 57 #include "qemu/main-loop.h" 58 #include "exec/log.h" 59 #include "sysemu/cpus.h" 60 #include "sysemu/cpu-timers.h" 61 #include "sysemu/tcg.h" 62 #include "qapi/error.h" 63 #include "hw/core/tcg-cpu-ops.h" 64 #include "internal.h" 65 66 /* #define DEBUG_TB_INVALIDATE */ 67 /* #define DEBUG_TB_FLUSH */ 68 /* make various TB consistency checks */ 69 /* #define DEBUG_TB_CHECK */ 70 71 #ifdef DEBUG_TB_INVALIDATE 72 #define DEBUG_TB_INVALIDATE_GATE 1 73 #else 74 #define DEBUG_TB_INVALIDATE_GATE 0 75 #endif 76 77 #ifdef DEBUG_TB_FLUSH 78 #define DEBUG_TB_FLUSH_GATE 1 79 #else 80 #define DEBUG_TB_FLUSH_GATE 0 81 #endif 82 83 #if !defined(CONFIG_USER_ONLY) 84 /* TB consistency checks only implemented for usermode emulation. */ 85 #undef DEBUG_TB_CHECK 86 #endif 87 88 #ifdef DEBUG_TB_CHECK 89 #define DEBUG_TB_CHECK_GATE 1 90 #else 91 #define DEBUG_TB_CHECK_GATE 0 92 #endif 93 94 /* Access to the various translations structures need to be serialised via locks 95 * for consistency. 96 * In user-mode emulation access to the memory related structures are protected 97 * with mmap_lock. 98 * In !user-mode we use per-page locks. 99 */ 100 #ifdef CONFIG_SOFTMMU 101 #define assert_memory_lock() 102 #else 103 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock()) 104 #endif 105 106 #define SMC_BITMAP_USE_THRESHOLD 10 107 108 typedef struct PageDesc { 109 /* list of TBs intersecting this ram page */ 110 uintptr_t first_tb; 111 #ifdef CONFIG_SOFTMMU 112 /* in order to optimize self modifying code, we count the number 113 of lookups we do to a given page to use a bitmap */ 114 unsigned long *code_bitmap; 115 unsigned int code_write_count; 116 #else 117 unsigned long flags; 118 void *target_data; 119 #endif 120 #ifndef CONFIG_USER_ONLY 121 QemuSpin lock; 122 #endif 123 } PageDesc; 124 125 /** 126 * struct page_entry - page descriptor entry 127 * @pd: pointer to the &struct PageDesc of the page this entry represents 128 * @index: page index of the page 129 * @locked: whether the page is locked 130 * 131 * This struct helps us keep track of the locked state of a page, without 132 * bloating &struct PageDesc. 133 * 134 * A page lock protects accesses to all fields of &struct PageDesc. 135 * 136 * See also: &struct page_collection. 137 */ 138 struct page_entry { 139 PageDesc *pd; 140 tb_page_addr_t index; 141 bool locked; 142 }; 143 144 /** 145 * struct page_collection - tracks a set of pages (i.e. &struct page_entry's) 146 * @tree: Binary search tree (BST) of the pages, with key == page index 147 * @max: Pointer to the page in @tree with the highest page index 148 * 149 * To avoid deadlock we lock pages in ascending order of page index. 150 * When operating on a set of pages, we need to keep track of them so that 151 * we can lock them in order and also unlock them later. For this we collect 152 * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the 153 * @tree implementation we use does not provide an O(1) operation to obtain the 154 * highest-ranked element, we use @max to keep track of the inserted page 155 * with the highest index. This is valuable because if a page is not in 156 * the tree and its index is higher than @max's, then we can lock it 157 * without breaking the locking order rule. 158 * 159 * Note on naming: 'struct page_set' would be shorter, but we already have a few 160 * page_set_*() helpers, so page_collection is used instead to avoid confusion. 161 * 162 * See also: page_collection_lock(). 163 */ 164 struct page_collection { 165 GTree *tree; 166 struct page_entry *max; 167 }; 168 169 /* list iterators for lists of tagged pointers in TranslationBlock */ 170 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \ 171 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \ 172 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \ 173 tb = (TranslationBlock *)((uintptr_t)tb & ~1)) 174 175 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \ 176 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next) 177 178 #define TB_FOR_EACH_JMP(head_tb, tb, n) \ 179 TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next) 180 181 /* 182 * In system mode we want L1_MAP to be based on ram offsets, 183 * while in user mode we want it to be based on virtual addresses. 184 * 185 * TODO: For user mode, see the caveat re host vs guest virtual 186 * address spaces near GUEST_ADDR_MAX. 187 */ 188 #if !defined(CONFIG_USER_ONLY) 189 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS 190 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS 191 #else 192 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS 193 #endif 194 #else 195 # define L1_MAP_ADDR_SPACE_BITS MIN(HOST_LONG_BITS, TARGET_ABI_BITS) 196 #endif 197 198 /* Size of the L2 (and L3, etc) page tables. */ 199 #define V_L2_BITS 10 200 #define V_L2_SIZE (1 << V_L2_BITS) 201 202 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */ 203 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS > 204 sizeof_field(TranslationBlock, trace_vcpu_dstate) 205 * BITS_PER_BYTE); 206 207 /* 208 * L1 Mapping properties 209 */ 210 static int v_l1_size; 211 static int v_l1_shift; 212 static int v_l2_levels; 213 214 /* The bottom level has pointers to PageDesc, and is indexed by 215 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size. 216 */ 217 #define V_L1_MIN_BITS 4 218 #define V_L1_MAX_BITS (V_L2_BITS + 3) 219 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS) 220 221 static void *l1_map[V_L1_MAX_SIZE]; 222 223 /* code generation context */ 224 TCGContext tcg_init_ctx; 225 __thread TCGContext *tcg_ctx; 226 TBContext tb_ctx; 227 bool parallel_cpus; 228 229 static void page_table_config_init(void) 230 { 231 uint32_t v_l1_bits; 232 233 assert(TARGET_PAGE_BITS); 234 /* The bits remaining after N lower levels of page tables. */ 235 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS; 236 if (v_l1_bits < V_L1_MIN_BITS) { 237 v_l1_bits += V_L2_BITS; 238 } 239 240 v_l1_size = 1 << v_l1_bits; 241 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits; 242 v_l2_levels = v_l1_shift / V_L2_BITS - 1; 243 244 assert(v_l1_bits <= V_L1_MAX_BITS); 245 assert(v_l1_shift % V_L2_BITS == 0); 246 assert(v_l2_levels >= 0); 247 } 248 249 static void cpu_gen_init(void) 250 { 251 tcg_context_init(&tcg_init_ctx); 252 } 253 254 /* Encode VAL as a signed leb128 sequence at P. 255 Return P incremented past the encoded value. */ 256 static uint8_t *encode_sleb128(uint8_t *p, target_long val) 257 { 258 int more, byte; 259 260 do { 261 byte = val & 0x7f; 262 val >>= 7; 263 more = !((val == 0 && (byte & 0x40) == 0) 264 || (val == -1 && (byte & 0x40) != 0)); 265 if (more) { 266 byte |= 0x80; 267 } 268 *p++ = byte; 269 } while (more); 270 271 return p; 272 } 273 274 /* Decode a signed leb128 sequence at *PP; increment *PP past the 275 decoded value. Return the decoded value. */ 276 static target_long decode_sleb128(const uint8_t **pp) 277 { 278 const uint8_t *p = *pp; 279 target_long val = 0; 280 int byte, shift = 0; 281 282 do { 283 byte = *p++; 284 val |= (target_ulong)(byte & 0x7f) << shift; 285 shift += 7; 286 } while (byte & 0x80); 287 if (shift < TARGET_LONG_BITS && (byte & 0x40)) { 288 val |= -(target_ulong)1 << shift; 289 } 290 291 *pp = p; 292 return val; 293 } 294 295 /* Encode the data collected about the instructions while compiling TB. 296 Place the data at BLOCK, and return the number of bytes consumed. 297 298 The logical table consists of TARGET_INSN_START_WORDS target_ulong's, 299 which come from the target's insn_start data, followed by a uintptr_t 300 which comes from the host pc of the end of the code implementing the insn. 301 302 Each line of the table is encoded as sleb128 deltas from the previous 303 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 304 That is, the first column is seeded with the guest pc, the last column 305 with the host pc, and the middle columns with zeros. */ 306 307 static int encode_search(TranslationBlock *tb, uint8_t *block) 308 { 309 uint8_t *highwater = tcg_ctx->code_gen_highwater; 310 uint8_t *p = block; 311 int i, j, n; 312 313 for (i = 0, n = tb->icount; i < n; ++i) { 314 target_ulong prev; 315 316 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 317 if (i == 0) { 318 prev = (j == 0 ? tb->pc : 0); 319 } else { 320 prev = tcg_ctx->gen_insn_data[i - 1][j]; 321 } 322 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev); 323 } 324 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]); 325 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev); 326 327 /* Test for (pending) buffer overflow. The assumption is that any 328 one row beginning below the high water mark cannot overrun 329 the buffer completely. Thus we can test for overflow after 330 encoding a row without having to check during encoding. */ 331 if (unlikely(p > highwater)) { 332 return -1; 333 } 334 } 335 336 return p - block; 337 } 338 339 /* The cpu state corresponding to 'searched_pc' is restored. 340 * When reset_icount is true, current TB will be interrupted and 341 * icount should be recalculated. 342 */ 343 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 344 uintptr_t searched_pc, bool reset_icount) 345 { 346 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc }; 347 uintptr_t host_pc = (uintptr_t)tb->tc.ptr; 348 CPUArchState *env = cpu->env_ptr; 349 const uint8_t *p = tb->tc.ptr + tb->tc.size; 350 int i, j, num_insns = tb->icount; 351 #ifdef CONFIG_PROFILER 352 TCGProfile *prof = &tcg_ctx->prof; 353 int64_t ti = profile_getclock(); 354 #endif 355 356 searched_pc -= GETPC_ADJ; 357 358 if (searched_pc < host_pc) { 359 return -1; 360 } 361 362 /* Reconstruct the stored insn data while looking for the point at 363 which the end of the insn exceeds the searched_pc. */ 364 for (i = 0; i < num_insns; ++i) { 365 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) { 366 data[j] += decode_sleb128(&p); 367 } 368 host_pc += decode_sleb128(&p); 369 if (host_pc > searched_pc) { 370 goto found; 371 } 372 } 373 return -1; 374 375 found: 376 if (reset_icount && (tb_cflags(tb) & CF_USE_ICOUNT)) { 377 assert(icount_enabled()); 378 /* Reset the cycle counter to the start of the block 379 and shift if to the number of actually executed instructions */ 380 cpu_neg(cpu)->icount_decr.u16.low += num_insns - i; 381 } 382 restore_state_to_opc(env, tb, data); 383 384 #ifdef CONFIG_PROFILER 385 qatomic_set(&prof->restore_time, 386 prof->restore_time + profile_getclock() - ti); 387 qatomic_set(&prof->restore_count, prof->restore_count + 1); 388 #endif 389 return 0; 390 } 391 392 void tb_destroy(TranslationBlock *tb) 393 { 394 qemu_spin_destroy(&tb->jmp_lock); 395 } 396 397 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit) 398 { 399 /* 400 * The host_pc has to be in the rx region of the code buffer. 401 * If it is not we will not be able to resolve it here. 402 * The two cases where host_pc will not be correct are: 403 * 404 * - fault during translation (instruction fetch) 405 * - fault from helper (not using GETPC() macro) 406 * 407 * Either way we need return early as we can't resolve it here. 408 */ 409 if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { 410 TranslationBlock *tb = tcg_tb_lookup(host_pc); 411 if (tb) { 412 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit); 413 return true; 414 } 415 } 416 return false; 417 } 418 419 static void page_init(void) 420 { 421 page_size_init(); 422 page_table_config_init(); 423 424 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY) 425 { 426 #ifdef HAVE_KINFO_GETVMMAP 427 struct kinfo_vmentry *freep; 428 int i, cnt; 429 430 freep = kinfo_getvmmap(getpid(), &cnt); 431 if (freep) { 432 mmap_lock(); 433 for (i = 0; i < cnt; i++) { 434 unsigned long startaddr, endaddr; 435 436 startaddr = freep[i].kve_start; 437 endaddr = freep[i].kve_end; 438 if (h2g_valid(startaddr)) { 439 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 440 441 if (h2g_valid(endaddr)) { 442 endaddr = h2g(endaddr); 443 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 444 } else { 445 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS 446 endaddr = ~0ul; 447 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 448 #endif 449 } 450 } 451 } 452 free(freep); 453 mmap_unlock(); 454 } 455 #else 456 FILE *f; 457 458 last_brk = (unsigned long)sbrk(0); 459 460 f = fopen("/compat/linux/proc/self/maps", "r"); 461 if (f) { 462 mmap_lock(); 463 464 do { 465 unsigned long startaddr, endaddr; 466 int n; 467 468 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr); 469 470 if (n == 2 && h2g_valid(startaddr)) { 471 startaddr = h2g(startaddr) & TARGET_PAGE_MASK; 472 473 if (h2g_valid(endaddr)) { 474 endaddr = h2g(endaddr); 475 } else { 476 endaddr = ~0ul; 477 } 478 page_set_flags(startaddr, endaddr, PAGE_RESERVED); 479 } 480 } while (!feof(f)); 481 482 fclose(f); 483 mmap_unlock(); 484 } 485 #endif 486 } 487 #endif 488 } 489 490 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc) 491 { 492 PageDesc *pd; 493 void **lp; 494 int i; 495 496 /* Level 1. Always allocated. */ 497 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1)); 498 499 /* Level 2..N-1. */ 500 for (i = v_l2_levels; i > 0; i--) { 501 void **p = qatomic_rcu_read(lp); 502 503 if (p == NULL) { 504 void *existing; 505 506 if (!alloc) { 507 return NULL; 508 } 509 p = g_new0(void *, V_L2_SIZE); 510 existing = qatomic_cmpxchg(lp, NULL, p); 511 if (unlikely(existing)) { 512 g_free(p); 513 p = existing; 514 } 515 } 516 517 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1)); 518 } 519 520 pd = qatomic_rcu_read(lp); 521 if (pd == NULL) { 522 void *existing; 523 524 if (!alloc) { 525 return NULL; 526 } 527 pd = g_new0(PageDesc, V_L2_SIZE); 528 #ifndef CONFIG_USER_ONLY 529 { 530 int i; 531 532 for (i = 0; i < V_L2_SIZE; i++) { 533 qemu_spin_init(&pd[i].lock); 534 } 535 } 536 #endif 537 existing = qatomic_cmpxchg(lp, NULL, pd); 538 if (unlikely(existing)) { 539 #ifndef CONFIG_USER_ONLY 540 { 541 int i; 542 543 for (i = 0; i < V_L2_SIZE; i++) { 544 qemu_spin_destroy(&pd[i].lock); 545 } 546 } 547 #endif 548 g_free(pd); 549 pd = existing; 550 } 551 } 552 553 return pd + (index & (V_L2_SIZE - 1)); 554 } 555 556 static inline PageDesc *page_find(tb_page_addr_t index) 557 { 558 return page_find_alloc(index, 0); 559 } 560 561 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, 562 PageDesc **ret_p2, tb_page_addr_t phys2, int alloc); 563 564 /* In user-mode page locks aren't used; mmap_lock is enough */ 565 #ifdef CONFIG_USER_ONLY 566 567 #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock()) 568 569 static inline void page_lock(PageDesc *pd) 570 { } 571 572 static inline void page_unlock(PageDesc *pd) 573 { } 574 575 static inline void page_lock_tb(const TranslationBlock *tb) 576 { } 577 578 static inline void page_unlock_tb(const TranslationBlock *tb) 579 { } 580 581 struct page_collection * 582 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end) 583 { 584 return NULL; 585 } 586 587 void page_collection_unlock(struct page_collection *set) 588 { } 589 #else /* !CONFIG_USER_ONLY */ 590 591 #ifdef CONFIG_DEBUG_TCG 592 593 static __thread GHashTable *ht_pages_locked_debug; 594 595 static void ht_pages_locked_debug_init(void) 596 { 597 if (ht_pages_locked_debug) { 598 return; 599 } 600 ht_pages_locked_debug = g_hash_table_new(NULL, NULL); 601 } 602 603 static bool page_is_locked(const PageDesc *pd) 604 { 605 PageDesc *found; 606 607 ht_pages_locked_debug_init(); 608 found = g_hash_table_lookup(ht_pages_locked_debug, pd); 609 return !!found; 610 } 611 612 static void page_lock__debug(PageDesc *pd) 613 { 614 ht_pages_locked_debug_init(); 615 g_assert(!page_is_locked(pd)); 616 g_hash_table_insert(ht_pages_locked_debug, pd, pd); 617 } 618 619 static void page_unlock__debug(const PageDesc *pd) 620 { 621 bool removed; 622 623 ht_pages_locked_debug_init(); 624 g_assert(page_is_locked(pd)); 625 removed = g_hash_table_remove(ht_pages_locked_debug, pd); 626 g_assert(removed); 627 } 628 629 static void 630 do_assert_page_locked(const PageDesc *pd, const char *file, int line) 631 { 632 if (unlikely(!page_is_locked(pd))) { 633 error_report("assert_page_lock: PageDesc %p not locked @ %s:%d", 634 pd, file, line); 635 abort(); 636 } 637 } 638 639 #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__) 640 641 void assert_no_pages_locked(void) 642 { 643 ht_pages_locked_debug_init(); 644 g_assert(g_hash_table_size(ht_pages_locked_debug) == 0); 645 } 646 647 #else /* !CONFIG_DEBUG_TCG */ 648 649 #define assert_page_locked(pd) 650 651 static inline void page_lock__debug(const PageDesc *pd) 652 { 653 } 654 655 static inline void page_unlock__debug(const PageDesc *pd) 656 { 657 } 658 659 #endif /* CONFIG_DEBUG_TCG */ 660 661 static inline void page_lock(PageDesc *pd) 662 { 663 page_lock__debug(pd); 664 qemu_spin_lock(&pd->lock); 665 } 666 667 static inline void page_unlock(PageDesc *pd) 668 { 669 qemu_spin_unlock(&pd->lock); 670 page_unlock__debug(pd); 671 } 672 673 /* lock the page(s) of a TB in the correct acquisition order */ 674 static inline void page_lock_tb(const TranslationBlock *tb) 675 { 676 page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], 0); 677 } 678 679 static inline void page_unlock_tb(const TranslationBlock *tb) 680 { 681 PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 682 683 page_unlock(p1); 684 if (unlikely(tb->page_addr[1] != -1)) { 685 PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 686 687 if (p2 != p1) { 688 page_unlock(p2); 689 } 690 } 691 } 692 693 static inline struct page_entry * 694 page_entry_new(PageDesc *pd, tb_page_addr_t index) 695 { 696 struct page_entry *pe = g_malloc(sizeof(*pe)); 697 698 pe->index = index; 699 pe->pd = pd; 700 pe->locked = false; 701 return pe; 702 } 703 704 static void page_entry_destroy(gpointer p) 705 { 706 struct page_entry *pe = p; 707 708 g_assert(pe->locked); 709 page_unlock(pe->pd); 710 g_free(pe); 711 } 712 713 /* returns false on success */ 714 static bool page_entry_trylock(struct page_entry *pe) 715 { 716 bool busy; 717 718 busy = qemu_spin_trylock(&pe->pd->lock); 719 if (!busy) { 720 g_assert(!pe->locked); 721 pe->locked = true; 722 page_lock__debug(pe->pd); 723 } 724 return busy; 725 } 726 727 static void do_page_entry_lock(struct page_entry *pe) 728 { 729 page_lock(pe->pd); 730 g_assert(!pe->locked); 731 pe->locked = true; 732 } 733 734 static gboolean page_entry_lock(gpointer key, gpointer value, gpointer data) 735 { 736 struct page_entry *pe = value; 737 738 do_page_entry_lock(pe); 739 return FALSE; 740 } 741 742 static gboolean page_entry_unlock(gpointer key, gpointer value, gpointer data) 743 { 744 struct page_entry *pe = value; 745 746 if (pe->locked) { 747 pe->locked = false; 748 page_unlock(pe->pd); 749 } 750 return FALSE; 751 } 752 753 /* 754 * Trylock a page, and if successful, add the page to a collection. 755 * Returns true ("busy") if the page could not be locked; false otherwise. 756 */ 757 static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr) 758 { 759 tb_page_addr_t index = addr >> TARGET_PAGE_BITS; 760 struct page_entry *pe; 761 PageDesc *pd; 762 763 pe = g_tree_lookup(set->tree, &index); 764 if (pe) { 765 return false; 766 } 767 768 pd = page_find(index); 769 if (pd == NULL) { 770 return false; 771 } 772 773 pe = page_entry_new(pd, index); 774 g_tree_insert(set->tree, &pe->index, pe); 775 776 /* 777 * If this is either (1) the first insertion or (2) a page whose index 778 * is higher than any other so far, just lock the page and move on. 779 */ 780 if (set->max == NULL || pe->index > set->max->index) { 781 set->max = pe; 782 do_page_entry_lock(pe); 783 return false; 784 } 785 /* 786 * Try to acquire out-of-order lock; if busy, return busy so that we acquire 787 * locks in order. 788 */ 789 return page_entry_trylock(pe); 790 } 791 792 static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata) 793 { 794 tb_page_addr_t a = *(const tb_page_addr_t *)ap; 795 tb_page_addr_t b = *(const tb_page_addr_t *)bp; 796 797 if (a == b) { 798 return 0; 799 } else if (a < b) { 800 return -1; 801 } 802 return 1; 803 } 804 805 /* 806 * Lock a range of pages ([@start,@end[) as well as the pages of all 807 * intersecting TBs. 808 * Locking order: acquire locks in ascending order of page index. 809 */ 810 struct page_collection * 811 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end) 812 { 813 struct page_collection *set = g_malloc(sizeof(*set)); 814 tb_page_addr_t index; 815 PageDesc *pd; 816 817 start >>= TARGET_PAGE_BITS; 818 end >>= TARGET_PAGE_BITS; 819 g_assert(start <= end); 820 821 set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL, 822 page_entry_destroy); 823 set->max = NULL; 824 assert_no_pages_locked(); 825 826 retry: 827 g_tree_foreach(set->tree, page_entry_lock, NULL); 828 829 for (index = start; index <= end; index++) { 830 TranslationBlock *tb; 831 int n; 832 833 pd = page_find(index); 834 if (pd == NULL) { 835 continue; 836 } 837 if (page_trylock_add(set, index << TARGET_PAGE_BITS)) { 838 g_tree_foreach(set->tree, page_entry_unlock, NULL); 839 goto retry; 840 } 841 assert_page_locked(pd); 842 PAGE_FOR_EACH_TB(pd, tb, n) { 843 if (page_trylock_add(set, tb->page_addr[0]) || 844 (tb->page_addr[1] != -1 && 845 page_trylock_add(set, tb->page_addr[1]))) { 846 /* drop all locks, and reacquire in order */ 847 g_tree_foreach(set->tree, page_entry_unlock, NULL); 848 goto retry; 849 } 850 } 851 } 852 return set; 853 } 854 855 void page_collection_unlock(struct page_collection *set) 856 { 857 /* entries are unlocked and freed via page_entry_destroy */ 858 g_tree_destroy(set->tree); 859 g_free(set); 860 } 861 862 #endif /* !CONFIG_USER_ONLY */ 863 864 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1, 865 PageDesc **ret_p2, tb_page_addr_t phys2, int alloc) 866 { 867 PageDesc *p1, *p2; 868 tb_page_addr_t page1; 869 tb_page_addr_t page2; 870 871 assert_memory_lock(); 872 g_assert(phys1 != -1); 873 874 page1 = phys1 >> TARGET_PAGE_BITS; 875 page2 = phys2 >> TARGET_PAGE_BITS; 876 877 p1 = page_find_alloc(page1, alloc); 878 if (ret_p1) { 879 *ret_p1 = p1; 880 } 881 if (likely(phys2 == -1)) { 882 page_lock(p1); 883 return; 884 } else if (page1 == page2) { 885 page_lock(p1); 886 if (ret_p2) { 887 *ret_p2 = p1; 888 } 889 return; 890 } 891 p2 = page_find_alloc(page2, alloc); 892 if (ret_p2) { 893 *ret_p2 = p2; 894 } 895 if (page1 < page2) { 896 page_lock(p1); 897 page_lock(p2); 898 } else { 899 page_lock(p2); 900 page_lock(p1); 901 } 902 } 903 904 /* Minimum size of the code gen buffer. This number is randomly chosen, 905 but not so small that we can't have a fair number of TB's live. */ 906 #define MIN_CODE_GEN_BUFFER_SIZE (1 * MiB) 907 908 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise 909 indicated, this is constrained by the range of direct branches on the 910 host cpu, as used by the TCG implementation of goto_tb. */ 911 #if defined(__x86_64__) 912 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) 913 #elif defined(__sparc__) 914 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) 915 #elif defined(__powerpc64__) 916 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) 917 #elif defined(__powerpc__) 918 # define MAX_CODE_GEN_BUFFER_SIZE (32 * MiB) 919 #elif defined(__aarch64__) 920 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) 921 #elif defined(__s390x__) 922 /* We have a +- 4GB range on the branches; leave some slop. */ 923 # define MAX_CODE_GEN_BUFFER_SIZE (3 * GiB) 924 #elif defined(__mips__) 925 /* We have a 256MB branch region, but leave room to make sure the 926 main executable is also within that region. */ 927 # define MAX_CODE_GEN_BUFFER_SIZE (128 * MiB) 928 #else 929 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1) 930 #endif 931 932 #if TCG_TARGET_REG_BITS == 32 933 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32 * MiB) 934 #ifdef CONFIG_USER_ONLY 935 /* 936 * For user mode on smaller 32 bit systems we may run into trouble 937 * allocating big chunks of data in the right place. On these systems 938 * we utilise a static code generation buffer directly in the binary. 939 */ 940 #define USE_STATIC_CODE_GEN_BUFFER 941 #endif 942 #else /* TCG_TARGET_REG_BITS == 64 */ 943 #ifdef CONFIG_USER_ONLY 944 /* 945 * As user-mode emulation typically means running multiple instances 946 * of the translator don't go too nuts with our default code gen 947 * buffer lest we make things too hard for the OS. 948 */ 949 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (128 * MiB) 950 #else 951 /* 952 * We expect most system emulation to run one or two guests per host. 953 * Users running large scale system emulation may want to tweak their 954 * runtime setup via the tb-size control on the command line. 955 */ 956 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (1 * GiB) 957 #endif 958 #endif 959 960 #define DEFAULT_CODE_GEN_BUFFER_SIZE \ 961 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \ 962 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE) 963 964 static size_t size_code_gen_buffer(size_t tb_size) 965 { 966 /* Size the buffer. */ 967 if (tb_size == 0) { 968 size_t phys_mem = qemu_get_host_physmem(); 969 if (phys_mem == 0) { 970 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE; 971 } else { 972 tb_size = MIN(DEFAULT_CODE_GEN_BUFFER_SIZE, phys_mem / 8); 973 } 974 } 975 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) { 976 tb_size = MIN_CODE_GEN_BUFFER_SIZE; 977 } 978 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) { 979 tb_size = MAX_CODE_GEN_BUFFER_SIZE; 980 } 981 return tb_size; 982 } 983 984 #ifdef __mips__ 985 /* In order to use J and JAL within the code_gen_buffer, we require 986 that the buffer not cross a 256MB boundary. */ 987 static inline bool cross_256mb(void *addr, size_t size) 988 { 989 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful; 990 } 991 992 /* We weren't able to allocate a buffer without crossing that boundary, 993 so make do with the larger portion of the buffer that doesn't cross. 994 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */ 995 static inline void *split_cross_256mb(void *buf1, size_t size1) 996 { 997 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful); 998 size_t size2 = buf1 + size1 - buf2; 999 1000 size1 = buf2 - buf1; 1001 if (size1 < size2) { 1002 size1 = size2; 1003 buf1 = buf2; 1004 } 1005 1006 tcg_ctx->code_gen_buffer_size = size1; 1007 return buf1; 1008 } 1009 #endif 1010 1011 #ifdef USE_STATIC_CODE_GEN_BUFFER 1012 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE] 1013 __attribute__((aligned(CODE_GEN_ALIGN))); 1014 1015 static bool alloc_code_gen_buffer(size_t tb_size, int splitwx, Error **errp) 1016 { 1017 void *buf, *end; 1018 size_t size; 1019 1020 if (splitwx > 0) { 1021 error_setg(errp, "jit split-wx not supported"); 1022 return false; 1023 } 1024 1025 /* page-align the beginning and end of the buffer */ 1026 buf = static_code_gen_buffer; 1027 end = static_code_gen_buffer + sizeof(static_code_gen_buffer); 1028 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size); 1029 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size); 1030 1031 size = end - buf; 1032 1033 /* Honor a command-line option limiting the size of the buffer. */ 1034 if (size > tb_size) { 1035 size = QEMU_ALIGN_DOWN(tb_size, qemu_real_host_page_size); 1036 } 1037 tcg_ctx->code_gen_buffer_size = size; 1038 1039 #ifdef __mips__ 1040 if (cross_256mb(buf, size)) { 1041 buf = split_cross_256mb(buf, size); 1042 size = tcg_ctx->code_gen_buffer_size; 1043 } 1044 #endif 1045 1046 if (qemu_mprotect_rwx(buf, size)) { 1047 error_setg_errno(errp, errno, "mprotect of jit buffer"); 1048 return false; 1049 } 1050 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 1051 1052 tcg_ctx->code_gen_buffer = buf; 1053 return true; 1054 } 1055 #elif defined(_WIN32) 1056 static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) 1057 { 1058 void *buf; 1059 1060 if (splitwx > 0) { 1061 error_setg(errp, "jit split-wx not supported"); 1062 return false; 1063 } 1064 1065 buf = VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, 1066 PAGE_EXECUTE_READWRITE); 1067 if (buf == NULL) { 1068 error_setg_win32(errp, GetLastError(), 1069 "allocate %zu bytes for jit buffer", size); 1070 return false; 1071 } 1072 1073 tcg_ctx->code_gen_buffer = buf; 1074 tcg_ctx->code_gen_buffer_size = size; 1075 return true; 1076 } 1077 #else 1078 static bool alloc_code_gen_buffer_anon(size_t size, int prot, 1079 int flags, Error **errp) 1080 { 1081 void *buf; 1082 1083 buf = mmap(NULL, size, prot, flags, -1, 0); 1084 if (buf == MAP_FAILED) { 1085 error_setg_errno(errp, errno, 1086 "allocate %zu bytes for jit buffer", size); 1087 return false; 1088 } 1089 tcg_ctx->code_gen_buffer_size = size; 1090 1091 #ifdef __mips__ 1092 if (cross_256mb(buf, size)) { 1093 /* 1094 * Try again, with the original still mapped, to avoid re-acquiring 1095 * the same 256mb crossing. 1096 */ 1097 size_t size2; 1098 void *buf2 = mmap(NULL, size, prot, flags, -1, 0); 1099 switch ((int)(buf2 != MAP_FAILED)) { 1100 case 1: 1101 if (!cross_256mb(buf2, size)) { 1102 /* Success! Use the new buffer. */ 1103 munmap(buf, size); 1104 break; 1105 } 1106 /* Failure. Work with what we had. */ 1107 munmap(buf2, size); 1108 /* fallthru */ 1109 default: 1110 /* Split the original buffer. Free the smaller half. */ 1111 buf2 = split_cross_256mb(buf, size); 1112 size2 = tcg_ctx->code_gen_buffer_size; 1113 if (buf == buf2) { 1114 munmap(buf + size2, size - size2); 1115 } else { 1116 munmap(buf, size - size2); 1117 } 1118 size = size2; 1119 break; 1120 } 1121 buf = buf2; 1122 } 1123 #endif 1124 1125 /* Request large pages for the buffer. */ 1126 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE); 1127 1128 tcg_ctx->code_gen_buffer = buf; 1129 return true; 1130 } 1131 1132 #ifndef CONFIG_TCG_INTERPRETER 1133 #ifdef CONFIG_POSIX 1134 #include "qemu/memfd.h" 1135 1136 static bool alloc_code_gen_buffer_splitwx_memfd(size_t size, Error **errp) 1137 { 1138 void *buf_rw = NULL, *buf_rx = MAP_FAILED; 1139 int fd = -1; 1140 1141 #ifdef __mips__ 1142 /* Find space for the RX mapping, vs the 256MiB regions. */ 1143 if (!alloc_code_gen_buffer_anon(size, PROT_NONE, 1144 MAP_PRIVATE | MAP_ANONYMOUS | 1145 MAP_NORESERVE, errp)) { 1146 return false; 1147 } 1148 /* The size of the mapping may have been adjusted. */ 1149 size = tcg_ctx->code_gen_buffer_size; 1150 buf_rx = tcg_ctx->code_gen_buffer; 1151 #endif 1152 1153 buf_rw = qemu_memfd_alloc("tcg-jit", size, 0, &fd, errp); 1154 if (buf_rw == NULL) { 1155 goto fail; 1156 } 1157 1158 #ifdef __mips__ 1159 void *tmp = mmap(buf_rx, size, PROT_READ | PROT_EXEC, 1160 MAP_SHARED | MAP_FIXED, fd, 0); 1161 if (tmp != buf_rx) { 1162 goto fail_rx; 1163 } 1164 #else 1165 buf_rx = mmap(NULL, size, PROT_READ | PROT_EXEC, MAP_SHARED, fd, 0); 1166 if (buf_rx == MAP_FAILED) { 1167 goto fail_rx; 1168 } 1169 #endif 1170 1171 close(fd); 1172 tcg_ctx->code_gen_buffer = buf_rw; 1173 tcg_ctx->code_gen_buffer_size = size; 1174 tcg_splitwx_diff = buf_rx - buf_rw; 1175 1176 /* Request large pages for the buffer and the splitwx. */ 1177 qemu_madvise(buf_rw, size, QEMU_MADV_HUGEPAGE); 1178 qemu_madvise(buf_rx, size, QEMU_MADV_HUGEPAGE); 1179 return true; 1180 1181 fail_rx: 1182 error_setg_errno(errp, errno, "failed to map shared memory for execute"); 1183 fail: 1184 if (buf_rx != MAP_FAILED) { 1185 munmap(buf_rx, size); 1186 } 1187 if (buf_rw) { 1188 munmap(buf_rw, size); 1189 } 1190 if (fd >= 0) { 1191 close(fd); 1192 } 1193 return false; 1194 } 1195 #endif /* CONFIG_POSIX */ 1196 1197 #ifdef CONFIG_DARWIN 1198 #include <mach/mach.h> 1199 1200 extern kern_return_t mach_vm_remap(vm_map_t target_task, 1201 mach_vm_address_t *target_address, 1202 mach_vm_size_t size, 1203 mach_vm_offset_t mask, 1204 int flags, 1205 vm_map_t src_task, 1206 mach_vm_address_t src_address, 1207 boolean_t copy, 1208 vm_prot_t *cur_protection, 1209 vm_prot_t *max_protection, 1210 vm_inherit_t inheritance); 1211 1212 static bool alloc_code_gen_buffer_splitwx_vmremap(size_t size, Error **errp) 1213 { 1214 kern_return_t ret; 1215 mach_vm_address_t buf_rw, buf_rx; 1216 vm_prot_t cur_prot, max_prot; 1217 1218 /* Map the read-write portion via normal anon memory. */ 1219 if (!alloc_code_gen_buffer_anon(size, PROT_READ | PROT_WRITE, 1220 MAP_PRIVATE | MAP_ANONYMOUS, errp)) { 1221 return false; 1222 } 1223 1224 buf_rw = (mach_vm_address_t)tcg_ctx->code_gen_buffer; 1225 buf_rx = 0; 1226 ret = mach_vm_remap(mach_task_self(), 1227 &buf_rx, 1228 size, 1229 0, 1230 VM_FLAGS_ANYWHERE, 1231 mach_task_self(), 1232 buf_rw, 1233 false, 1234 &cur_prot, 1235 &max_prot, 1236 VM_INHERIT_NONE); 1237 if (ret != KERN_SUCCESS) { 1238 /* TODO: Convert "ret" to a human readable error message. */ 1239 error_setg(errp, "vm_remap for jit splitwx failed"); 1240 munmap((void *)buf_rw, size); 1241 return false; 1242 } 1243 1244 if (mprotect((void *)buf_rx, size, PROT_READ | PROT_EXEC) != 0) { 1245 error_setg_errno(errp, errno, "mprotect for jit splitwx"); 1246 munmap((void *)buf_rx, size); 1247 munmap((void *)buf_rw, size); 1248 return false; 1249 } 1250 1251 tcg_splitwx_diff = buf_rx - buf_rw; 1252 return true; 1253 } 1254 #endif /* CONFIG_DARWIN */ 1255 #endif /* CONFIG_TCG_INTERPRETER */ 1256 1257 static bool alloc_code_gen_buffer_splitwx(size_t size, Error **errp) 1258 { 1259 #ifndef CONFIG_TCG_INTERPRETER 1260 # ifdef CONFIG_DARWIN 1261 return alloc_code_gen_buffer_splitwx_vmremap(size, errp); 1262 # endif 1263 # ifdef CONFIG_POSIX 1264 return alloc_code_gen_buffer_splitwx_memfd(size, errp); 1265 # endif 1266 #endif 1267 error_setg(errp, "jit split-wx not supported"); 1268 return false; 1269 } 1270 1271 static bool alloc_code_gen_buffer(size_t size, int splitwx, Error **errp) 1272 { 1273 ERRP_GUARD(); 1274 int prot, flags; 1275 1276 if (splitwx) { 1277 if (alloc_code_gen_buffer_splitwx(size, errp)) { 1278 return true; 1279 } 1280 /* 1281 * If splitwx force-on (1), fail; 1282 * if splitwx default-on (-1), fall through to splitwx off. 1283 */ 1284 if (splitwx > 0) { 1285 return false; 1286 } 1287 error_free_or_abort(errp); 1288 } 1289 1290 prot = PROT_READ | PROT_WRITE | PROT_EXEC; 1291 flags = MAP_PRIVATE | MAP_ANONYMOUS; 1292 #ifdef CONFIG_TCG_INTERPRETER 1293 /* The tcg interpreter does not need execute permission. */ 1294 prot = PROT_READ | PROT_WRITE; 1295 #elif defined(CONFIG_DARWIN) 1296 /* Applicable to both iOS and macOS (Apple Silicon). */ 1297 if (!splitwx) { 1298 flags |= MAP_JIT; 1299 } 1300 #endif 1301 1302 return alloc_code_gen_buffer_anon(size, prot, flags, errp); 1303 } 1304 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */ 1305 1306 static bool tb_cmp(const void *ap, const void *bp) 1307 { 1308 const TranslationBlock *a = ap; 1309 const TranslationBlock *b = bp; 1310 1311 return a->pc == b->pc && 1312 a->cs_base == b->cs_base && 1313 a->flags == b->flags && 1314 (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) && 1315 a->trace_vcpu_dstate == b->trace_vcpu_dstate && 1316 a->page_addr[0] == b->page_addr[0] && 1317 a->page_addr[1] == b->page_addr[1]; 1318 } 1319 1320 static void tb_htable_init(void) 1321 { 1322 unsigned int mode = QHT_MODE_AUTO_RESIZE; 1323 1324 qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode); 1325 } 1326 1327 /* Must be called before using the QEMU cpus. 'tb_size' is the size 1328 (in bytes) allocated to the translation buffer. Zero means default 1329 size. */ 1330 void tcg_exec_init(unsigned long tb_size, int splitwx) 1331 { 1332 bool ok; 1333 1334 tcg_allowed = true; 1335 cpu_gen_init(); 1336 page_init(); 1337 tb_htable_init(); 1338 1339 ok = alloc_code_gen_buffer(size_code_gen_buffer(tb_size), 1340 splitwx, &error_fatal); 1341 assert(ok); 1342 1343 #if defined(CONFIG_SOFTMMU) 1344 /* There's no guest base to take into account, so go ahead and 1345 initialize the prologue now. */ 1346 tcg_prologue_init(tcg_ctx); 1347 #endif 1348 } 1349 1350 /* call with @p->lock held */ 1351 static inline void invalidate_page_bitmap(PageDesc *p) 1352 { 1353 assert_page_locked(p); 1354 #ifdef CONFIG_SOFTMMU 1355 g_free(p->code_bitmap); 1356 p->code_bitmap = NULL; 1357 p->code_write_count = 0; 1358 #endif 1359 } 1360 1361 /* Set to NULL all the 'first_tb' fields in all PageDescs. */ 1362 static void page_flush_tb_1(int level, void **lp) 1363 { 1364 int i; 1365 1366 if (*lp == NULL) { 1367 return; 1368 } 1369 if (level == 0) { 1370 PageDesc *pd = *lp; 1371 1372 for (i = 0; i < V_L2_SIZE; ++i) { 1373 page_lock(&pd[i]); 1374 pd[i].first_tb = (uintptr_t)NULL; 1375 invalidate_page_bitmap(pd + i); 1376 page_unlock(&pd[i]); 1377 } 1378 } else { 1379 void **pp = *lp; 1380 1381 for (i = 0; i < V_L2_SIZE; ++i) { 1382 page_flush_tb_1(level - 1, pp + i); 1383 } 1384 } 1385 } 1386 1387 static void page_flush_tb(void) 1388 { 1389 int i, l1_sz = v_l1_size; 1390 1391 for (i = 0; i < l1_sz; i++) { 1392 page_flush_tb_1(v_l2_levels, l1_map + i); 1393 } 1394 } 1395 1396 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data) 1397 { 1398 const TranslationBlock *tb = value; 1399 size_t *size = data; 1400 1401 *size += tb->tc.size; 1402 return false; 1403 } 1404 1405 /* flush all the translation blocks */ 1406 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count) 1407 { 1408 bool did_flush = false; 1409 1410 mmap_lock(); 1411 /* If it is already been done on request of another CPU, 1412 * just retry. 1413 */ 1414 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) { 1415 goto done; 1416 } 1417 did_flush = true; 1418 1419 if (DEBUG_TB_FLUSH_GATE) { 1420 size_t nb_tbs = tcg_nb_tbs(); 1421 size_t host_size = 0; 1422 1423 tcg_tb_foreach(tb_host_size_iter, &host_size); 1424 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n", 1425 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0); 1426 } 1427 1428 CPU_FOREACH(cpu) { 1429 cpu_tb_jmp_cache_clear(cpu); 1430 } 1431 1432 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE); 1433 page_flush_tb(); 1434 1435 tcg_region_reset_all(); 1436 /* XXX: flush processor icache at this point if cache flush is 1437 expensive */ 1438 qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1); 1439 1440 done: 1441 mmap_unlock(); 1442 if (did_flush) { 1443 qemu_plugin_flush_cb(); 1444 } 1445 } 1446 1447 void tb_flush(CPUState *cpu) 1448 { 1449 if (tcg_enabled()) { 1450 unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count); 1451 1452 if (cpu_in_exclusive_context(cpu)) { 1453 do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count)); 1454 } else { 1455 async_safe_run_on_cpu(cpu, do_tb_flush, 1456 RUN_ON_CPU_HOST_INT(tb_flush_count)); 1457 } 1458 } 1459 } 1460 1461 /* 1462 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only, 1463 * so in order to prevent bit rot we compile them unconditionally in user-mode, 1464 * and let the optimizer get rid of them by wrapping their user-only callers 1465 * with if (DEBUG_TB_CHECK_GATE). 1466 */ 1467 #ifdef CONFIG_USER_ONLY 1468 1469 static void do_tb_invalidate_check(void *p, uint32_t hash, void *userp) 1470 { 1471 TranslationBlock *tb = p; 1472 target_ulong addr = *(target_ulong *)userp; 1473 1474 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) { 1475 printf("ERROR invalidate: address=" TARGET_FMT_lx 1476 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size); 1477 } 1478 } 1479 1480 /* verify that all the pages have correct rights for code 1481 * 1482 * Called with mmap_lock held. 1483 */ 1484 static void tb_invalidate_check(target_ulong address) 1485 { 1486 address &= TARGET_PAGE_MASK; 1487 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address); 1488 } 1489 1490 static void do_tb_page_check(void *p, uint32_t hash, void *userp) 1491 { 1492 TranslationBlock *tb = p; 1493 int flags1, flags2; 1494 1495 flags1 = page_get_flags(tb->pc); 1496 flags2 = page_get_flags(tb->pc + tb->size - 1); 1497 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) { 1498 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n", 1499 (long)tb->pc, tb->size, flags1, flags2); 1500 } 1501 } 1502 1503 /* verify that all the pages have correct rights for code */ 1504 static void tb_page_check(void) 1505 { 1506 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL); 1507 } 1508 1509 #endif /* CONFIG_USER_ONLY */ 1510 1511 /* 1512 * user-mode: call with mmap_lock held 1513 * !user-mode: call with @pd->lock held 1514 */ 1515 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb) 1516 { 1517 TranslationBlock *tb1; 1518 uintptr_t *pprev; 1519 unsigned int n1; 1520 1521 assert_page_locked(pd); 1522 pprev = &pd->first_tb; 1523 PAGE_FOR_EACH_TB(pd, tb1, n1) { 1524 if (tb1 == tb) { 1525 *pprev = tb1->page_next[n1]; 1526 return; 1527 } 1528 pprev = &tb1->page_next[n1]; 1529 } 1530 g_assert_not_reached(); 1531 } 1532 1533 /* remove @orig from its @n_orig-th jump list */ 1534 static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig) 1535 { 1536 uintptr_t ptr, ptr_locked; 1537 TranslationBlock *dest; 1538 TranslationBlock *tb; 1539 uintptr_t *pprev; 1540 int n; 1541 1542 /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */ 1543 ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1); 1544 dest = (TranslationBlock *)(ptr & ~1); 1545 if (dest == NULL) { 1546 return; 1547 } 1548 1549 qemu_spin_lock(&dest->jmp_lock); 1550 /* 1551 * While acquiring the lock, the jump might have been removed if the 1552 * destination TB was invalidated; check again. 1553 */ 1554 ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]); 1555 if (ptr_locked != ptr) { 1556 qemu_spin_unlock(&dest->jmp_lock); 1557 /* 1558 * The only possibility is that the jump was unlinked via 1559 * tb_jump_unlink(dest). Seeing here another destination would be a bug, 1560 * because we set the LSB above. 1561 */ 1562 g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID); 1563 return; 1564 } 1565 /* 1566 * We first acquired the lock, and since the destination pointer matches, 1567 * we know for sure that @orig is in the jmp list. 1568 */ 1569 pprev = &dest->jmp_list_head; 1570 TB_FOR_EACH_JMP(dest, tb, n) { 1571 if (tb == orig && n == n_orig) { 1572 *pprev = tb->jmp_list_next[n]; 1573 /* no need to set orig->jmp_dest[n]; setting the LSB was enough */ 1574 qemu_spin_unlock(&dest->jmp_lock); 1575 return; 1576 } 1577 pprev = &tb->jmp_list_next[n]; 1578 } 1579 g_assert_not_reached(); 1580 } 1581 1582 /* reset the jump entry 'n' of a TB so that it is not chained to 1583 another TB */ 1584 static inline void tb_reset_jump(TranslationBlock *tb, int n) 1585 { 1586 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]); 1587 tb_set_jmp_target(tb, n, addr); 1588 } 1589 1590 /* remove any jumps to the TB */ 1591 static inline void tb_jmp_unlink(TranslationBlock *dest) 1592 { 1593 TranslationBlock *tb; 1594 int n; 1595 1596 qemu_spin_lock(&dest->jmp_lock); 1597 1598 TB_FOR_EACH_JMP(dest, tb, n) { 1599 tb_reset_jump(tb, n); 1600 qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1); 1601 /* No need to clear the list entry; setting the dest ptr is enough */ 1602 } 1603 dest->jmp_list_head = (uintptr_t)NULL; 1604 1605 qemu_spin_unlock(&dest->jmp_lock); 1606 } 1607 1608 /* 1609 * In user-mode, call with mmap_lock held. 1610 * In !user-mode, if @rm_from_page_list is set, call with the TB's pages' 1611 * locks held. 1612 */ 1613 static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list) 1614 { 1615 CPUState *cpu; 1616 PageDesc *p; 1617 uint32_t h; 1618 tb_page_addr_t phys_pc; 1619 1620 assert_memory_lock(); 1621 1622 /* make sure no further incoming jumps will be chained to this TB */ 1623 qemu_spin_lock(&tb->jmp_lock); 1624 qatomic_set(&tb->cflags, tb->cflags | CF_INVALID); 1625 qemu_spin_unlock(&tb->jmp_lock); 1626 1627 /* remove the TB from the hash list */ 1628 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 1629 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb_cflags(tb) & CF_HASH_MASK, 1630 tb->trace_vcpu_dstate); 1631 if (!qht_remove(&tb_ctx.htable, tb, h)) { 1632 return; 1633 } 1634 1635 /* remove the TB from the page list */ 1636 if (rm_from_page_list) { 1637 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS); 1638 tb_page_remove(p, tb); 1639 invalidate_page_bitmap(p); 1640 if (tb->page_addr[1] != -1) { 1641 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS); 1642 tb_page_remove(p, tb); 1643 invalidate_page_bitmap(p); 1644 } 1645 } 1646 1647 /* remove the TB from the hash list */ 1648 h = tb_jmp_cache_hash_func(tb->pc); 1649 CPU_FOREACH(cpu) { 1650 if (qatomic_read(&cpu->tb_jmp_cache[h]) == tb) { 1651 qatomic_set(&cpu->tb_jmp_cache[h], NULL); 1652 } 1653 } 1654 1655 /* suppress this TB from the two jump lists */ 1656 tb_remove_from_jmp_list(tb, 0); 1657 tb_remove_from_jmp_list(tb, 1); 1658 1659 /* suppress any remaining jumps to this TB */ 1660 tb_jmp_unlink(tb); 1661 1662 qatomic_set(&tcg_ctx->tb_phys_invalidate_count, 1663 tcg_ctx->tb_phys_invalidate_count + 1); 1664 } 1665 1666 static void tb_phys_invalidate__locked(TranslationBlock *tb) 1667 { 1668 qemu_thread_jit_write(); 1669 do_tb_phys_invalidate(tb, true); 1670 qemu_thread_jit_execute(); 1671 } 1672 1673 /* invalidate one TB 1674 * 1675 * Called with mmap_lock held in user-mode. 1676 */ 1677 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr) 1678 { 1679 if (page_addr == -1 && tb->page_addr[0] != -1) { 1680 page_lock_tb(tb); 1681 do_tb_phys_invalidate(tb, true); 1682 page_unlock_tb(tb); 1683 } else { 1684 do_tb_phys_invalidate(tb, false); 1685 } 1686 } 1687 1688 #ifdef CONFIG_SOFTMMU 1689 /* call with @p->lock held */ 1690 static void build_page_bitmap(PageDesc *p) 1691 { 1692 int n, tb_start, tb_end; 1693 TranslationBlock *tb; 1694 1695 assert_page_locked(p); 1696 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE); 1697 1698 PAGE_FOR_EACH_TB(p, tb, n) { 1699 /* NOTE: this is subtle as a TB may span two physical pages */ 1700 if (n == 0) { 1701 /* NOTE: tb_end may be after the end of the page, but 1702 it is not a problem */ 1703 tb_start = tb->pc & ~TARGET_PAGE_MASK; 1704 tb_end = tb_start + tb->size; 1705 if (tb_end > TARGET_PAGE_SIZE) { 1706 tb_end = TARGET_PAGE_SIZE; 1707 } 1708 } else { 1709 tb_start = 0; 1710 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 1711 } 1712 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start); 1713 } 1714 } 1715 #endif 1716 1717 /* add the tb in the target page and protect it if necessary 1718 * 1719 * Called with mmap_lock held for user-mode emulation. 1720 * Called with @p->lock held in !user-mode. 1721 */ 1722 static inline void tb_page_add(PageDesc *p, TranslationBlock *tb, 1723 unsigned int n, tb_page_addr_t page_addr) 1724 { 1725 #ifndef CONFIG_USER_ONLY 1726 bool page_already_protected; 1727 #endif 1728 1729 assert_page_locked(p); 1730 1731 tb->page_addr[n] = page_addr; 1732 tb->page_next[n] = p->first_tb; 1733 #ifndef CONFIG_USER_ONLY 1734 page_already_protected = p->first_tb != (uintptr_t)NULL; 1735 #endif 1736 p->first_tb = (uintptr_t)tb | n; 1737 invalidate_page_bitmap(p); 1738 1739 #if defined(CONFIG_USER_ONLY) 1740 if (p->flags & PAGE_WRITE) { 1741 target_ulong addr; 1742 PageDesc *p2; 1743 int prot; 1744 1745 /* force the host page as non writable (writes will have a 1746 page fault + mprotect overhead) */ 1747 page_addr &= qemu_host_page_mask; 1748 prot = 0; 1749 for (addr = page_addr; addr < page_addr + qemu_host_page_size; 1750 addr += TARGET_PAGE_SIZE) { 1751 1752 p2 = page_find(addr >> TARGET_PAGE_BITS); 1753 if (!p2) { 1754 continue; 1755 } 1756 prot |= p2->flags; 1757 p2->flags &= ~PAGE_WRITE; 1758 } 1759 mprotect(g2h_untagged(page_addr), qemu_host_page_size, 1760 (prot & PAGE_BITS) & ~PAGE_WRITE); 1761 if (DEBUG_TB_INVALIDATE_GATE) { 1762 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr); 1763 } 1764 } 1765 #else 1766 /* if some code is already present, then the pages are already 1767 protected. So we handle the case where only the first TB is 1768 allocated in a physical page */ 1769 if (!page_already_protected) { 1770 tlb_protect_code(page_addr); 1771 } 1772 #endif 1773 } 1774 1775 /* 1776 * Add a new TB and link it to the physical page tables. phys_page2 is 1777 * (-1) to indicate that only one page contains the TB. 1778 * 1779 * Called with mmap_lock held for user-mode emulation. 1780 * 1781 * Returns a pointer @tb, or a pointer to an existing TB that matches @tb. 1782 * Note that in !user-mode, another thread might have already added a TB 1783 * for the same block of guest code that @tb corresponds to. In that case, 1784 * the caller should discard the original @tb, and use instead the returned TB. 1785 */ 1786 static TranslationBlock * 1787 tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc, 1788 tb_page_addr_t phys_page2) 1789 { 1790 PageDesc *p; 1791 PageDesc *p2 = NULL; 1792 void *existing_tb = NULL; 1793 uint32_t h; 1794 1795 assert_memory_lock(); 1796 1797 /* 1798 * Add the TB to the page list, acquiring first the pages's locks. 1799 * We keep the locks held until after inserting the TB in the hash table, 1800 * so that if the insertion fails we know for sure that the TBs are still 1801 * in the page descriptors. 1802 * Note that inserting into the hash table first isn't an option, since 1803 * we can only insert TBs that are fully initialized. 1804 */ 1805 page_lock_pair(&p, phys_pc, &p2, phys_page2, 1); 1806 tb_page_add(p, tb, 0, phys_pc & TARGET_PAGE_MASK); 1807 if (p2) { 1808 tb_page_add(p2, tb, 1, phys_page2); 1809 } else { 1810 tb->page_addr[1] = -1; 1811 } 1812 1813 /* add in the hash table */ 1814 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK, 1815 tb->trace_vcpu_dstate); 1816 qht_insert(&tb_ctx.htable, tb, h, &existing_tb); 1817 1818 /* remove TB from the page(s) if we couldn't insert it */ 1819 if (unlikely(existing_tb)) { 1820 tb_page_remove(p, tb); 1821 invalidate_page_bitmap(p); 1822 if (p2) { 1823 tb_page_remove(p2, tb); 1824 invalidate_page_bitmap(p2); 1825 } 1826 tb = existing_tb; 1827 } 1828 1829 if (p2 && p2 != p) { 1830 page_unlock(p2); 1831 } 1832 page_unlock(p); 1833 1834 #ifdef CONFIG_USER_ONLY 1835 if (DEBUG_TB_CHECK_GATE) { 1836 tb_page_check(); 1837 } 1838 #endif 1839 return tb; 1840 } 1841 1842 /* Called with mmap_lock held for user mode emulation. */ 1843 TranslationBlock *tb_gen_code(CPUState *cpu, 1844 target_ulong pc, target_ulong cs_base, 1845 uint32_t flags, int cflags) 1846 { 1847 CPUArchState *env = cpu->env_ptr; 1848 TranslationBlock *tb, *existing_tb; 1849 tb_page_addr_t phys_pc, phys_page2; 1850 target_ulong virt_page2; 1851 tcg_insn_unit *gen_code_buf; 1852 int gen_code_size, search_size, max_insns; 1853 #ifdef CONFIG_PROFILER 1854 TCGProfile *prof = &tcg_ctx->prof; 1855 int64_t ti; 1856 #endif 1857 1858 assert_memory_lock(); 1859 qemu_thread_jit_write(); 1860 1861 phys_pc = get_page_addr_code(env, pc); 1862 1863 if (phys_pc == -1) { 1864 /* Generate a one-shot TB with 1 insn in it */ 1865 cflags = (cflags & ~CF_COUNT_MASK) | 1; 1866 } 1867 1868 cflags &= ~CF_CLUSTER_MASK; 1869 cflags |= cpu->cluster_index << CF_CLUSTER_SHIFT; 1870 1871 max_insns = cflags & CF_COUNT_MASK; 1872 if (max_insns == 0) { 1873 max_insns = CF_COUNT_MASK; 1874 } 1875 if (max_insns > TCG_MAX_INSNS) { 1876 max_insns = TCG_MAX_INSNS; 1877 } 1878 if (cpu->singlestep_enabled || singlestep) { 1879 max_insns = 1; 1880 } 1881 1882 buffer_overflow: 1883 tb = tcg_tb_alloc(tcg_ctx); 1884 if (unlikely(!tb)) { 1885 /* flush must be done */ 1886 tb_flush(cpu); 1887 mmap_unlock(); 1888 /* Make the execution loop process the flush as soon as possible. */ 1889 cpu->exception_index = EXCP_INTERRUPT; 1890 cpu_loop_exit(cpu); 1891 } 1892 1893 gen_code_buf = tcg_ctx->code_gen_ptr; 1894 tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf); 1895 tb->pc = pc; 1896 tb->cs_base = cs_base; 1897 tb->flags = flags; 1898 tb->cflags = cflags; 1899 tb->trace_vcpu_dstate = *cpu->trace_dstate; 1900 tcg_ctx->tb_cflags = cflags; 1901 tb_overflow: 1902 1903 #ifdef CONFIG_PROFILER 1904 /* includes aborted translations because of exceptions */ 1905 qatomic_set(&prof->tb_count1, prof->tb_count1 + 1); 1906 ti = profile_getclock(); 1907 #endif 1908 1909 gen_code_size = sigsetjmp(tcg_ctx->jmp_trans, 0); 1910 if (unlikely(gen_code_size != 0)) { 1911 goto error_return; 1912 } 1913 1914 tcg_func_start(tcg_ctx); 1915 1916 tcg_ctx->cpu = env_cpu(env); 1917 gen_intermediate_code(cpu, tb, max_insns); 1918 tcg_ctx->cpu = NULL; 1919 max_insns = tb->icount; 1920 1921 trace_translate_block(tb, tb->pc, tb->tc.ptr); 1922 1923 /* generate machine code */ 1924 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID; 1925 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID; 1926 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset; 1927 if (TCG_TARGET_HAS_direct_jump) { 1928 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg; 1929 tcg_ctx->tb_jmp_target_addr = NULL; 1930 } else { 1931 tcg_ctx->tb_jmp_insn_offset = NULL; 1932 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg; 1933 } 1934 1935 #ifdef CONFIG_PROFILER 1936 qatomic_set(&prof->tb_count, prof->tb_count + 1); 1937 qatomic_set(&prof->interm_time, 1938 prof->interm_time + profile_getclock() - ti); 1939 ti = profile_getclock(); 1940 #endif 1941 1942 gen_code_size = tcg_gen_code(tcg_ctx, tb); 1943 if (unlikely(gen_code_size < 0)) { 1944 error_return: 1945 switch (gen_code_size) { 1946 case -1: 1947 /* 1948 * Overflow of code_gen_buffer, or the current slice of it. 1949 * 1950 * TODO: We don't need to re-do gen_intermediate_code, nor 1951 * should we re-do the tcg optimization currently hidden 1952 * inside tcg_gen_code. All that should be required is to 1953 * flush the TBs, allocate a new TB, re-initialize it per 1954 * above, and re-do the actual code generation. 1955 */ 1956 qemu_log_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT, 1957 "Restarting code generation for " 1958 "code_gen_buffer overflow\n"); 1959 goto buffer_overflow; 1960 1961 case -2: 1962 /* 1963 * The code generated for the TranslationBlock is too large. 1964 * The maximum size allowed by the unwind info is 64k. 1965 * There may be stricter constraints from relocations 1966 * in the tcg backend. 1967 * 1968 * Try again with half as many insns as we attempted this time. 1969 * If a single insn overflows, there's a bug somewhere... 1970 */ 1971 assert(max_insns > 1); 1972 max_insns /= 2; 1973 qemu_log_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT, 1974 "Restarting code generation with " 1975 "smaller translation block (max %d insns)\n", 1976 max_insns); 1977 goto tb_overflow; 1978 1979 default: 1980 g_assert_not_reached(); 1981 } 1982 } 1983 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 1984 if (unlikely(search_size < 0)) { 1985 goto buffer_overflow; 1986 } 1987 tb->tc.size = gen_code_size; 1988 1989 #ifdef CONFIG_PROFILER 1990 qatomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti); 1991 qatomic_set(&prof->code_in_len, prof->code_in_len + tb->size); 1992 qatomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size); 1993 qatomic_set(&prof->search_out_len, prof->search_out_len + search_size); 1994 #endif 1995 1996 #ifdef DEBUG_DISAS 1997 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 1998 qemu_log_in_addr_range(tb->pc)) { 1999 FILE *logfile = qemu_log_lock(); 2000 int code_size, data_size; 2001 const tcg_target_ulong *rx_data_gen_ptr; 2002 size_t chunk_start; 2003 int insn = 0; 2004 2005 if (tcg_ctx->data_gen_ptr) { 2006 rx_data_gen_ptr = tcg_splitwx_to_rx(tcg_ctx->data_gen_ptr); 2007 code_size = (const void *)rx_data_gen_ptr - tb->tc.ptr; 2008 data_size = gen_code_size - code_size; 2009 } else { 2010 rx_data_gen_ptr = 0; 2011 code_size = gen_code_size; 2012 data_size = 0; 2013 } 2014 2015 /* Dump header and the first instruction */ 2016 qemu_log("OUT: [size=%d]\n", gen_code_size); 2017 qemu_log(" -- guest addr 0x" TARGET_FMT_lx " + tb prologue\n", 2018 tcg_ctx->gen_insn_data[insn][0]); 2019 chunk_start = tcg_ctx->gen_insn_end_off[insn]; 2020 log_disas(tb->tc.ptr, chunk_start); 2021 2022 /* 2023 * Dump each instruction chunk, wrapping up empty chunks into 2024 * the next instruction. The whole array is offset so the 2025 * first entry is the beginning of the 2nd instruction. 2026 */ 2027 while (insn < tb->icount) { 2028 size_t chunk_end = tcg_ctx->gen_insn_end_off[insn]; 2029 if (chunk_end > chunk_start) { 2030 qemu_log(" -- guest addr 0x" TARGET_FMT_lx "\n", 2031 tcg_ctx->gen_insn_data[insn][0]); 2032 log_disas(tb->tc.ptr + chunk_start, chunk_end - chunk_start); 2033 chunk_start = chunk_end; 2034 } 2035 insn++; 2036 } 2037 2038 if (chunk_start < code_size) { 2039 qemu_log(" -- tb slow paths + alignment\n"); 2040 log_disas(tb->tc.ptr + chunk_start, code_size - chunk_start); 2041 } 2042 2043 /* Finally dump any data we may have after the block */ 2044 if (data_size) { 2045 int i; 2046 qemu_log(" data: [size=%d]\n", data_size); 2047 for (i = 0; i < data_size / sizeof(tcg_target_ulong); i++) { 2048 qemu_log("0x%08" PRIxPTR ": .quad 0x%" TCG_PRIlx "\n", 2049 (uintptr_t)&rx_data_gen_ptr[i], rx_data_gen_ptr[i]); 2050 } 2051 } 2052 qemu_log("\n"); 2053 qemu_log_flush(); 2054 qemu_log_unlock(logfile); 2055 } 2056 #endif 2057 2058 qatomic_set(&tcg_ctx->code_gen_ptr, (void *) 2059 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 2060 CODE_GEN_ALIGN)); 2061 2062 /* init jump list */ 2063 qemu_spin_init(&tb->jmp_lock); 2064 tb->jmp_list_head = (uintptr_t)NULL; 2065 tb->jmp_list_next[0] = (uintptr_t)NULL; 2066 tb->jmp_list_next[1] = (uintptr_t)NULL; 2067 tb->jmp_dest[0] = (uintptr_t)NULL; 2068 tb->jmp_dest[1] = (uintptr_t)NULL; 2069 2070 /* init original jump addresses which have been set during tcg_gen_code() */ 2071 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 2072 tb_reset_jump(tb, 0); 2073 } 2074 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 2075 tb_reset_jump(tb, 1); 2076 } 2077 2078 /* 2079 * If the TB is not associated with a physical RAM page then 2080 * it must be a temporary one-insn TB, and we have nothing to do 2081 * except fill in the page_addr[] fields. Return early before 2082 * attempting to link to other TBs or add to the lookup table. 2083 */ 2084 if (phys_pc == -1) { 2085 tb->page_addr[0] = tb->page_addr[1] = -1; 2086 return tb; 2087 } 2088 2089 /* check next page if needed */ 2090 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK; 2091 phys_page2 = -1; 2092 if ((pc & TARGET_PAGE_MASK) != virt_page2) { 2093 phys_page2 = get_page_addr_code(env, virt_page2); 2094 } 2095 /* 2096 * No explicit memory barrier is required -- tb_link_page() makes the 2097 * TB visible in a consistent state. 2098 */ 2099 existing_tb = tb_link_page(tb, phys_pc, phys_page2); 2100 /* if the TB already exists, discard what we just translated */ 2101 if (unlikely(existing_tb != tb)) { 2102 uintptr_t orig_aligned = (uintptr_t)gen_code_buf; 2103 2104 orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize); 2105 qatomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned); 2106 tb_destroy(tb); 2107 return existing_tb; 2108 } 2109 tcg_tb_insert(tb); 2110 return tb; 2111 } 2112 2113 /* 2114 * @p must be non-NULL. 2115 * user-mode: call with mmap_lock held. 2116 * !user-mode: call with all @pages locked. 2117 */ 2118 static void 2119 tb_invalidate_phys_page_range__locked(struct page_collection *pages, 2120 PageDesc *p, tb_page_addr_t start, 2121 tb_page_addr_t end, 2122 uintptr_t retaddr) 2123 { 2124 TranslationBlock *tb; 2125 tb_page_addr_t tb_start, tb_end; 2126 int n; 2127 #ifdef TARGET_HAS_PRECISE_SMC 2128 CPUState *cpu = current_cpu; 2129 CPUArchState *env = NULL; 2130 bool current_tb_not_found = retaddr != 0; 2131 bool current_tb_modified = false; 2132 TranslationBlock *current_tb = NULL; 2133 target_ulong current_pc = 0; 2134 target_ulong current_cs_base = 0; 2135 uint32_t current_flags = 0; 2136 #endif /* TARGET_HAS_PRECISE_SMC */ 2137 2138 assert_page_locked(p); 2139 2140 #if defined(TARGET_HAS_PRECISE_SMC) 2141 if (cpu != NULL) { 2142 env = cpu->env_ptr; 2143 } 2144 #endif 2145 2146 /* we remove all the TBs in the range [start, end[ */ 2147 /* XXX: see if in some cases it could be faster to invalidate all 2148 the code */ 2149 PAGE_FOR_EACH_TB(p, tb, n) { 2150 assert_page_locked(p); 2151 /* NOTE: this is subtle as a TB may span two physical pages */ 2152 if (n == 0) { 2153 /* NOTE: tb_end may be after the end of the page, but 2154 it is not a problem */ 2155 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK); 2156 tb_end = tb_start + tb->size; 2157 } else { 2158 tb_start = tb->page_addr[1]; 2159 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK); 2160 } 2161 if (!(tb_end <= start || tb_start >= end)) { 2162 #ifdef TARGET_HAS_PRECISE_SMC 2163 if (current_tb_not_found) { 2164 current_tb_not_found = false; 2165 /* now we have a real cpu fault */ 2166 current_tb = tcg_tb_lookup(retaddr); 2167 } 2168 if (current_tb == tb && 2169 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { 2170 /* 2171 * If we are modifying the current TB, we must stop 2172 * its execution. We could be more precise by checking 2173 * that the modification is after the current PC, but it 2174 * would require a specialized function to partially 2175 * restore the CPU state. 2176 */ 2177 current_tb_modified = true; 2178 cpu_restore_state_from_tb(cpu, current_tb, retaddr, true); 2179 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 2180 ¤t_flags); 2181 } 2182 #endif /* TARGET_HAS_PRECISE_SMC */ 2183 tb_phys_invalidate__locked(tb); 2184 } 2185 } 2186 #if !defined(CONFIG_USER_ONLY) 2187 /* if no code remaining, no need to continue to use slow writes */ 2188 if (!p->first_tb) { 2189 invalidate_page_bitmap(p); 2190 tlb_unprotect_code(start); 2191 } 2192 #endif 2193 #ifdef TARGET_HAS_PRECISE_SMC 2194 if (current_tb_modified) { 2195 page_collection_unlock(pages); 2196 /* Force execution of one insn next time. */ 2197 cpu->cflags_next_tb = 1 | curr_cflags(); 2198 mmap_unlock(); 2199 cpu_loop_exit_noexc(cpu); 2200 } 2201 #endif 2202 } 2203 2204 /* 2205 * Invalidate all TBs which intersect with the target physical address range 2206 * [start;end[. NOTE: start and end must refer to the *same* physical page. 2207 * 'is_cpu_write_access' should be true if called from a real cpu write 2208 * access: the virtual CPU will exit the current TB if code is modified inside 2209 * this TB. 2210 * 2211 * Called with mmap_lock held for user-mode emulation 2212 */ 2213 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end) 2214 { 2215 struct page_collection *pages; 2216 PageDesc *p; 2217 2218 assert_memory_lock(); 2219 2220 p = page_find(start >> TARGET_PAGE_BITS); 2221 if (p == NULL) { 2222 return; 2223 } 2224 pages = page_collection_lock(start, end); 2225 tb_invalidate_phys_page_range__locked(pages, p, start, end, 0); 2226 page_collection_unlock(pages); 2227 } 2228 2229 /* 2230 * Invalidate all TBs which intersect with the target physical address range 2231 * [start;end[. NOTE: start and end may refer to *different* physical pages. 2232 * 'is_cpu_write_access' should be true if called from a real cpu write 2233 * access: the virtual CPU will exit the current TB if code is modified inside 2234 * this TB. 2235 * 2236 * Called with mmap_lock held for user-mode emulation. 2237 */ 2238 #ifdef CONFIG_SOFTMMU 2239 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end) 2240 #else 2241 void tb_invalidate_phys_range(target_ulong start, target_ulong end) 2242 #endif 2243 { 2244 struct page_collection *pages; 2245 tb_page_addr_t next; 2246 2247 assert_memory_lock(); 2248 2249 pages = page_collection_lock(start, end); 2250 for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE; 2251 start < end; 2252 start = next, next += TARGET_PAGE_SIZE) { 2253 PageDesc *pd = page_find(start >> TARGET_PAGE_BITS); 2254 tb_page_addr_t bound = MIN(next, end); 2255 2256 if (pd == NULL) { 2257 continue; 2258 } 2259 tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0); 2260 } 2261 page_collection_unlock(pages); 2262 } 2263 2264 #ifdef CONFIG_SOFTMMU 2265 /* len must be <= 8 and start must be a multiple of len. 2266 * Called via softmmu_template.h when code areas are written to with 2267 * iothread mutex not held. 2268 * 2269 * Call with all @pages in the range [@start, @start + len[ locked. 2270 */ 2271 void tb_invalidate_phys_page_fast(struct page_collection *pages, 2272 tb_page_addr_t start, int len, 2273 uintptr_t retaddr) 2274 { 2275 PageDesc *p; 2276 2277 assert_memory_lock(); 2278 2279 p = page_find(start >> TARGET_PAGE_BITS); 2280 if (!p) { 2281 return; 2282 } 2283 2284 assert_page_locked(p); 2285 if (!p->code_bitmap && 2286 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) { 2287 build_page_bitmap(p); 2288 } 2289 if (p->code_bitmap) { 2290 unsigned int nr; 2291 unsigned long b; 2292 2293 nr = start & ~TARGET_PAGE_MASK; 2294 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1)); 2295 if (b & ((1 << len) - 1)) { 2296 goto do_invalidate; 2297 } 2298 } else { 2299 do_invalidate: 2300 tb_invalidate_phys_page_range__locked(pages, p, start, start + len, 2301 retaddr); 2302 } 2303 } 2304 #else 2305 /* Called with mmap_lock held. If pc is not 0 then it indicates the 2306 * host PC of the faulting store instruction that caused this invalidate. 2307 * Returns true if the caller needs to abort execution of the current 2308 * TB (because it was modified by this store and the guest CPU has 2309 * precise-SMC semantics). 2310 */ 2311 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc) 2312 { 2313 TranslationBlock *tb; 2314 PageDesc *p; 2315 int n; 2316 #ifdef TARGET_HAS_PRECISE_SMC 2317 TranslationBlock *current_tb = NULL; 2318 CPUState *cpu = current_cpu; 2319 CPUArchState *env = NULL; 2320 int current_tb_modified = 0; 2321 target_ulong current_pc = 0; 2322 target_ulong current_cs_base = 0; 2323 uint32_t current_flags = 0; 2324 #endif 2325 2326 assert_memory_lock(); 2327 2328 addr &= TARGET_PAGE_MASK; 2329 p = page_find(addr >> TARGET_PAGE_BITS); 2330 if (!p) { 2331 return false; 2332 } 2333 2334 #ifdef TARGET_HAS_PRECISE_SMC 2335 if (p->first_tb && pc != 0) { 2336 current_tb = tcg_tb_lookup(pc); 2337 } 2338 if (cpu != NULL) { 2339 env = cpu->env_ptr; 2340 } 2341 #endif 2342 assert_page_locked(p); 2343 PAGE_FOR_EACH_TB(p, tb, n) { 2344 #ifdef TARGET_HAS_PRECISE_SMC 2345 if (current_tb == tb && 2346 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { 2347 /* If we are modifying the current TB, we must stop 2348 its execution. We could be more precise by checking 2349 that the modification is after the current PC, but it 2350 would require a specialized function to partially 2351 restore the CPU state */ 2352 2353 current_tb_modified = 1; 2354 cpu_restore_state_from_tb(cpu, current_tb, pc, true); 2355 cpu_get_tb_cpu_state(env, ¤t_pc, ¤t_cs_base, 2356 ¤t_flags); 2357 } 2358 #endif /* TARGET_HAS_PRECISE_SMC */ 2359 tb_phys_invalidate(tb, addr); 2360 } 2361 p->first_tb = (uintptr_t)NULL; 2362 #ifdef TARGET_HAS_PRECISE_SMC 2363 if (current_tb_modified) { 2364 /* Force execution of one insn next time. */ 2365 cpu->cflags_next_tb = 1 | curr_cflags(); 2366 return true; 2367 } 2368 #endif 2369 2370 return false; 2371 } 2372 #endif 2373 2374 /* user-mode: call with mmap_lock held */ 2375 void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr) 2376 { 2377 TranslationBlock *tb; 2378 2379 assert_memory_lock(); 2380 2381 tb = tcg_tb_lookup(retaddr); 2382 if (tb) { 2383 /* We can use retranslation to find the PC. */ 2384 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 2385 tb_phys_invalidate(tb, -1); 2386 } else { 2387 /* The exception probably happened in a helper. The CPU state should 2388 have been saved before calling it. Fetch the PC from there. */ 2389 CPUArchState *env = cpu->env_ptr; 2390 target_ulong pc, cs_base; 2391 tb_page_addr_t addr; 2392 uint32_t flags; 2393 2394 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags); 2395 addr = get_page_addr_code(env, pc); 2396 if (addr != -1) { 2397 tb_invalidate_phys_range(addr, addr + 1); 2398 } 2399 } 2400 } 2401 2402 #ifndef CONFIG_USER_ONLY 2403 /* 2404 * In deterministic execution mode, instructions doing device I/Os 2405 * must be at the end of the TB. 2406 * 2407 * Called by softmmu_template.h, with iothread mutex not held. 2408 */ 2409 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 2410 { 2411 TranslationBlock *tb; 2412 CPUClass *cc; 2413 uint32_t n; 2414 2415 tb = tcg_tb_lookup(retaddr); 2416 if (!tb) { 2417 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 2418 (void *)retaddr); 2419 } 2420 cpu_restore_state_from_tb(cpu, tb, retaddr, true); 2421 2422 /* 2423 * Some guests must re-execute the branch when re-executing a delay 2424 * slot instruction. When this is the case, adjust icount and N 2425 * to account for the re-execution of the branch. 2426 */ 2427 n = 1; 2428 cc = CPU_GET_CLASS(cpu); 2429 if (cc->tcg_ops->io_recompile_replay_branch && 2430 cc->tcg_ops->io_recompile_replay_branch(cpu, tb)) { 2431 cpu_neg(cpu)->icount_decr.u16.low++; 2432 n = 2; 2433 } 2434 2435 /* 2436 * Exit the loop and potentially generate a new TB executing the 2437 * just the I/O insns. We also limit instrumentation to memory 2438 * operations only (which execute after completion) so we don't 2439 * double instrument the instruction. 2440 */ 2441 cpu->cflags_next_tb = curr_cflags() | CF_MEMI_ONLY | CF_LAST_IO | n; 2442 2443 qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc, 2444 "cpu_io_recompile: rewound execution of TB to " 2445 TARGET_FMT_lx "\n", tb->pc); 2446 2447 cpu_loop_exit_noexc(cpu); 2448 } 2449 2450 static void print_qht_statistics(struct qht_stats hst) 2451 { 2452 uint32_t hgram_opts; 2453 size_t hgram_bins; 2454 char *hgram; 2455 2456 if (!hst.head_buckets) { 2457 return; 2458 } 2459 qemu_printf("TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n", 2460 hst.used_head_buckets, hst.head_buckets, 2461 (double)hst.used_head_buckets / hst.head_buckets * 100); 2462 2463 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 2464 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT; 2465 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) { 2466 hgram_opts |= QDIST_PR_NODECIMAL; 2467 } 2468 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts); 2469 qemu_printf("TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n", 2470 qdist_avg(&hst.occupancy) * 100, hgram); 2471 g_free(hgram); 2472 2473 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS; 2474 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain); 2475 if (hgram_bins > 10) { 2476 hgram_bins = 10; 2477 } else { 2478 hgram_bins = 0; 2479 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE; 2480 } 2481 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts); 2482 qemu_printf("TB hash avg chain %0.3f buckets. Histogram: %s\n", 2483 qdist_avg(&hst.chain), hgram); 2484 g_free(hgram); 2485 } 2486 2487 struct tb_tree_stats { 2488 size_t nb_tbs; 2489 size_t host_size; 2490 size_t target_size; 2491 size_t max_target_size; 2492 size_t direct_jmp_count; 2493 size_t direct_jmp2_count; 2494 size_t cross_page; 2495 }; 2496 2497 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data) 2498 { 2499 const TranslationBlock *tb = value; 2500 struct tb_tree_stats *tst = data; 2501 2502 tst->nb_tbs++; 2503 tst->host_size += tb->tc.size; 2504 tst->target_size += tb->size; 2505 if (tb->size > tst->max_target_size) { 2506 tst->max_target_size = tb->size; 2507 } 2508 if (tb->page_addr[1] != -1) { 2509 tst->cross_page++; 2510 } 2511 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) { 2512 tst->direct_jmp_count++; 2513 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) { 2514 tst->direct_jmp2_count++; 2515 } 2516 } 2517 return false; 2518 } 2519 2520 void dump_exec_info(void) 2521 { 2522 struct tb_tree_stats tst = {}; 2523 struct qht_stats hst; 2524 size_t nb_tbs, flush_full, flush_part, flush_elide; 2525 2526 tcg_tb_foreach(tb_tree_stats_iter, &tst); 2527 nb_tbs = tst.nb_tbs; 2528 /* XXX: avoid using doubles ? */ 2529 qemu_printf("Translation buffer state:\n"); 2530 /* 2531 * Report total code size including the padding and TB structs; 2532 * otherwise users might think "-accel tcg,tb-size" is not honoured. 2533 * For avg host size we use the precise numbers from tb_tree_stats though. 2534 */ 2535 qemu_printf("gen code size %zu/%zu\n", 2536 tcg_code_size(), tcg_code_capacity()); 2537 qemu_printf("TB count %zu\n", nb_tbs); 2538 qemu_printf("TB avg target size %zu max=%zu bytes\n", 2539 nb_tbs ? tst.target_size / nb_tbs : 0, 2540 tst.max_target_size); 2541 qemu_printf("TB avg host size %zu bytes (expansion ratio: %0.1f)\n", 2542 nb_tbs ? tst.host_size / nb_tbs : 0, 2543 tst.target_size ? (double)tst.host_size / tst.target_size : 0); 2544 qemu_printf("cross page TB count %zu (%zu%%)\n", tst.cross_page, 2545 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0); 2546 qemu_printf("direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n", 2547 tst.direct_jmp_count, 2548 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0, 2549 tst.direct_jmp2_count, 2550 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0); 2551 2552 qht_statistics_init(&tb_ctx.htable, &hst); 2553 print_qht_statistics(hst); 2554 qht_statistics_destroy(&hst); 2555 2556 qemu_printf("\nStatistics:\n"); 2557 qemu_printf("TB flush count %u\n", 2558 qatomic_read(&tb_ctx.tb_flush_count)); 2559 qemu_printf("TB invalidate count %zu\n", 2560 tcg_tb_phys_invalidate_count()); 2561 2562 tlb_flush_counts(&flush_full, &flush_part, &flush_elide); 2563 qemu_printf("TLB full flushes %zu\n", flush_full); 2564 qemu_printf("TLB partial flushes %zu\n", flush_part); 2565 qemu_printf("TLB elided flushes %zu\n", flush_elide); 2566 tcg_dump_info(); 2567 } 2568 2569 void dump_opcount_info(void) 2570 { 2571 tcg_dump_op_count(); 2572 } 2573 2574 #else /* CONFIG_USER_ONLY */ 2575 2576 void cpu_interrupt(CPUState *cpu, int mask) 2577 { 2578 g_assert(qemu_mutex_iothread_locked()); 2579 cpu->interrupt_request |= mask; 2580 qatomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1); 2581 } 2582 2583 /* 2584 * Walks guest process memory "regions" one by one 2585 * and calls callback function 'fn' for each region. 2586 */ 2587 struct walk_memory_regions_data { 2588 walk_memory_regions_fn fn; 2589 void *priv; 2590 target_ulong start; 2591 int prot; 2592 }; 2593 2594 static int walk_memory_regions_end(struct walk_memory_regions_data *data, 2595 target_ulong end, int new_prot) 2596 { 2597 if (data->start != -1u) { 2598 int rc = data->fn(data->priv, data->start, end, data->prot); 2599 if (rc != 0) { 2600 return rc; 2601 } 2602 } 2603 2604 data->start = (new_prot ? end : -1u); 2605 data->prot = new_prot; 2606 2607 return 0; 2608 } 2609 2610 static int walk_memory_regions_1(struct walk_memory_regions_data *data, 2611 target_ulong base, int level, void **lp) 2612 { 2613 target_ulong pa; 2614 int i, rc; 2615 2616 if (*lp == NULL) { 2617 return walk_memory_regions_end(data, base, 0); 2618 } 2619 2620 if (level == 0) { 2621 PageDesc *pd = *lp; 2622 2623 for (i = 0; i < V_L2_SIZE; ++i) { 2624 int prot = pd[i].flags; 2625 2626 pa = base | (i << TARGET_PAGE_BITS); 2627 if (prot != data->prot) { 2628 rc = walk_memory_regions_end(data, pa, prot); 2629 if (rc != 0) { 2630 return rc; 2631 } 2632 } 2633 } 2634 } else { 2635 void **pp = *lp; 2636 2637 for (i = 0; i < V_L2_SIZE; ++i) { 2638 pa = base | ((target_ulong)i << 2639 (TARGET_PAGE_BITS + V_L2_BITS * level)); 2640 rc = walk_memory_regions_1(data, pa, level - 1, pp + i); 2641 if (rc != 0) { 2642 return rc; 2643 } 2644 } 2645 } 2646 2647 return 0; 2648 } 2649 2650 int walk_memory_regions(void *priv, walk_memory_regions_fn fn) 2651 { 2652 struct walk_memory_regions_data data; 2653 uintptr_t i, l1_sz = v_l1_size; 2654 2655 data.fn = fn; 2656 data.priv = priv; 2657 data.start = -1u; 2658 data.prot = 0; 2659 2660 for (i = 0; i < l1_sz; i++) { 2661 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS); 2662 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i); 2663 if (rc != 0) { 2664 return rc; 2665 } 2666 } 2667 2668 return walk_memory_regions_end(&data, 0, 0); 2669 } 2670 2671 static int dump_region(void *priv, target_ulong start, 2672 target_ulong end, unsigned long prot) 2673 { 2674 FILE *f = (FILE *)priv; 2675 2676 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx 2677 " "TARGET_FMT_lx" %c%c%c\n", 2678 start, end, end - start, 2679 ((prot & PAGE_READ) ? 'r' : '-'), 2680 ((prot & PAGE_WRITE) ? 'w' : '-'), 2681 ((prot & PAGE_EXEC) ? 'x' : '-')); 2682 2683 return 0; 2684 } 2685 2686 /* dump memory mappings */ 2687 void page_dump(FILE *f) 2688 { 2689 const int length = sizeof(target_ulong) * 2; 2690 (void) fprintf(f, "%-*s %-*s %-*s %s\n", 2691 length, "start", length, "end", length, "size", "prot"); 2692 walk_memory_regions(f, dump_region); 2693 } 2694 2695 int page_get_flags(target_ulong address) 2696 { 2697 PageDesc *p; 2698 2699 p = page_find(address >> TARGET_PAGE_BITS); 2700 if (!p) { 2701 return 0; 2702 } 2703 return p->flags; 2704 } 2705 2706 /* Modify the flags of a page and invalidate the code if necessary. 2707 The flag PAGE_WRITE_ORG is positioned automatically depending 2708 on PAGE_WRITE. The mmap_lock should already be held. */ 2709 void page_set_flags(target_ulong start, target_ulong end, int flags) 2710 { 2711 target_ulong addr, len; 2712 bool reset_target_data; 2713 2714 /* This function should never be called with addresses outside the 2715 guest address space. If this assert fires, it probably indicates 2716 a missing call to h2g_valid. */ 2717 assert(end - 1 <= GUEST_ADDR_MAX); 2718 assert(start < end); 2719 assert_memory_lock(); 2720 2721 start = start & TARGET_PAGE_MASK; 2722 end = TARGET_PAGE_ALIGN(end); 2723 2724 if (flags & PAGE_WRITE) { 2725 flags |= PAGE_WRITE_ORG; 2726 } 2727 reset_target_data = !(flags & PAGE_VALID) || (flags & PAGE_RESET); 2728 flags &= ~PAGE_RESET; 2729 2730 for (addr = start, len = end - start; 2731 len != 0; 2732 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2733 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1); 2734 2735 /* If the write protection bit is set, then we invalidate 2736 the code inside. */ 2737 if (!(p->flags & PAGE_WRITE) && 2738 (flags & PAGE_WRITE) && 2739 p->first_tb) { 2740 tb_invalidate_phys_page(addr, 0); 2741 } 2742 if (reset_target_data && p->target_data) { 2743 g_free(p->target_data); 2744 p->target_data = NULL; 2745 } 2746 p->flags = flags; 2747 } 2748 } 2749 2750 void *page_get_target_data(target_ulong address) 2751 { 2752 PageDesc *p = page_find(address >> TARGET_PAGE_BITS); 2753 return p ? p->target_data : NULL; 2754 } 2755 2756 void *page_alloc_target_data(target_ulong address, size_t size) 2757 { 2758 PageDesc *p = page_find(address >> TARGET_PAGE_BITS); 2759 void *ret = NULL; 2760 2761 if (p->flags & PAGE_VALID) { 2762 ret = p->target_data; 2763 if (!ret) { 2764 p->target_data = ret = g_malloc0(size); 2765 } 2766 } 2767 return ret; 2768 } 2769 2770 int page_check_range(target_ulong start, target_ulong len, int flags) 2771 { 2772 PageDesc *p; 2773 target_ulong end; 2774 target_ulong addr; 2775 2776 /* This function should never be called with addresses outside the 2777 guest address space. If this assert fires, it probably indicates 2778 a missing call to h2g_valid. */ 2779 if (TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS) { 2780 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS)); 2781 } 2782 2783 if (len == 0) { 2784 return 0; 2785 } 2786 if (start + len - 1 < start) { 2787 /* We've wrapped around. */ 2788 return -1; 2789 } 2790 2791 /* must do before we loose bits in the next step */ 2792 end = TARGET_PAGE_ALIGN(start + len); 2793 start = start & TARGET_PAGE_MASK; 2794 2795 for (addr = start, len = end - start; 2796 len != 0; 2797 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) { 2798 p = page_find(addr >> TARGET_PAGE_BITS); 2799 if (!p) { 2800 return -1; 2801 } 2802 if (!(p->flags & PAGE_VALID)) { 2803 return -1; 2804 } 2805 2806 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) { 2807 return -1; 2808 } 2809 if (flags & PAGE_WRITE) { 2810 if (!(p->flags & PAGE_WRITE_ORG)) { 2811 return -1; 2812 } 2813 /* unprotect the page if it was put read-only because it 2814 contains translated code */ 2815 if (!(p->flags & PAGE_WRITE)) { 2816 if (!page_unprotect(addr, 0)) { 2817 return -1; 2818 } 2819 } 2820 } 2821 } 2822 return 0; 2823 } 2824 2825 /* called from signal handler: invalidate the code and unprotect the 2826 * page. Return 0 if the fault was not handled, 1 if it was handled, 2827 * and 2 if it was handled but the caller must cause the TB to be 2828 * immediately exited. (We can only return 2 if the 'pc' argument is 2829 * non-zero.) 2830 */ 2831 int page_unprotect(target_ulong address, uintptr_t pc) 2832 { 2833 unsigned int prot; 2834 bool current_tb_invalidated; 2835 PageDesc *p; 2836 target_ulong host_start, host_end, addr; 2837 2838 /* Technically this isn't safe inside a signal handler. However we 2839 know this only ever happens in a synchronous SEGV handler, so in 2840 practice it seems to be ok. */ 2841 mmap_lock(); 2842 2843 p = page_find(address >> TARGET_PAGE_BITS); 2844 if (!p) { 2845 mmap_unlock(); 2846 return 0; 2847 } 2848 2849 /* if the page was really writable, then we change its 2850 protection back to writable */ 2851 if (p->flags & PAGE_WRITE_ORG) { 2852 current_tb_invalidated = false; 2853 if (p->flags & PAGE_WRITE) { 2854 /* If the page is actually marked WRITE then assume this is because 2855 * this thread raced with another one which got here first and 2856 * set the page to PAGE_WRITE and did the TB invalidate for us. 2857 */ 2858 #ifdef TARGET_HAS_PRECISE_SMC 2859 TranslationBlock *current_tb = tcg_tb_lookup(pc); 2860 if (current_tb) { 2861 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID; 2862 } 2863 #endif 2864 } else { 2865 host_start = address & qemu_host_page_mask; 2866 host_end = host_start + qemu_host_page_size; 2867 2868 prot = 0; 2869 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) { 2870 p = page_find(addr >> TARGET_PAGE_BITS); 2871 p->flags |= PAGE_WRITE; 2872 prot |= p->flags; 2873 2874 /* and since the content will be modified, we must invalidate 2875 the corresponding translated code. */ 2876 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc); 2877 #ifdef CONFIG_USER_ONLY 2878 if (DEBUG_TB_CHECK_GATE) { 2879 tb_invalidate_check(addr); 2880 } 2881 #endif 2882 } 2883 mprotect((void *)g2h_untagged(host_start), qemu_host_page_size, 2884 prot & PAGE_BITS); 2885 } 2886 mmap_unlock(); 2887 /* If current TB was invalidated return to main loop */ 2888 return current_tb_invalidated ? 2 : 1; 2889 } 2890 mmap_unlock(); 2891 return 0; 2892 } 2893 #endif /* CONFIG_USER_ONLY */ 2894 2895 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */ 2896 void tcg_flush_softmmu_tlb(CPUState *cs) 2897 { 2898 #ifdef CONFIG_SOFTMMU 2899 tlb_flush(cs); 2900 #endif 2901 } 2902