xref: /qemu/accel/tcg/cpu-exec.c (revision 408b2b3d)
1 /*
2  *  emulator main execution loop
3  *
4  *  Copyright (c) 2003-2005 Fabrice Bellard
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "qemu/qemu-print.h"
22 #include "qapi/error.h"
23 #include "qapi/type-helpers.h"
24 #include "hw/core/tcg-cpu-ops.h"
25 #include "trace.h"
26 #include "disas/disas.h"
27 #include "exec/exec-all.h"
28 #include "tcg/tcg.h"
29 #include "qemu/atomic.h"
30 #include "qemu/rcu.h"
31 #include "exec/log.h"
32 #include "qemu/main-loop.h"
33 #include "sysemu/cpus.h"
34 #include "exec/cpu-all.h"
35 #include "sysemu/cpu-timers.h"
36 #include "exec/replay-core.h"
37 #include "sysemu/tcg.h"
38 #include "exec/helper-proto-common.h"
39 #include "tb-jmp-cache.h"
40 #include "tb-hash.h"
41 #include "tb-context.h"
42 #include "internal-common.h"
43 #include "internal-target.h"
44 #if defined(CONFIG_USER_ONLY)
45 #include "user-retaddr.h"
46 #endif
47 
48 /* -icount align implementation. */
49 
50 typedef struct SyncClocks {
51     int64_t diff_clk;
52     int64_t last_cpu_icount;
53     int64_t realtime_clock;
54 } SyncClocks;
55 
56 #if !defined(CONFIG_USER_ONLY)
57 /* Allow the guest to have a max 3ms advance.
58  * The difference between the 2 clocks could therefore
59  * oscillate around 0.
60  */
61 #define VM_CLOCK_ADVANCE 3000000
62 #define THRESHOLD_REDUCE 1.5
63 #define MAX_DELAY_PRINT_RATE 2000000000LL
64 #define MAX_NB_PRINTS 100
65 
66 int64_t max_delay;
67 int64_t max_advance;
68 
align_clocks(SyncClocks * sc,CPUState * cpu)69 static void align_clocks(SyncClocks *sc, CPUState *cpu)
70 {
71     int64_t cpu_icount;
72 
73     if (!icount_align_option) {
74         return;
75     }
76 
77     cpu_icount = cpu->icount_extra + cpu->neg.icount_decr.u16.low;
78     sc->diff_clk += icount_to_ns(sc->last_cpu_icount - cpu_icount);
79     sc->last_cpu_icount = cpu_icount;
80 
81     if (sc->diff_clk > VM_CLOCK_ADVANCE) {
82 #ifndef _WIN32
83         struct timespec sleep_delay, rem_delay;
84         sleep_delay.tv_sec = sc->diff_clk / 1000000000LL;
85         sleep_delay.tv_nsec = sc->diff_clk % 1000000000LL;
86         if (nanosleep(&sleep_delay, &rem_delay) < 0) {
87             sc->diff_clk = rem_delay.tv_sec * 1000000000LL + rem_delay.tv_nsec;
88         } else {
89             sc->diff_clk = 0;
90         }
91 #else
92         Sleep(sc->diff_clk / SCALE_MS);
93         sc->diff_clk = 0;
94 #endif
95     }
96 }
97 
print_delay(const SyncClocks * sc)98 static void print_delay(const SyncClocks *sc)
99 {
100     static float threshold_delay;
101     static int64_t last_realtime_clock;
102     static int nb_prints;
103 
104     if (icount_align_option &&
105         sc->realtime_clock - last_realtime_clock >= MAX_DELAY_PRINT_RATE &&
106         nb_prints < MAX_NB_PRINTS) {
107         if ((-sc->diff_clk / (float)1000000000LL > threshold_delay) ||
108             (-sc->diff_clk / (float)1000000000LL <
109              (threshold_delay - THRESHOLD_REDUCE))) {
110             threshold_delay = (-sc->diff_clk / 1000000000LL) + 1;
111             qemu_printf("Warning: The guest is now late by %.1f to %.1f seconds\n",
112                         threshold_delay - 1,
113                         threshold_delay);
114             nb_prints++;
115             last_realtime_clock = sc->realtime_clock;
116         }
117     }
118 }
119 
init_delay_params(SyncClocks * sc,CPUState * cpu)120 static void init_delay_params(SyncClocks *sc, CPUState *cpu)
121 {
122     if (!icount_align_option) {
123         return;
124     }
125     sc->realtime_clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT);
126     sc->diff_clk = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - sc->realtime_clock;
127     sc->last_cpu_icount
128         = cpu->icount_extra + cpu->neg.icount_decr.u16.low;
129     if (sc->diff_clk < max_delay) {
130         max_delay = sc->diff_clk;
131     }
132     if (sc->diff_clk > max_advance) {
133         max_advance = sc->diff_clk;
134     }
135 
136     /* Print every 2s max if the guest is late. We limit the number
137        of printed messages to NB_PRINT_MAX(currently 100) */
138     print_delay(sc);
139 }
140 #else
align_clocks(SyncClocks * sc,const CPUState * cpu)141 static void align_clocks(SyncClocks *sc, const CPUState *cpu)
142 {
143 }
144 
init_delay_params(SyncClocks * sc,const CPUState * cpu)145 static void init_delay_params(SyncClocks *sc, const CPUState *cpu)
146 {
147 }
148 #endif /* CONFIG USER ONLY */
149 
tcg_cflags_has(CPUState * cpu,uint32_t flags)150 bool tcg_cflags_has(CPUState *cpu, uint32_t flags)
151 {
152     return cpu->tcg_cflags & flags;
153 }
154 
tcg_cflags_set(CPUState * cpu,uint32_t flags)155 void tcg_cflags_set(CPUState *cpu, uint32_t flags)
156 {
157     cpu->tcg_cflags |= flags;
158 }
159 
curr_cflags(CPUState * cpu)160 uint32_t curr_cflags(CPUState *cpu)
161 {
162     uint32_t cflags = cpu->tcg_cflags;
163 
164     /*
165      * Record gdb single-step.  We should be exiting the TB by raising
166      * EXCP_DEBUG, but to simplify other tests, disable chaining too.
167      *
168      * For singlestep and -d nochain, suppress goto_tb so that
169      * we can log -d cpu,exec after every TB.
170      */
171     if (unlikely(cpu->singlestep_enabled)) {
172         cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | CF_SINGLE_STEP | 1;
173     } else if (qatomic_read(&one_insn_per_tb)) {
174         cflags |= CF_NO_GOTO_TB | 1;
175     } else if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
176         cflags |= CF_NO_GOTO_TB;
177     }
178 
179     return cflags;
180 }
181 
182 struct tb_desc {
183     vaddr pc;
184     uint64_t cs_base;
185     CPUArchState *env;
186     tb_page_addr_t page_addr0;
187     uint32_t flags;
188     uint32_t cflags;
189 };
190 
tb_lookup_cmp(const void * p,const void * d)191 static bool tb_lookup_cmp(const void *p, const void *d)
192 {
193     const TranslationBlock *tb = p;
194     const struct tb_desc *desc = d;
195 
196     if ((tb_cflags(tb) & CF_PCREL || tb->pc == desc->pc) &&
197         tb_page_addr0(tb) == desc->page_addr0 &&
198         tb->cs_base == desc->cs_base &&
199         tb->flags == desc->flags &&
200         tb_cflags(tb) == desc->cflags) {
201         /* check next page if needed */
202         tb_page_addr_t tb_phys_page1 = tb_page_addr1(tb);
203         if (tb_phys_page1 == -1) {
204             return true;
205         } else {
206             tb_page_addr_t phys_page1;
207             vaddr virt_page1;
208 
209             /*
210              * We know that the first page matched, and an otherwise valid TB
211              * encountered an incomplete instruction at the end of that page,
212              * therefore we know that generating a new TB from the current PC
213              * must also require reading from the next page -- even if the
214              * second pages do not match, and therefore the resulting insn
215              * is different for the new TB.  Therefore any exception raised
216              * here by the faulting lookup is not premature.
217              */
218             virt_page1 = TARGET_PAGE_ALIGN(desc->pc);
219             phys_page1 = get_page_addr_code(desc->env, virt_page1);
220             if (tb_phys_page1 == phys_page1) {
221                 return true;
222             }
223         }
224     }
225     return false;
226 }
227 
tb_htable_lookup(CPUState * cpu,vaddr pc,uint64_t cs_base,uint32_t flags,uint32_t cflags)228 static TranslationBlock *tb_htable_lookup(CPUState *cpu, vaddr pc,
229                                           uint64_t cs_base, uint32_t flags,
230                                           uint32_t cflags)
231 {
232     tb_page_addr_t phys_pc;
233     struct tb_desc desc;
234     uint32_t h;
235 
236     desc.env = cpu_env(cpu);
237     desc.cs_base = cs_base;
238     desc.flags = flags;
239     desc.cflags = cflags;
240     desc.pc = pc;
241     phys_pc = get_page_addr_code(desc.env, pc);
242     if (phys_pc == -1) {
243         return NULL;
244     }
245     desc.page_addr0 = phys_pc;
246     h = tb_hash_func(phys_pc, (cflags & CF_PCREL ? 0 : pc),
247                      flags, cs_base, cflags);
248     return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp);
249 }
250 
251 /* Might cause an exception, so have a longjmp destination ready */
tb_lookup(CPUState * cpu,vaddr pc,uint64_t cs_base,uint32_t flags,uint32_t cflags)252 static inline TranslationBlock *tb_lookup(CPUState *cpu, vaddr pc,
253                                           uint64_t cs_base, uint32_t flags,
254                                           uint32_t cflags)
255 {
256     TranslationBlock *tb;
257     CPUJumpCache *jc;
258     uint32_t hash;
259 
260     /* we should never be trying to look up an INVALID tb */
261     tcg_debug_assert(!(cflags & CF_INVALID));
262 
263     hash = tb_jmp_cache_hash_func(pc);
264     jc = cpu->tb_jmp_cache;
265 
266     tb = qatomic_read(&jc->array[hash].tb);
267     if (likely(tb &&
268                jc->array[hash].pc == pc &&
269                tb->cs_base == cs_base &&
270                tb->flags == flags &&
271                tb_cflags(tb) == cflags)) {
272         goto hit;
273     }
274 
275     tb = tb_htable_lookup(cpu, pc, cs_base, flags, cflags);
276     if (tb == NULL) {
277         return NULL;
278     }
279 
280     jc->array[hash].pc = pc;
281     qatomic_set(&jc->array[hash].tb, tb);
282 
283 hit:
284     /*
285      * As long as tb is not NULL, the contents are consistent.  Therefore,
286      * the virtual PC has to match for non-CF_PCREL translations.
287      */
288     assert((tb_cflags(tb) & CF_PCREL) || tb->pc == pc);
289     return tb;
290 }
291 
log_cpu_exec(vaddr pc,CPUState * cpu,const TranslationBlock * tb)292 static void log_cpu_exec(vaddr pc, CPUState *cpu,
293                          const TranslationBlock *tb)
294 {
295     if (qemu_log_in_addr_range(pc)) {
296         qemu_log_mask(CPU_LOG_EXEC,
297                       "Trace %d: %p [%08" PRIx64
298                       "/%016" VADDR_PRIx "/%08x/%08x] %s\n",
299                       cpu->cpu_index, tb->tc.ptr, tb->cs_base, pc,
300                       tb->flags, tb->cflags, lookup_symbol(pc));
301 
302         if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
303             FILE *logfile = qemu_log_trylock();
304             if (logfile) {
305                 int flags = 0;
306 
307                 if (qemu_loglevel_mask(CPU_LOG_TB_FPU)) {
308                     flags |= CPU_DUMP_FPU;
309                 }
310 #if defined(TARGET_I386)
311                 flags |= CPU_DUMP_CCOP;
312 #endif
313                 if (qemu_loglevel_mask(CPU_LOG_TB_VPU)) {
314                     flags |= CPU_DUMP_VPU;
315                 }
316                 cpu_dump_state(cpu, logfile, flags);
317                 qemu_log_unlock(logfile);
318             }
319         }
320     }
321 }
322 
check_for_breakpoints_slow(CPUState * cpu,vaddr pc,uint32_t * cflags)323 static bool check_for_breakpoints_slow(CPUState *cpu, vaddr pc,
324                                        uint32_t *cflags)
325 {
326     CPUBreakpoint *bp;
327     bool match_page = false;
328 
329     /*
330      * Singlestep overrides breakpoints.
331      * This requirement is visible in the record-replay tests, where
332      * we would fail to make forward progress in reverse-continue.
333      *
334      * TODO: gdb singlestep should only override gdb breakpoints,
335      * so that one could (gdb) singlestep into the guest kernel's
336      * architectural breakpoint handler.
337      */
338     if (cpu->singlestep_enabled) {
339         return false;
340     }
341 
342     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
343         /*
344          * If we have an exact pc match, trigger the breakpoint.
345          * Otherwise, note matches within the page.
346          */
347         if (pc == bp->pc) {
348             bool match_bp = false;
349 
350             if (bp->flags & BP_GDB) {
351                 match_bp = true;
352             } else if (bp->flags & BP_CPU) {
353 #ifdef CONFIG_USER_ONLY
354                 g_assert_not_reached();
355 #else
356                 const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
357                 assert(tcg_ops->debug_check_breakpoint);
358                 match_bp = tcg_ops->debug_check_breakpoint(cpu);
359 #endif
360             }
361 
362             if (match_bp) {
363                 cpu->exception_index = EXCP_DEBUG;
364                 return true;
365             }
366         } else if (((pc ^ bp->pc) & TARGET_PAGE_MASK) == 0) {
367             match_page = true;
368         }
369     }
370 
371     /*
372      * Within the same page as a breakpoint, single-step,
373      * returning to helper_lookup_tb_ptr after each insn looking
374      * for the actual breakpoint.
375      *
376      * TODO: Perhaps better to record all of the TBs associated
377      * with a given virtual page that contains a breakpoint, and
378      * then invalidate them when a new overlapping breakpoint is
379      * set on the page.  Non-overlapping TBs would not be
380      * invalidated, nor would any TB need to be invalidated as
381      * breakpoints are removed.
382      */
383     if (match_page) {
384         *cflags = (*cflags & ~CF_COUNT_MASK) | CF_NO_GOTO_TB | CF_BP_PAGE | 1;
385     }
386     return false;
387 }
388 
check_for_breakpoints(CPUState * cpu,vaddr pc,uint32_t * cflags)389 static inline bool check_for_breakpoints(CPUState *cpu, vaddr pc,
390                                          uint32_t *cflags)
391 {
392     return unlikely(!QTAILQ_EMPTY(&cpu->breakpoints)) &&
393         check_for_breakpoints_slow(cpu, pc, cflags);
394 }
395 
396 /**
397  * helper_lookup_tb_ptr: quick check for next tb
398  * @env: current cpu state
399  *
400  * Look for an existing TB matching the current cpu state.
401  * If found, return the code pointer.  If not found, return
402  * the tcg epilogue so that we return into cpu_tb_exec.
403  */
HELPER(lookup_tb_ptr)404 const void *HELPER(lookup_tb_ptr)(CPUArchState *env)
405 {
406     CPUState *cpu = env_cpu(env);
407     TranslationBlock *tb;
408     vaddr pc;
409     uint64_t cs_base;
410     uint32_t flags, cflags;
411 
412     /*
413      * By definition we've just finished a TB, so I/O is OK.
414      * Avoid the possibility of calling cpu_io_recompile() if
415      * a page table walk triggered by tb_lookup() calling
416      * probe_access_internal() happens to touch an MMIO device.
417      * The next TB, if we chain to it, will clear the flag again.
418      */
419     cpu->neg.can_do_io = true;
420     cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
421 
422     cflags = curr_cflags(cpu);
423     if (check_for_breakpoints(cpu, pc, &cflags)) {
424         cpu_loop_exit(cpu);
425     }
426 
427     tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
428     if (tb == NULL) {
429         return tcg_code_gen_epilogue;
430     }
431 
432     if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) {
433         log_cpu_exec(pc, cpu, tb);
434     }
435 
436     return tb->tc.ptr;
437 }
438 
439 /* Execute a TB, and fix up the CPU state afterwards if necessary */
440 /*
441  * Disable CFI checks.
442  * TCG creates binary blobs at runtime, with the transformed code.
443  * A TB is a blob of binary code, created at runtime and called with an
444  * indirect function call. Since such function did not exist at compile time,
445  * the CFI runtime has no way to verify its signature and would fail.
446  * TCG is not considered a security-sensitive part of QEMU so this does not
447  * affect the impact of CFI in environment with high security requirements
448  */
449 static inline TranslationBlock * QEMU_DISABLE_CFI
cpu_tb_exec(CPUState * cpu,TranslationBlock * itb,int * tb_exit)450 cpu_tb_exec(CPUState *cpu, TranslationBlock *itb, int *tb_exit)
451 {
452     uintptr_t ret;
453     TranslationBlock *last_tb;
454     const void *tb_ptr = itb->tc.ptr;
455 
456     if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) {
457         log_cpu_exec(log_pc(cpu, itb), cpu, itb);
458     }
459 
460     qemu_thread_jit_execute();
461     ret = tcg_qemu_tb_exec(cpu_env(cpu), tb_ptr);
462     cpu->neg.can_do_io = true;
463     qemu_plugin_disable_mem_helpers(cpu);
464     /*
465      * TODO: Delay swapping back to the read-write region of the TB
466      * until we actually need to modify the TB.  The read-only copy,
467      * coming from the rx region, shares the same host TLB entry as
468      * the code that executed the exit_tb opcode that arrived here.
469      * If we insist on touching both the RX and the RW pages, we
470      * double the host TLB pressure.
471      */
472     last_tb = tcg_splitwx_to_rw((void *)(ret & ~TB_EXIT_MASK));
473     *tb_exit = ret & TB_EXIT_MASK;
474 
475     trace_exec_tb_exit(last_tb, *tb_exit);
476 
477     if (*tb_exit > TB_EXIT_IDX1) {
478         /* We didn't start executing this TB (eg because the instruction
479          * counter hit zero); we must restore the guest PC to the address
480          * of the start of the TB.
481          */
482         CPUClass *cc = cpu->cc;
483         const TCGCPUOps *tcg_ops = cc->tcg_ops;
484 
485         if (tcg_ops->synchronize_from_tb) {
486             tcg_ops->synchronize_from_tb(cpu, last_tb);
487         } else {
488             tcg_debug_assert(!(tb_cflags(last_tb) & CF_PCREL));
489             assert(cc->set_pc);
490             cc->set_pc(cpu, last_tb->pc);
491         }
492         if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
493             vaddr pc = log_pc(cpu, last_tb);
494             if (qemu_log_in_addr_range(pc)) {
495                 qemu_log("Stopped execution of TB chain before %p [%016"
496                          VADDR_PRIx "] %s\n",
497                          last_tb->tc.ptr, pc, lookup_symbol(pc));
498             }
499         }
500     }
501 
502     /*
503      * If gdb single-step, and we haven't raised another exception,
504      * raise a debug exception.  Single-step with another exception
505      * is handled in cpu_handle_exception.
506      */
507     if (unlikely(cpu->singlestep_enabled) && cpu->exception_index == -1) {
508         cpu->exception_index = EXCP_DEBUG;
509         cpu_loop_exit(cpu);
510     }
511 
512     return last_tb;
513 }
514 
515 
cpu_exec_enter(CPUState * cpu)516 static void cpu_exec_enter(CPUState *cpu)
517 {
518     const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
519 
520     if (tcg_ops->cpu_exec_enter) {
521         tcg_ops->cpu_exec_enter(cpu);
522     }
523 }
524 
cpu_exec_exit(CPUState * cpu)525 static void cpu_exec_exit(CPUState *cpu)
526 {
527     const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
528 
529     if (tcg_ops->cpu_exec_exit) {
530         tcg_ops->cpu_exec_exit(cpu);
531     }
532 }
533 
cpu_exec_longjmp_cleanup(CPUState * cpu)534 static void cpu_exec_longjmp_cleanup(CPUState *cpu)
535 {
536     /* Non-buggy compilers preserve this; assert the correct value. */
537     g_assert(cpu == current_cpu);
538 
539 #ifdef CONFIG_USER_ONLY
540     clear_helper_retaddr();
541     if (have_mmap_lock()) {
542         mmap_unlock();
543     }
544 #else
545     /*
546      * For softmmu, a tlb_fill fault during translation will land here,
547      * and we need to release any page locks held.  In system mode we
548      * have one tcg_ctx per thread, so we know it was this cpu doing
549      * the translation.
550      *
551      * Alternative 1: Install a cleanup to be called via an exception
552      * handling safe longjmp.  It seems plausible that all our hosts
553      * support such a thing.  We'd have to properly register unwind info
554      * for the JIT for EH, rather that just for GDB.
555      *
556      * Alternative 2: Set and restore cpu->jmp_env in tb_gen_code to
557      * capture the cpu_loop_exit longjmp, perform the cleanup, and
558      * jump again to arrive here.
559      */
560     if (tcg_ctx->gen_tb) {
561         tb_unlock_pages(tcg_ctx->gen_tb);
562         tcg_ctx->gen_tb = NULL;
563     }
564 #endif
565     if (bql_locked()) {
566         bql_unlock();
567     }
568     assert_no_pages_locked();
569 }
570 
cpu_exec_step_atomic(CPUState * cpu)571 void cpu_exec_step_atomic(CPUState *cpu)
572 {
573     CPUArchState *env = cpu_env(cpu);
574     TranslationBlock *tb;
575     vaddr pc;
576     uint64_t cs_base;
577     uint32_t flags, cflags;
578     int tb_exit;
579 
580     if (sigsetjmp(cpu->jmp_env, 0) == 0) {
581         start_exclusive();
582         g_assert(cpu == current_cpu);
583         g_assert(!cpu->running);
584         cpu->running = true;
585 
586         cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
587 
588         cflags = curr_cflags(cpu);
589         /* Execute in a serial context. */
590         cflags &= ~CF_PARALLEL;
591         /* After 1 insn, return and release the exclusive lock. */
592         cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | 1;
593         /*
594          * No need to check_for_breakpoints here.
595          * We only arrive in cpu_exec_step_atomic after beginning execution
596          * of an insn that includes an atomic operation we can't handle.
597          * Any breakpoint for this insn will have been recognized earlier.
598          */
599 
600         tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
601         if (tb == NULL) {
602             mmap_lock();
603             tb = tb_gen_code(cpu, pc, cs_base, flags, cflags);
604             mmap_unlock();
605         }
606 
607         cpu_exec_enter(cpu);
608         /* execute the generated code */
609         trace_exec_tb(tb, pc);
610         cpu_tb_exec(cpu, tb, &tb_exit);
611         cpu_exec_exit(cpu);
612     } else {
613         cpu_exec_longjmp_cleanup(cpu);
614     }
615 
616     /*
617      * As we start the exclusive region before codegen we must still
618      * be in the region if we longjump out of either the codegen or
619      * the execution.
620      */
621     g_assert(cpu_in_exclusive_context(cpu));
622     cpu->running = false;
623     end_exclusive();
624 }
625 
tb_set_jmp_target(TranslationBlock * tb,int n,uintptr_t addr)626 void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr)
627 {
628     /*
629      * Get the rx view of the structure, from which we find the
630      * executable code address, and tb_target_set_jmp_target can
631      * produce a pc-relative displacement to jmp_target_addr[n].
632      */
633     const TranslationBlock *c_tb = tcg_splitwx_to_rx(tb);
634     uintptr_t offset = tb->jmp_insn_offset[n];
635     uintptr_t jmp_rx = (uintptr_t)tb->tc.ptr + offset;
636     uintptr_t jmp_rw = jmp_rx - tcg_splitwx_diff;
637 
638     tb->jmp_target_addr[n] = addr;
639     tb_target_set_jmp_target(c_tb, n, jmp_rx, jmp_rw);
640 }
641 
tb_add_jump(TranslationBlock * tb,int n,TranslationBlock * tb_next)642 static inline void tb_add_jump(TranslationBlock *tb, int n,
643                                TranslationBlock *tb_next)
644 {
645     uintptr_t old;
646 
647     qemu_thread_jit_write();
648     assert(n < ARRAY_SIZE(tb->jmp_list_next));
649     qemu_spin_lock(&tb_next->jmp_lock);
650 
651     /* make sure the destination TB is valid */
652     if (tb_next->cflags & CF_INVALID) {
653         goto out_unlock_next;
654     }
655     /* Atomically claim the jump destination slot only if it was NULL */
656     old = qatomic_cmpxchg(&tb->jmp_dest[n], (uintptr_t)NULL,
657                           (uintptr_t)tb_next);
658     if (old) {
659         goto out_unlock_next;
660     }
661 
662     /* patch the native jump address */
663     tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc.ptr);
664 
665     /* add in TB jmp list */
666     tb->jmp_list_next[n] = tb_next->jmp_list_head;
667     tb_next->jmp_list_head = (uintptr_t)tb | n;
668 
669     qemu_spin_unlock(&tb_next->jmp_lock);
670 
671     qemu_log_mask(CPU_LOG_EXEC, "Linking TBs %p index %d -> %p\n",
672                   tb->tc.ptr, n, tb_next->tc.ptr);
673     return;
674 
675  out_unlock_next:
676     qemu_spin_unlock(&tb_next->jmp_lock);
677     return;
678 }
679 
cpu_handle_halt(CPUState * cpu)680 static inline bool cpu_handle_halt(CPUState *cpu)
681 {
682 #ifndef CONFIG_USER_ONLY
683     if (cpu->halted) {
684         const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
685         bool leave_halt;
686 
687         if (tcg_ops->cpu_exec_halt) {
688             leave_halt = tcg_ops->cpu_exec_halt(cpu);
689         } else {
690             leave_halt = cpu_has_work(cpu);
691         }
692         if (!leave_halt) {
693             return true;
694         }
695 
696         cpu->halted = 0;
697     }
698 #endif /* !CONFIG_USER_ONLY */
699 
700     return false;
701 }
702 
cpu_handle_debug_exception(CPUState * cpu)703 static inline void cpu_handle_debug_exception(CPUState *cpu)
704 {
705     const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
706     CPUWatchpoint *wp;
707 
708     if (!cpu->watchpoint_hit) {
709         QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
710             wp->flags &= ~BP_WATCHPOINT_HIT;
711         }
712     }
713 
714     if (tcg_ops->debug_excp_handler) {
715         tcg_ops->debug_excp_handler(cpu);
716     }
717 }
718 
cpu_handle_exception(CPUState * cpu,int * ret)719 static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
720 {
721     if (cpu->exception_index < 0) {
722 #ifndef CONFIG_USER_ONLY
723         if (replay_has_exception()
724             && cpu->neg.icount_decr.u16.low + cpu->icount_extra == 0) {
725             /* Execute just one insn to trigger exception pending in the log */
726             cpu->cflags_next_tb = (curr_cflags(cpu) & ~CF_USE_ICOUNT)
727                 | CF_NOIRQ | 1;
728         }
729 #endif
730         return false;
731     }
732 
733     if (cpu->exception_index >= EXCP_INTERRUPT) {
734         /* exit request from the cpu execution loop */
735         *ret = cpu->exception_index;
736         if (*ret == EXCP_DEBUG) {
737             cpu_handle_debug_exception(cpu);
738         }
739         cpu->exception_index = -1;
740         return true;
741     }
742 
743 #if defined(CONFIG_USER_ONLY)
744     /*
745      * If user mode only, we simulate a fake exception which will be
746      * handled outside the cpu execution loop.
747      */
748 #if defined(TARGET_I386)
749     const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
750     tcg_ops->fake_user_interrupt(cpu);
751 #endif /* TARGET_I386 */
752     *ret = cpu->exception_index;
753     cpu->exception_index = -1;
754     return true;
755 #else
756     if (replay_exception()) {
757         const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
758 
759         bql_lock();
760         tcg_ops->do_interrupt(cpu);
761         bql_unlock();
762         cpu->exception_index = -1;
763 
764         if (unlikely(cpu->singlestep_enabled)) {
765             /*
766              * After processing the exception, ensure an EXCP_DEBUG is
767              * raised when single-stepping so that GDB doesn't miss the
768              * next instruction.
769              */
770             *ret = EXCP_DEBUG;
771             cpu_handle_debug_exception(cpu);
772             return true;
773         }
774     } else if (!replay_has_interrupt()) {
775         /* give a chance to iothread in replay mode */
776         *ret = EXCP_INTERRUPT;
777         return true;
778     }
779 #endif
780 
781     return false;
782 }
783 
icount_exit_request(CPUState * cpu)784 static inline bool icount_exit_request(CPUState *cpu)
785 {
786     if (!icount_enabled()) {
787         return false;
788     }
789     if (cpu->cflags_next_tb != -1 && !(cpu->cflags_next_tb & CF_USE_ICOUNT)) {
790         return false;
791     }
792     return cpu->neg.icount_decr.u16.low + cpu->icount_extra == 0;
793 }
794 
cpu_handle_interrupt(CPUState * cpu,TranslationBlock ** last_tb)795 static inline bool cpu_handle_interrupt(CPUState *cpu,
796                                         TranslationBlock **last_tb)
797 {
798     /*
799      * If we have requested custom cflags with CF_NOIRQ we should
800      * skip checking here. Any pending interrupts will get picked up
801      * by the next TB we execute under normal cflags.
802      */
803     if (cpu->cflags_next_tb != -1 && cpu->cflags_next_tb & CF_NOIRQ) {
804         return false;
805     }
806 
807     /* Clear the interrupt flag now since we're processing
808      * cpu->interrupt_request and cpu->exit_request.
809      * Ensure zeroing happens before reading cpu->exit_request or
810      * cpu->interrupt_request (see also smp_wmb in cpu_exit())
811      */
812     qatomic_set_mb(&cpu->neg.icount_decr.u16.high, 0);
813 
814     if (unlikely(qatomic_read(&cpu->interrupt_request))) {
815         int interrupt_request;
816         bql_lock();
817         interrupt_request = cpu->interrupt_request;
818         if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) {
819             /* Mask out external interrupts for this step. */
820             interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK;
821         }
822         if (interrupt_request & CPU_INTERRUPT_DEBUG) {
823             cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
824             cpu->exception_index = EXCP_DEBUG;
825             bql_unlock();
826             return true;
827         }
828 #if !defined(CONFIG_USER_ONLY)
829         if (replay_mode == REPLAY_MODE_PLAY && !replay_has_interrupt()) {
830             /* Do nothing */
831         } else if (interrupt_request & CPU_INTERRUPT_HALT) {
832             replay_interrupt();
833             cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
834             cpu->halted = 1;
835             cpu->exception_index = EXCP_HLT;
836             bql_unlock();
837             return true;
838         }
839 #if defined(TARGET_I386)
840         else if (interrupt_request & CPU_INTERRUPT_INIT) {
841             X86CPU *x86_cpu = X86_CPU(cpu);
842             CPUArchState *env = &x86_cpu->env;
843             replay_interrupt();
844             cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0, 0);
845             do_cpu_init(x86_cpu);
846             cpu->exception_index = EXCP_HALTED;
847             bql_unlock();
848             return true;
849         }
850 #else
851         else if (interrupt_request & CPU_INTERRUPT_RESET) {
852             replay_interrupt();
853             cpu_reset(cpu);
854             bql_unlock();
855             return true;
856         }
857 #endif /* !TARGET_I386 */
858         /* The target hook has 3 exit conditions:
859            False when the interrupt isn't processed,
860            True when it is, and we should restart on a new TB,
861            and via longjmp via cpu_loop_exit.  */
862         else {
863             const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
864 
865             if (tcg_ops->cpu_exec_interrupt &&
866                 tcg_ops->cpu_exec_interrupt(cpu, interrupt_request)) {
867                 if (!tcg_ops->need_replay_interrupt ||
868                     tcg_ops->need_replay_interrupt(interrupt_request)) {
869                     replay_interrupt();
870                 }
871                 /*
872                  * After processing the interrupt, ensure an EXCP_DEBUG is
873                  * raised when single-stepping so that GDB doesn't miss the
874                  * next instruction.
875                  */
876                 if (unlikely(cpu->singlestep_enabled)) {
877                     cpu->exception_index = EXCP_DEBUG;
878                     bql_unlock();
879                     return true;
880                 }
881                 cpu->exception_index = -1;
882                 *last_tb = NULL;
883             }
884             /* The target hook may have updated the 'cpu->interrupt_request';
885              * reload the 'interrupt_request' value */
886             interrupt_request = cpu->interrupt_request;
887         }
888 #endif /* !CONFIG_USER_ONLY */
889         if (interrupt_request & CPU_INTERRUPT_EXITTB) {
890             cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
891             /* ensure that no TB jump will be modified as
892                the program flow was changed */
893             *last_tb = NULL;
894         }
895 
896         /* If we exit via cpu_loop_exit/longjmp it is reset in cpu_exec */
897         bql_unlock();
898     }
899 
900     /* Finally, check if we need to exit to the main loop.  */
901     if (unlikely(qatomic_read(&cpu->exit_request)) || icount_exit_request(cpu)) {
902         qatomic_set(&cpu->exit_request, 0);
903         if (cpu->exception_index == -1) {
904             cpu->exception_index = EXCP_INTERRUPT;
905         }
906         return true;
907     }
908 
909     return false;
910 }
911 
cpu_loop_exec_tb(CPUState * cpu,TranslationBlock * tb,vaddr pc,TranslationBlock ** last_tb,int * tb_exit)912 static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb,
913                                     vaddr pc, TranslationBlock **last_tb,
914                                     int *tb_exit)
915 {
916     trace_exec_tb(tb, pc);
917     tb = cpu_tb_exec(cpu, tb, tb_exit);
918     if (*tb_exit != TB_EXIT_REQUESTED) {
919         *last_tb = tb;
920         return;
921     }
922 
923     *last_tb = NULL;
924     if (cpu_loop_exit_requested(cpu)) {
925         /* Something asked us to stop executing chained TBs; just
926          * continue round the main loop. Whatever requested the exit
927          * will also have set something else (eg exit_request or
928          * interrupt_request) which will be handled by
929          * cpu_handle_interrupt.  cpu_handle_interrupt will also
930          * clear cpu->icount_decr.u16.high.
931          */
932         return;
933     }
934 
935     /* Instruction counter expired.  */
936     assert(icount_enabled());
937 #ifndef CONFIG_USER_ONLY
938     /* Ensure global icount has gone forward */
939     icount_update(cpu);
940     /* Refill decrementer and continue execution.  */
941     int32_t insns_left = MIN(0xffff, cpu->icount_budget);
942     cpu->neg.icount_decr.u16.low = insns_left;
943     cpu->icount_extra = cpu->icount_budget - insns_left;
944 
945     /*
946      * If the next tb has more instructions than we have left to
947      * execute we need to ensure we find/generate a TB with exactly
948      * insns_left instructions in it.
949      */
950     if (insns_left > 0 && insns_left < tb->icount)  {
951         assert(insns_left <= CF_COUNT_MASK);
952         assert(cpu->icount_extra == 0);
953         cpu->cflags_next_tb = (tb->cflags & ~CF_COUNT_MASK) | insns_left;
954     }
955 #endif
956 }
957 
958 /* main execution loop */
959 
960 static int __attribute__((noinline))
cpu_exec_loop(CPUState * cpu,SyncClocks * sc)961 cpu_exec_loop(CPUState *cpu, SyncClocks *sc)
962 {
963     int ret;
964 
965     /* if an exception is pending, we execute it here */
966     while (!cpu_handle_exception(cpu, &ret)) {
967         TranslationBlock *last_tb = NULL;
968         int tb_exit = 0;
969 
970         while (!cpu_handle_interrupt(cpu, &last_tb)) {
971             TranslationBlock *tb;
972             vaddr pc;
973             uint64_t cs_base;
974             uint32_t flags, cflags;
975 
976             cpu_get_tb_cpu_state(cpu_env(cpu), &pc, &cs_base, &flags);
977 
978             /*
979              * When requested, use an exact setting for cflags for the next
980              * execution.  This is used for icount, precise smc, and stop-
981              * after-access watchpoints.  Since this request should never
982              * have CF_INVALID set, -1 is a convenient invalid value that
983              * does not require tcg headers for cpu_common_reset.
984              */
985             cflags = cpu->cflags_next_tb;
986             if (cflags == -1) {
987                 cflags = curr_cflags(cpu);
988             } else {
989                 cpu->cflags_next_tb = -1;
990             }
991 
992             if (check_for_breakpoints(cpu, pc, &cflags)) {
993                 break;
994             }
995 
996             tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
997             if (tb == NULL) {
998                 CPUJumpCache *jc;
999                 uint32_t h;
1000 
1001                 mmap_lock();
1002                 tb = tb_gen_code(cpu, pc, cs_base, flags, cflags);
1003                 mmap_unlock();
1004 
1005                 /*
1006                  * We add the TB in the virtual pc hash table
1007                  * for the fast lookup
1008                  */
1009                 h = tb_jmp_cache_hash_func(pc);
1010                 jc = cpu->tb_jmp_cache;
1011                 jc->array[h].pc = pc;
1012                 qatomic_set(&jc->array[h].tb, tb);
1013             }
1014 
1015 #ifndef CONFIG_USER_ONLY
1016             /*
1017              * We don't take care of direct jumps when address mapping
1018              * changes in system emulation.  So it's not safe to make a
1019              * direct jump to a TB spanning two pages because the mapping
1020              * for the second page can change.
1021              */
1022             if (tb_page_addr1(tb) != -1) {
1023                 last_tb = NULL;
1024             }
1025 #endif
1026             /* See if we can patch the calling TB. */
1027             if (last_tb) {
1028                 tb_add_jump(last_tb, tb_exit, tb);
1029             }
1030 
1031             cpu_loop_exec_tb(cpu, tb, pc, &last_tb, &tb_exit);
1032 
1033             /* Try to align the host and virtual clocks
1034                if the guest is in advance */
1035             align_clocks(sc, cpu);
1036         }
1037     }
1038     return ret;
1039 }
1040 
cpu_exec_setjmp(CPUState * cpu,SyncClocks * sc)1041 static int cpu_exec_setjmp(CPUState *cpu, SyncClocks *sc)
1042 {
1043     /* Prepare setjmp context for exception handling. */
1044     if (unlikely(sigsetjmp(cpu->jmp_env, 0) != 0)) {
1045         cpu_exec_longjmp_cleanup(cpu);
1046     }
1047 
1048     return cpu_exec_loop(cpu, sc);
1049 }
1050 
cpu_exec(CPUState * cpu)1051 int cpu_exec(CPUState *cpu)
1052 {
1053     int ret;
1054     SyncClocks sc = { 0 };
1055 
1056     /* replay_interrupt may need current_cpu */
1057     current_cpu = cpu;
1058 
1059     if (cpu_handle_halt(cpu)) {
1060         return EXCP_HALTED;
1061     }
1062 
1063     RCU_READ_LOCK_GUARD();
1064     cpu_exec_enter(cpu);
1065 
1066     /*
1067      * Calculate difference between guest clock and host clock.
1068      * This delay includes the delay of the last cycle, so
1069      * what we have to do is sleep until it is 0. As for the
1070      * advance/delay we gain here, we try to fix it next time.
1071      */
1072     init_delay_params(&sc, cpu);
1073 
1074     ret = cpu_exec_setjmp(cpu, &sc);
1075 
1076     cpu_exec_exit(cpu);
1077     return ret;
1078 }
1079 
tcg_exec_realizefn(CPUState * cpu,Error ** errp)1080 bool tcg_exec_realizefn(CPUState *cpu, Error **errp)
1081 {
1082     static bool tcg_target_initialized;
1083 
1084     if (!tcg_target_initialized) {
1085         cpu->cc->tcg_ops->initialize();
1086         tcg_target_initialized = true;
1087     }
1088 
1089     cpu->tb_jmp_cache = g_new0(CPUJumpCache, 1);
1090     tlb_init(cpu);
1091 #ifndef CONFIG_USER_ONLY
1092     tcg_iommu_init_notifier_list(cpu);
1093 #endif /* !CONFIG_USER_ONLY */
1094     /* qemu_plugin_vcpu_init_hook delayed until cpu_index assigned. */
1095 
1096     return true;
1097 }
1098 
1099 /* undo the initializations in reverse order */
tcg_exec_unrealizefn(CPUState * cpu)1100 void tcg_exec_unrealizefn(CPUState *cpu)
1101 {
1102 #ifndef CONFIG_USER_ONLY
1103     tcg_iommu_free_notifier_list(cpu);
1104 #endif /* !CONFIG_USER_ONLY */
1105 
1106     tlb_destroy(cpu);
1107     g_free_rcu(cpu->tb_jmp_cache, rcu);
1108 }
1109