1 /*
2 * User emulator execution
3 *
4 * Copyright (c) 2003-2005 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "qemu/osdep.h"
20 #include "hw/core/tcg-cpu-ops.h"
21 #include "disas/disas.h"
22 #include "exec/exec-all.h"
23 #include "tcg/tcg.h"
24 #include "qemu/bitops.h"
25 #include "qemu/rcu.h"
26 #include "exec/cpu_ldst.h"
27 #include "qemu/main-loop.h"
28 #include "exec/translate-all.h"
29 #include "exec/page-protection.h"
30 #include "exec/helper-proto.h"
31 #include "qemu/atomic128.h"
32 #include "trace/trace-root.h"
33 #include "tcg/tcg-ldst.h"
34 #include "internal-common.h"
35 #include "internal-target.h"
36 #include "user-retaddr.h"
37
38 __thread uintptr_t helper_retaddr;
39
40 //#define DEBUG_SIGNAL
41
cpu_interrupt(CPUState * cpu,int mask)42 void cpu_interrupt(CPUState *cpu, int mask)
43 {
44 g_assert(bql_locked());
45 cpu->interrupt_request |= mask;
46 qatomic_set(&cpu->neg.icount_decr.u16.high, -1);
47 }
48
49 /*
50 * Adjust the pc to pass to cpu_restore_state; return the memop type.
51 */
adjust_signal_pc(uintptr_t * pc,bool is_write)52 MMUAccessType adjust_signal_pc(uintptr_t *pc, bool is_write)
53 {
54 switch (helper_retaddr) {
55 default:
56 /*
57 * Fault during host memory operation within a helper function.
58 * The helper's host return address, saved here, gives us a
59 * pointer into the generated code that will unwind to the
60 * correct guest pc.
61 */
62 *pc = helper_retaddr;
63 break;
64
65 case 0:
66 /*
67 * Fault during host memory operation within generated code.
68 * (Or, a unrelated bug within qemu, but we can't tell from here).
69 *
70 * We take the host pc from the signal frame. However, we cannot
71 * use that value directly. Within cpu_restore_state_from_tb, we
72 * assume PC comes from GETPC(), as used by the helper functions,
73 * so we adjust the address by -GETPC_ADJ to form an address that
74 * is within the call insn, so that the address does not accidentally
75 * match the beginning of the next guest insn. However, when the
76 * pc comes from the signal frame it points to the actual faulting
77 * host memory insn and not the return from a call insn.
78 *
79 * Therefore, adjust to compensate for what will be done later
80 * by cpu_restore_state_from_tb.
81 */
82 *pc += GETPC_ADJ;
83 break;
84
85 case 1:
86 /*
87 * Fault during host read for translation, or loosely, "execution".
88 *
89 * The guest pc is already pointing to the start of the TB for which
90 * code is being generated. If the guest translator manages the
91 * page crossings correctly, this is exactly the correct address
92 * (and if the translator doesn't handle page boundaries correctly
93 * there's little we can do about that here). Therefore, do not
94 * trigger the unwinder.
95 */
96 *pc = 0;
97 return MMU_INST_FETCH;
98 }
99
100 return is_write ? MMU_DATA_STORE : MMU_DATA_LOAD;
101 }
102
103 /**
104 * handle_sigsegv_accerr_write:
105 * @cpu: the cpu context
106 * @old_set: the sigset_t from the signal ucontext_t
107 * @host_pc: the host pc, adjusted for the signal
108 * @guest_addr: the guest address of the fault
109 *
110 * Return true if the write fault has been handled, and should be re-tried.
111 *
112 * Note that it is important that we don't call page_unprotect() unless
113 * this is really a "write to nonwritable page" fault, because
114 * page_unprotect() assumes that if it is called for an access to
115 * a page that's writable this means we had two threads racing and
116 * another thread got there first and already made the page writable;
117 * so we will retry the access. If we were to call page_unprotect()
118 * for some other kind of fault that should really be passed to the
119 * guest, we'd end up in an infinite loop of retrying the faulting access.
120 */
handle_sigsegv_accerr_write(CPUState * cpu,sigset_t * old_set,uintptr_t host_pc,abi_ptr guest_addr)121 bool handle_sigsegv_accerr_write(CPUState *cpu, sigset_t *old_set,
122 uintptr_t host_pc, abi_ptr guest_addr)
123 {
124 switch (page_unprotect(guest_addr, host_pc)) {
125 case 0:
126 /*
127 * Fault not caused by a page marked unwritable to protect
128 * cached translations, must be the guest binary's problem.
129 */
130 return false;
131 case 1:
132 /*
133 * Fault caused by protection of cached translation; TBs
134 * invalidated, so resume execution.
135 */
136 return true;
137 case 2:
138 /*
139 * Fault caused by protection of cached translation, and the
140 * currently executing TB was modified and must be exited immediately.
141 */
142 sigprocmask(SIG_SETMASK, old_set, NULL);
143 cpu_loop_exit_noexc(cpu);
144 /* NORETURN */
145 default:
146 g_assert_not_reached();
147 }
148 }
149
150 typedef struct PageFlagsNode {
151 struct rcu_head rcu;
152 IntervalTreeNode itree;
153 int flags;
154 } PageFlagsNode;
155
156 static IntervalTreeRoot pageflags_root;
157
pageflags_find(target_ulong start,target_ulong last)158 static PageFlagsNode *pageflags_find(target_ulong start, target_ulong last)
159 {
160 IntervalTreeNode *n;
161
162 n = interval_tree_iter_first(&pageflags_root, start, last);
163 return n ? container_of(n, PageFlagsNode, itree) : NULL;
164 }
165
pageflags_next(PageFlagsNode * p,target_ulong start,target_ulong last)166 static PageFlagsNode *pageflags_next(PageFlagsNode *p, target_ulong start,
167 target_ulong last)
168 {
169 IntervalTreeNode *n;
170
171 n = interval_tree_iter_next(&p->itree, start, last);
172 return n ? container_of(n, PageFlagsNode, itree) : NULL;
173 }
174
walk_memory_regions(void * priv,walk_memory_regions_fn fn)175 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
176 {
177 IntervalTreeNode *n;
178 int rc = 0;
179
180 mmap_lock();
181 for (n = interval_tree_iter_first(&pageflags_root, 0, -1);
182 n != NULL;
183 n = interval_tree_iter_next(n, 0, -1)) {
184 PageFlagsNode *p = container_of(n, PageFlagsNode, itree);
185
186 rc = fn(priv, n->start, n->last + 1, p->flags);
187 if (rc != 0) {
188 break;
189 }
190 }
191 mmap_unlock();
192
193 return rc;
194 }
195
dump_region(void * priv,target_ulong start,target_ulong end,unsigned long prot)196 static int dump_region(void *priv, target_ulong start,
197 target_ulong end, unsigned long prot)
198 {
199 FILE *f = (FILE *)priv;
200
201 fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx" "TARGET_FMT_lx" %c%c%c\n",
202 start, end, end - start,
203 ((prot & PAGE_READ) ? 'r' : '-'),
204 ((prot & PAGE_WRITE) ? 'w' : '-'),
205 ((prot & PAGE_EXEC) ? 'x' : '-'));
206 return 0;
207 }
208
209 /* dump memory mappings */
page_dump(FILE * f)210 void page_dump(FILE *f)
211 {
212 const int length = sizeof(target_ulong) * 2;
213
214 fprintf(f, "%-*s %-*s %-*s %s\n",
215 length, "start", length, "end", length, "size", "prot");
216 walk_memory_regions(f, dump_region);
217 }
218
page_get_flags(target_ulong address)219 int page_get_flags(target_ulong address)
220 {
221 PageFlagsNode *p = pageflags_find(address, address);
222
223 /*
224 * See util/interval-tree.c re lockless lookups: no false positives but
225 * there are false negatives. If we find nothing, retry with the mmap
226 * lock acquired.
227 */
228 if (p) {
229 return p->flags;
230 }
231 if (have_mmap_lock()) {
232 return 0;
233 }
234
235 mmap_lock();
236 p = pageflags_find(address, address);
237 mmap_unlock();
238 return p ? p->flags : 0;
239 }
240
241 /* A subroutine of page_set_flags: insert a new node for [start,last]. */
pageflags_create(target_ulong start,target_ulong last,int flags)242 static void pageflags_create(target_ulong start, target_ulong last, int flags)
243 {
244 PageFlagsNode *p = g_new(PageFlagsNode, 1);
245
246 p->itree.start = start;
247 p->itree.last = last;
248 p->flags = flags;
249 interval_tree_insert(&p->itree, &pageflags_root);
250 }
251
252 /* A subroutine of page_set_flags: remove everything in [start,last]. */
pageflags_unset(target_ulong start,target_ulong last)253 static bool pageflags_unset(target_ulong start, target_ulong last)
254 {
255 bool inval_tb = false;
256
257 while (true) {
258 PageFlagsNode *p = pageflags_find(start, last);
259 target_ulong p_last;
260
261 if (!p) {
262 break;
263 }
264
265 if (p->flags & PAGE_EXEC) {
266 inval_tb = true;
267 }
268
269 interval_tree_remove(&p->itree, &pageflags_root);
270 p_last = p->itree.last;
271
272 if (p->itree.start < start) {
273 /* Truncate the node from the end, or split out the middle. */
274 p->itree.last = start - 1;
275 interval_tree_insert(&p->itree, &pageflags_root);
276 if (last < p_last) {
277 pageflags_create(last + 1, p_last, p->flags);
278 break;
279 }
280 } else if (p_last <= last) {
281 /* Range completely covers node -- remove it. */
282 g_free_rcu(p, rcu);
283 } else {
284 /* Truncate the node from the start. */
285 p->itree.start = last + 1;
286 interval_tree_insert(&p->itree, &pageflags_root);
287 break;
288 }
289 }
290
291 return inval_tb;
292 }
293
294 /*
295 * A subroutine of page_set_flags: nothing overlaps [start,last],
296 * but check adjacent mappings and maybe merge into a single range.
297 */
pageflags_create_merge(target_ulong start,target_ulong last,int flags)298 static void pageflags_create_merge(target_ulong start, target_ulong last,
299 int flags)
300 {
301 PageFlagsNode *next = NULL, *prev = NULL;
302
303 if (start > 0) {
304 prev = pageflags_find(start - 1, start - 1);
305 if (prev) {
306 if (prev->flags == flags) {
307 interval_tree_remove(&prev->itree, &pageflags_root);
308 } else {
309 prev = NULL;
310 }
311 }
312 }
313 if (last + 1 != 0) {
314 next = pageflags_find(last + 1, last + 1);
315 if (next) {
316 if (next->flags == flags) {
317 interval_tree_remove(&next->itree, &pageflags_root);
318 } else {
319 next = NULL;
320 }
321 }
322 }
323
324 if (prev) {
325 if (next) {
326 prev->itree.last = next->itree.last;
327 g_free_rcu(next, rcu);
328 } else {
329 prev->itree.last = last;
330 }
331 interval_tree_insert(&prev->itree, &pageflags_root);
332 } else if (next) {
333 next->itree.start = start;
334 interval_tree_insert(&next->itree, &pageflags_root);
335 } else {
336 pageflags_create(start, last, flags);
337 }
338 }
339
340 /*
341 * Allow the target to decide if PAGE_TARGET_[12] may be reset.
342 * By default, they are not kept.
343 */
344 #ifndef PAGE_TARGET_STICKY
345 #define PAGE_TARGET_STICKY 0
346 #endif
347 #define PAGE_STICKY (PAGE_ANON | PAGE_PASSTHROUGH | PAGE_TARGET_STICKY)
348
349 /* A subroutine of page_set_flags: add flags to [start,last]. */
pageflags_set_clear(target_ulong start,target_ulong last,int set_flags,int clear_flags)350 static bool pageflags_set_clear(target_ulong start, target_ulong last,
351 int set_flags, int clear_flags)
352 {
353 PageFlagsNode *p;
354 target_ulong p_start, p_last;
355 int p_flags, merge_flags;
356 bool inval_tb = false;
357
358 restart:
359 p = pageflags_find(start, last);
360 if (!p) {
361 if (set_flags) {
362 pageflags_create_merge(start, last, set_flags);
363 }
364 goto done;
365 }
366
367 p_start = p->itree.start;
368 p_last = p->itree.last;
369 p_flags = p->flags;
370 /* Using mprotect on a page does not change sticky bits. */
371 merge_flags = (p_flags & ~clear_flags) | set_flags;
372
373 /*
374 * Need to flush if an overlapping executable region
375 * removes exec, or adds write.
376 */
377 if ((p_flags & PAGE_EXEC)
378 && (!(merge_flags & PAGE_EXEC)
379 || (merge_flags & ~p_flags & PAGE_WRITE))) {
380 inval_tb = true;
381 }
382
383 /*
384 * If there is an exact range match, update and return without
385 * attempting to merge with adjacent regions.
386 */
387 if (start == p_start && last == p_last) {
388 if (merge_flags) {
389 p->flags = merge_flags;
390 } else {
391 interval_tree_remove(&p->itree, &pageflags_root);
392 g_free_rcu(p, rcu);
393 }
394 goto done;
395 }
396
397 /*
398 * If sticky bits affect the original mapping, then we must be more
399 * careful about the existing intervals and the separate flags.
400 */
401 if (set_flags != merge_flags) {
402 if (p_start < start) {
403 interval_tree_remove(&p->itree, &pageflags_root);
404 p->itree.last = start - 1;
405 interval_tree_insert(&p->itree, &pageflags_root);
406
407 if (last < p_last) {
408 if (merge_flags) {
409 pageflags_create(start, last, merge_flags);
410 }
411 pageflags_create(last + 1, p_last, p_flags);
412 } else {
413 if (merge_flags) {
414 pageflags_create(start, p_last, merge_flags);
415 }
416 if (p_last < last) {
417 start = p_last + 1;
418 goto restart;
419 }
420 }
421 } else {
422 if (start < p_start && set_flags) {
423 pageflags_create(start, p_start - 1, set_flags);
424 }
425 if (last < p_last) {
426 interval_tree_remove(&p->itree, &pageflags_root);
427 p->itree.start = last + 1;
428 interval_tree_insert(&p->itree, &pageflags_root);
429 if (merge_flags) {
430 pageflags_create(start, last, merge_flags);
431 }
432 } else {
433 if (merge_flags) {
434 p->flags = merge_flags;
435 } else {
436 interval_tree_remove(&p->itree, &pageflags_root);
437 g_free_rcu(p, rcu);
438 }
439 if (p_last < last) {
440 start = p_last + 1;
441 goto restart;
442 }
443 }
444 }
445 goto done;
446 }
447
448 /* If flags are not changing for this range, incorporate it. */
449 if (set_flags == p_flags) {
450 if (start < p_start) {
451 interval_tree_remove(&p->itree, &pageflags_root);
452 p->itree.start = start;
453 interval_tree_insert(&p->itree, &pageflags_root);
454 }
455 if (p_last < last) {
456 start = p_last + 1;
457 goto restart;
458 }
459 goto done;
460 }
461
462 /* Maybe split out head and/or tail ranges with the original flags. */
463 interval_tree_remove(&p->itree, &pageflags_root);
464 if (p_start < start) {
465 p->itree.last = start - 1;
466 interval_tree_insert(&p->itree, &pageflags_root);
467
468 if (p_last < last) {
469 goto restart;
470 }
471 if (last < p_last) {
472 pageflags_create(last + 1, p_last, p_flags);
473 }
474 } else if (last < p_last) {
475 p->itree.start = last + 1;
476 interval_tree_insert(&p->itree, &pageflags_root);
477 } else {
478 g_free_rcu(p, rcu);
479 goto restart;
480 }
481 if (set_flags) {
482 pageflags_create(start, last, set_flags);
483 }
484
485 done:
486 return inval_tb;
487 }
488
489 /*
490 * Modify the flags of a page and invalidate the code if necessary.
491 * The flag PAGE_WRITE_ORG is positioned automatically depending
492 * on PAGE_WRITE. The mmap_lock should already be held.
493 */
page_set_flags(target_ulong start,target_ulong last,int flags)494 void page_set_flags(target_ulong start, target_ulong last, int flags)
495 {
496 bool reset = false;
497 bool inval_tb = false;
498
499 /* This function should never be called with addresses outside the
500 guest address space. If this assert fires, it probably indicates
501 a missing call to h2g_valid. */
502 assert(start <= last);
503 assert(last <= GUEST_ADDR_MAX);
504 /* Only set PAGE_ANON with new mappings. */
505 assert(!(flags & PAGE_ANON) || (flags & PAGE_RESET));
506 assert_memory_lock();
507
508 start &= TARGET_PAGE_MASK;
509 last |= ~TARGET_PAGE_MASK;
510
511 if (!(flags & PAGE_VALID)) {
512 flags = 0;
513 } else {
514 reset = flags & PAGE_RESET;
515 flags &= ~PAGE_RESET;
516 if (flags & PAGE_WRITE) {
517 flags |= PAGE_WRITE_ORG;
518 }
519 }
520
521 if (!flags || reset) {
522 page_reset_target_data(start, last);
523 inval_tb |= pageflags_unset(start, last);
524 }
525 if (flags) {
526 inval_tb |= pageflags_set_clear(start, last, flags,
527 ~(reset ? 0 : PAGE_STICKY));
528 }
529 if (inval_tb) {
530 tb_invalidate_phys_range(start, last);
531 }
532 }
533
page_check_range(target_ulong start,target_ulong len,int flags)534 bool page_check_range(target_ulong start, target_ulong len, int flags)
535 {
536 target_ulong last;
537 int locked; /* tri-state: =0: unlocked, +1: global, -1: local */
538 bool ret;
539
540 if (len == 0) {
541 return true; /* trivial length */
542 }
543
544 last = start + len - 1;
545 if (last < start) {
546 return false; /* wrap around */
547 }
548
549 locked = have_mmap_lock();
550 while (true) {
551 PageFlagsNode *p = pageflags_find(start, last);
552 int missing;
553
554 if (!p) {
555 if (!locked) {
556 /*
557 * Lockless lookups have false negatives.
558 * Retry with the lock held.
559 */
560 mmap_lock();
561 locked = -1;
562 p = pageflags_find(start, last);
563 }
564 if (!p) {
565 ret = false; /* entire region invalid */
566 break;
567 }
568 }
569 if (start < p->itree.start) {
570 ret = false; /* initial bytes invalid */
571 break;
572 }
573
574 missing = flags & ~p->flags;
575 if (missing & ~PAGE_WRITE) {
576 ret = false; /* page doesn't match */
577 break;
578 }
579 if (missing & PAGE_WRITE) {
580 if (!(p->flags & PAGE_WRITE_ORG)) {
581 ret = false; /* page not writable */
582 break;
583 }
584 /* Asking about writable, but has been protected: undo. */
585 if (!page_unprotect(start, 0)) {
586 ret = false;
587 break;
588 }
589 /* TODO: page_unprotect should take a range, not a single page. */
590 if (last - start < TARGET_PAGE_SIZE) {
591 ret = true; /* ok */
592 break;
593 }
594 start += TARGET_PAGE_SIZE;
595 continue;
596 }
597
598 if (last <= p->itree.last) {
599 ret = true; /* ok */
600 break;
601 }
602 start = p->itree.last + 1;
603 }
604
605 /* Release the lock if acquired locally. */
606 if (locked < 0) {
607 mmap_unlock();
608 }
609 return ret;
610 }
611
page_check_range_empty(target_ulong start,target_ulong last)612 bool page_check_range_empty(target_ulong start, target_ulong last)
613 {
614 assert(last >= start);
615 assert_memory_lock();
616 return pageflags_find(start, last) == NULL;
617 }
618
page_find_range_empty(target_ulong min,target_ulong max,target_ulong len,target_ulong align)619 target_ulong page_find_range_empty(target_ulong min, target_ulong max,
620 target_ulong len, target_ulong align)
621 {
622 target_ulong len_m1, align_m1;
623
624 assert(min <= max);
625 assert(max <= GUEST_ADDR_MAX);
626 assert(len != 0);
627 assert(is_power_of_2(align));
628 assert_memory_lock();
629
630 len_m1 = len - 1;
631 align_m1 = align - 1;
632
633 /* Iteratively narrow the search region. */
634 while (1) {
635 PageFlagsNode *p;
636
637 /* Align min and double-check there's enough space remaining. */
638 min = (min + align_m1) & ~align_m1;
639 if (min > max) {
640 return -1;
641 }
642 if (len_m1 > max - min) {
643 return -1;
644 }
645
646 p = pageflags_find(min, min + len_m1);
647 if (p == NULL) {
648 /* Found! */
649 return min;
650 }
651 if (max <= p->itree.last) {
652 /* Existing allocation fills the remainder of the search region. */
653 return -1;
654 }
655 /* Skip across existing allocation. */
656 min = p->itree.last + 1;
657 }
658 }
659
page_protect(tb_page_addr_t address)660 void page_protect(tb_page_addr_t address)
661 {
662 PageFlagsNode *p;
663 target_ulong start, last;
664 int host_page_size = qemu_real_host_page_size();
665 int prot;
666
667 assert_memory_lock();
668
669 if (host_page_size <= TARGET_PAGE_SIZE) {
670 start = address & TARGET_PAGE_MASK;
671 last = start + TARGET_PAGE_SIZE - 1;
672 } else {
673 start = address & -host_page_size;
674 last = start + host_page_size - 1;
675 }
676
677 p = pageflags_find(start, last);
678 if (!p) {
679 return;
680 }
681 prot = p->flags;
682
683 if (unlikely(p->itree.last < last)) {
684 /* More than one protection region covers the one host page. */
685 assert(TARGET_PAGE_SIZE < host_page_size);
686 while ((p = pageflags_next(p, start, last)) != NULL) {
687 prot |= p->flags;
688 }
689 }
690
691 if (prot & PAGE_WRITE) {
692 pageflags_set_clear(start, last, 0, PAGE_WRITE);
693 mprotect(g2h_untagged(start), last - start + 1,
694 prot & (PAGE_READ | PAGE_EXEC) ? PROT_READ : PROT_NONE);
695 }
696 }
697
698 /*
699 * Called from signal handler: invalidate the code and unprotect the
700 * page. Return 0 if the fault was not handled, 1 if it was handled,
701 * and 2 if it was handled but the caller must cause the TB to be
702 * immediately exited. (We can only return 2 if the 'pc' argument is
703 * non-zero.)
704 */
page_unprotect(target_ulong address,uintptr_t pc)705 int page_unprotect(target_ulong address, uintptr_t pc)
706 {
707 PageFlagsNode *p;
708 bool current_tb_invalidated;
709
710 /*
711 * Technically this isn't safe inside a signal handler. However we
712 * know this only ever happens in a synchronous SEGV handler, so in
713 * practice it seems to be ok.
714 */
715 mmap_lock();
716
717 p = pageflags_find(address, address);
718
719 /* If this address was not really writable, nothing to do. */
720 if (!p || !(p->flags & PAGE_WRITE_ORG)) {
721 mmap_unlock();
722 return 0;
723 }
724
725 current_tb_invalidated = false;
726 if (p->flags & PAGE_WRITE) {
727 /*
728 * If the page is actually marked WRITE then assume this is because
729 * this thread raced with another one which got here first and
730 * set the page to PAGE_WRITE and did the TB invalidate for us.
731 */
732 #ifdef TARGET_HAS_PRECISE_SMC
733 TranslationBlock *current_tb = tcg_tb_lookup(pc);
734 if (current_tb) {
735 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID;
736 }
737 #endif
738 } else {
739 int host_page_size = qemu_real_host_page_size();
740 target_ulong start, len, i;
741 int prot;
742
743 if (host_page_size <= TARGET_PAGE_SIZE) {
744 start = address & TARGET_PAGE_MASK;
745 len = TARGET_PAGE_SIZE;
746 prot = p->flags | PAGE_WRITE;
747 pageflags_set_clear(start, start + len - 1, PAGE_WRITE, 0);
748 current_tb_invalidated = tb_invalidate_phys_page_unwind(start, pc);
749 } else {
750 start = address & -host_page_size;
751 len = host_page_size;
752 prot = 0;
753
754 for (i = 0; i < len; i += TARGET_PAGE_SIZE) {
755 target_ulong addr = start + i;
756
757 p = pageflags_find(addr, addr);
758 if (p) {
759 prot |= p->flags;
760 if (p->flags & PAGE_WRITE_ORG) {
761 prot |= PAGE_WRITE;
762 pageflags_set_clear(addr, addr + TARGET_PAGE_SIZE - 1,
763 PAGE_WRITE, 0);
764 }
765 }
766 /*
767 * Since the content will be modified, we must invalidate
768 * the corresponding translated code.
769 */
770 current_tb_invalidated |=
771 tb_invalidate_phys_page_unwind(addr, pc);
772 }
773 }
774 if (prot & PAGE_EXEC) {
775 prot = (prot & ~PAGE_EXEC) | PAGE_READ;
776 }
777 mprotect((void *)g2h_untagged(start), len, prot & PAGE_RWX);
778 }
779 mmap_unlock();
780
781 /* If current TB was invalidated return to main loop */
782 return current_tb_invalidated ? 2 : 1;
783 }
784
probe_access_internal(CPUArchState * env,vaddr addr,int fault_size,MMUAccessType access_type,bool nonfault,uintptr_t ra)785 static int probe_access_internal(CPUArchState *env, vaddr addr,
786 int fault_size, MMUAccessType access_type,
787 bool nonfault, uintptr_t ra)
788 {
789 int acc_flag;
790 bool maperr;
791
792 switch (access_type) {
793 case MMU_DATA_STORE:
794 acc_flag = PAGE_WRITE_ORG;
795 break;
796 case MMU_DATA_LOAD:
797 acc_flag = PAGE_READ;
798 break;
799 case MMU_INST_FETCH:
800 acc_flag = PAGE_EXEC;
801 break;
802 default:
803 g_assert_not_reached();
804 }
805
806 if (guest_addr_valid_untagged(addr)) {
807 int page_flags = page_get_flags(addr);
808 if (page_flags & acc_flag) {
809 if ((acc_flag == PAGE_READ || acc_flag == PAGE_WRITE)
810 && cpu_plugin_mem_cbs_enabled(env_cpu(env))) {
811 return TLB_MMIO;
812 }
813 return 0; /* success */
814 }
815 maperr = !(page_flags & PAGE_VALID);
816 } else {
817 maperr = true;
818 }
819
820 if (nonfault) {
821 return TLB_INVALID_MASK;
822 }
823
824 cpu_loop_exit_sigsegv(env_cpu(env), addr, access_type, maperr, ra);
825 }
826
probe_access_flags(CPUArchState * env,vaddr addr,int size,MMUAccessType access_type,int mmu_idx,bool nonfault,void ** phost,uintptr_t ra)827 int probe_access_flags(CPUArchState *env, vaddr addr, int size,
828 MMUAccessType access_type, int mmu_idx,
829 bool nonfault, void **phost, uintptr_t ra)
830 {
831 int flags;
832
833 g_assert(-(addr | TARGET_PAGE_MASK) >= size);
834 flags = probe_access_internal(env, addr, size, access_type, nonfault, ra);
835 *phost = (flags & TLB_INVALID_MASK) ? NULL : g2h(env_cpu(env), addr);
836 return flags;
837 }
838
probe_access(CPUArchState * env,vaddr addr,int size,MMUAccessType access_type,int mmu_idx,uintptr_t ra)839 void *probe_access(CPUArchState *env, vaddr addr, int size,
840 MMUAccessType access_type, int mmu_idx, uintptr_t ra)
841 {
842 int flags;
843
844 g_assert(-(addr | TARGET_PAGE_MASK) >= size);
845 flags = probe_access_internal(env, addr, size, access_type, false, ra);
846 g_assert((flags & ~TLB_MMIO) == 0);
847
848 return size ? g2h(env_cpu(env), addr) : NULL;
849 }
850
get_page_addr_code_hostp(CPUArchState * env,vaddr addr,void ** hostp)851 tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr,
852 void **hostp)
853 {
854 int flags;
855
856 flags = probe_access_internal(env, addr, 1, MMU_INST_FETCH, false, 0);
857 g_assert(flags == 0);
858
859 if (hostp) {
860 *hostp = g2h_untagged(addr);
861 }
862 return addr;
863 }
864
865 #ifdef TARGET_PAGE_DATA_SIZE
866 /*
867 * Allocate chunks of target data together. For the only current user,
868 * if we allocate one hunk per page, we have overhead of 40/128 or 40%.
869 * Therefore, allocate memory for 64 pages at a time for overhead < 1%.
870 */
871 #define TPD_PAGES 64
872 #define TBD_MASK (TARGET_PAGE_MASK * TPD_PAGES)
873
874 typedef struct TargetPageDataNode {
875 struct rcu_head rcu;
876 IntervalTreeNode itree;
877 char data[] __attribute__((aligned));
878 } TargetPageDataNode;
879
880 static IntervalTreeRoot targetdata_root;
881
page_reset_target_data(target_ulong start,target_ulong last)882 void page_reset_target_data(target_ulong start, target_ulong last)
883 {
884 IntervalTreeNode *n, *next;
885
886 assert_memory_lock();
887
888 start &= TARGET_PAGE_MASK;
889 last |= ~TARGET_PAGE_MASK;
890
891 for (n = interval_tree_iter_first(&targetdata_root, start, last),
892 next = n ? interval_tree_iter_next(n, start, last) : NULL;
893 n != NULL;
894 n = next,
895 next = next ? interval_tree_iter_next(n, start, last) : NULL) {
896 target_ulong n_start, n_last, p_ofs, p_len;
897 TargetPageDataNode *t = container_of(n, TargetPageDataNode, itree);
898
899 if (n->start >= start && n->last <= last) {
900 interval_tree_remove(n, &targetdata_root);
901 g_free_rcu(t, rcu);
902 continue;
903 }
904
905 if (n->start < start) {
906 n_start = start;
907 p_ofs = (start - n->start) >> TARGET_PAGE_BITS;
908 } else {
909 n_start = n->start;
910 p_ofs = 0;
911 }
912 n_last = MIN(last, n->last);
913 p_len = (n_last + 1 - n_start) >> TARGET_PAGE_BITS;
914
915 memset(t->data + p_ofs * TARGET_PAGE_DATA_SIZE, 0,
916 p_len * TARGET_PAGE_DATA_SIZE);
917 }
918 }
919
page_get_target_data(target_ulong address)920 void *page_get_target_data(target_ulong address)
921 {
922 IntervalTreeNode *n;
923 TargetPageDataNode *t;
924 target_ulong page, region, p_ofs;
925
926 page = address & TARGET_PAGE_MASK;
927 region = address & TBD_MASK;
928
929 n = interval_tree_iter_first(&targetdata_root, page, page);
930 if (!n) {
931 /*
932 * See util/interval-tree.c re lockless lookups: no false positives
933 * but there are false negatives. If we find nothing, retry with
934 * the mmap lock acquired. We also need the lock for the
935 * allocation + insert.
936 */
937 mmap_lock();
938 n = interval_tree_iter_first(&targetdata_root, page, page);
939 if (!n) {
940 t = g_malloc0(sizeof(TargetPageDataNode)
941 + TPD_PAGES * TARGET_PAGE_DATA_SIZE);
942 n = &t->itree;
943 n->start = region;
944 n->last = region | ~TBD_MASK;
945 interval_tree_insert(n, &targetdata_root);
946 }
947 mmap_unlock();
948 }
949
950 t = container_of(n, TargetPageDataNode, itree);
951 p_ofs = (page - region) >> TARGET_PAGE_BITS;
952 return t->data + p_ofs * TARGET_PAGE_DATA_SIZE;
953 }
954 #else
page_reset_target_data(target_ulong start,target_ulong last)955 void page_reset_target_data(target_ulong start, target_ulong last) { }
956 #endif /* TARGET_PAGE_DATA_SIZE */
957
958 /* The system-mode versions of these helpers are in cputlb.c. */
959
cpu_mmu_lookup(CPUState * cpu,vaddr addr,MemOp mop,uintptr_t ra,MMUAccessType type)960 static void *cpu_mmu_lookup(CPUState *cpu, vaddr addr,
961 MemOp mop, uintptr_t ra, MMUAccessType type)
962 {
963 int a_bits = get_alignment_bits(mop);
964 void *ret;
965
966 /* Enforce guest required alignment. */
967 if (unlikely(addr & ((1 << a_bits) - 1))) {
968 cpu_loop_exit_sigbus(cpu, addr, type, ra);
969 }
970
971 ret = g2h(cpu, addr);
972 set_helper_retaddr(ra);
973 return ret;
974 }
975
976 #include "ldst_atomicity.c.inc"
977
do_ld1_mmu(CPUState * cpu,vaddr addr,MemOpIdx oi,uintptr_t ra,MMUAccessType access_type)978 static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
979 uintptr_t ra, MMUAccessType access_type)
980 {
981 void *haddr;
982 uint8_t ret;
983
984 cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
985 haddr = cpu_mmu_lookup(cpu, addr, get_memop(oi), ra, access_type);
986 ret = ldub_p(haddr);
987 clear_helper_retaddr();
988 return ret;
989 }
990
do_ld2_mmu(CPUState * cpu,vaddr addr,MemOpIdx oi,uintptr_t ra,MMUAccessType access_type)991 static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
992 uintptr_t ra, MMUAccessType access_type)
993 {
994 void *haddr;
995 uint16_t ret;
996 MemOp mop = get_memop(oi);
997
998 cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
999 haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type);
1000 ret = load_atom_2(cpu, ra, haddr, mop);
1001 clear_helper_retaddr();
1002
1003 if (mop & MO_BSWAP) {
1004 ret = bswap16(ret);
1005 }
1006 return ret;
1007 }
1008
do_ld4_mmu(CPUState * cpu,vaddr addr,MemOpIdx oi,uintptr_t ra,MMUAccessType access_type)1009 static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
1010 uintptr_t ra, MMUAccessType access_type)
1011 {
1012 void *haddr;
1013 uint32_t ret;
1014 MemOp mop = get_memop(oi);
1015
1016 cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
1017 haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type);
1018 ret = load_atom_4(cpu, ra, haddr, mop);
1019 clear_helper_retaddr();
1020
1021 if (mop & MO_BSWAP) {
1022 ret = bswap32(ret);
1023 }
1024 return ret;
1025 }
1026
do_ld8_mmu(CPUState * cpu,vaddr addr,MemOpIdx oi,uintptr_t ra,MMUAccessType access_type)1027 static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
1028 uintptr_t ra, MMUAccessType access_type)
1029 {
1030 void *haddr;
1031 uint64_t ret;
1032 MemOp mop = get_memop(oi);
1033
1034 cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
1035 haddr = cpu_mmu_lookup(cpu, addr, mop, ra, access_type);
1036 ret = load_atom_8(cpu, ra, haddr, mop);
1037 clear_helper_retaddr();
1038
1039 if (mop & MO_BSWAP) {
1040 ret = bswap64(ret);
1041 }
1042 return ret;
1043 }
1044
do_ld16_mmu(CPUState * cpu,abi_ptr addr,MemOpIdx oi,uintptr_t ra)1045 static Int128 do_ld16_mmu(CPUState *cpu, abi_ptr addr,
1046 MemOpIdx oi, uintptr_t ra)
1047 {
1048 void *haddr;
1049 Int128 ret;
1050 MemOp mop = get_memop(oi);
1051
1052 tcg_debug_assert((mop & MO_SIZE) == MO_128);
1053 cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
1054 haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_LOAD);
1055 ret = load_atom_16(cpu, ra, haddr, mop);
1056 clear_helper_retaddr();
1057
1058 if (mop & MO_BSWAP) {
1059 ret = bswap128(ret);
1060 }
1061 return ret;
1062 }
1063
do_st1_mmu(CPUState * cpu,vaddr addr,uint8_t val,MemOpIdx oi,uintptr_t ra)1064 static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val,
1065 MemOpIdx oi, uintptr_t ra)
1066 {
1067 void *haddr;
1068
1069 cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
1070 haddr = cpu_mmu_lookup(cpu, addr, get_memop(oi), ra, MMU_DATA_STORE);
1071 stb_p(haddr, val);
1072 clear_helper_retaddr();
1073 }
1074
do_st2_mmu(CPUState * cpu,vaddr addr,uint16_t val,MemOpIdx oi,uintptr_t ra)1075 static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val,
1076 MemOpIdx oi, uintptr_t ra)
1077 {
1078 void *haddr;
1079 MemOp mop = get_memop(oi);
1080
1081 cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
1082 haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE);
1083
1084 if (mop & MO_BSWAP) {
1085 val = bswap16(val);
1086 }
1087 store_atom_2(cpu, ra, haddr, mop, val);
1088 clear_helper_retaddr();
1089 }
1090
do_st4_mmu(CPUState * cpu,vaddr addr,uint32_t val,MemOpIdx oi,uintptr_t ra)1091 static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val,
1092 MemOpIdx oi, uintptr_t ra)
1093 {
1094 void *haddr;
1095 MemOp mop = get_memop(oi);
1096
1097 cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
1098 haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE);
1099
1100 if (mop & MO_BSWAP) {
1101 val = bswap32(val);
1102 }
1103 store_atom_4(cpu, ra, haddr, mop, val);
1104 clear_helper_retaddr();
1105 }
1106
do_st8_mmu(CPUState * cpu,vaddr addr,uint64_t val,MemOpIdx oi,uintptr_t ra)1107 static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val,
1108 MemOpIdx oi, uintptr_t ra)
1109 {
1110 void *haddr;
1111 MemOp mop = get_memop(oi);
1112
1113 cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
1114 haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE);
1115
1116 if (mop & MO_BSWAP) {
1117 val = bswap64(val);
1118 }
1119 store_atom_8(cpu, ra, haddr, mop, val);
1120 clear_helper_retaddr();
1121 }
1122
do_st16_mmu(CPUState * cpu,vaddr addr,Int128 val,MemOpIdx oi,uintptr_t ra)1123 static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
1124 MemOpIdx oi, uintptr_t ra)
1125 {
1126 void *haddr;
1127 MemOpIdx mop = get_memop(oi);
1128
1129 cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
1130 haddr = cpu_mmu_lookup(cpu, addr, mop, ra, MMU_DATA_STORE);
1131
1132 if (mop & MO_BSWAP) {
1133 val = bswap128(val);
1134 }
1135 store_atom_16(cpu, ra, haddr, mop, val);
1136 clear_helper_retaddr();
1137 }
1138
cpu_ldub_code(CPUArchState * env,abi_ptr ptr)1139 uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr ptr)
1140 {
1141 uint32_t ret;
1142
1143 set_helper_retaddr(1);
1144 ret = ldub_p(g2h_untagged(ptr));
1145 clear_helper_retaddr();
1146 return ret;
1147 }
1148
cpu_lduw_code(CPUArchState * env,abi_ptr ptr)1149 uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr ptr)
1150 {
1151 uint32_t ret;
1152
1153 set_helper_retaddr(1);
1154 ret = lduw_p(g2h_untagged(ptr));
1155 clear_helper_retaddr();
1156 return ret;
1157 }
1158
cpu_ldl_code(CPUArchState * env,abi_ptr ptr)1159 uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr ptr)
1160 {
1161 uint32_t ret;
1162
1163 set_helper_retaddr(1);
1164 ret = ldl_p(g2h_untagged(ptr));
1165 clear_helper_retaddr();
1166 return ret;
1167 }
1168
cpu_ldq_code(CPUArchState * env,abi_ptr ptr)1169 uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr ptr)
1170 {
1171 uint64_t ret;
1172
1173 set_helper_retaddr(1);
1174 ret = ldq_p(g2h_untagged(ptr));
1175 clear_helper_retaddr();
1176 return ret;
1177 }
1178
cpu_ldb_code_mmu(CPUArchState * env,abi_ptr addr,MemOpIdx oi,uintptr_t ra)1179 uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr,
1180 MemOpIdx oi, uintptr_t ra)
1181 {
1182 void *haddr;
1183 uint8_t ret;
1184
1185 haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH);
1186 ret = ldub_p(haddr);
1187 clear_helper_retaddr();
1188 return ret;
1189 }
1190
cpu_ldw_code_mmu(CPUArchState * env,abi_ptr addr,MemOpIdx oi,uintptr_t ra)1191 uint16_t cpu_ldw_code_mmu(CPUArchState *env, abi_ptr addr,
1192 MemOpIdx oi, uintptr_t ra)
1193 {
1194 void *haddr;
1195 uint16_t ret;
1196
1197 haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH);
1198 ret = lduw_p(haddr);
1199 clear_helper_retaddr();
1200 if (get_memop(oi) & MO_BSWAP) {
1201 ret = bswap16(ret);
1202 }
1203 return ret;
1204 }
1205
cpu_ldl_code_mmu(CPUArchState * env,abi_ptr addr,MemOpIdx oi,uintptr_t ra)1206 uint32_t cpu_ldl_code_mmu(CPUArchState *env, abi_ptr addr,
1207 MemOpIdx oi, uintptr_t ra)
1208 {
1209 void *haddr;
1210 uint32_t ret;
1211
1212 haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_INST_FETCH);
1213 ret = ldl_p(haddr);
1214 clear_helper_retaddr();
1215 if (get_memop(oi) & MO_BSWAP) {
1216 ret = bswap32(ret);
1217 }
1218 return ret;
1219 }
1220
cpu_ldq_code_mmu(CPUArchState * env,abi_ptr addr,MemOpIdx oi,uintptr_t ra)1221 uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr,
1222 MemOpIdx oi, uintptr_t ra)
1223 {
1224 void *haddr;
1225 uint64_t ret;
1226
1227 haddr = cpu_mmu_lookup(env_cpu(env), addr, oi, ra, MMU_DATA_LOAD);
1228 ret = ldq_p(haddr);
1229 clear_helper_retaddr();
1230 if (get_memop(oi) & MO_BSWAP) {
1231 ret = bswap64(ret);
1232 }
1233 return ret;
1234 }
1235
1236 #include "ldst_common.c.inc"
1237
1238 /*
1239 * Do not allow unaligned operations to proceed. Return the host address.
1240 */
atomic_mmu_lookup(CPUState * cpu,vaddr addr,MemOpIdx oi,int size,uintptr_t retaddr)1241 static void *atomic_mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
1242 int size, uintptr_t retaddr)
1243 {
1244 MemOp mop = get_memop(oi);
1245 int a_bits = get_alignment_bits(mop);
1246 void *ret;
1247
1248 /* Enforce guest required alignment. */
1249 if (unlikely(addr & ((1 << a_bits) - 1))) {
1250 cpu_loop_exit_sigbus(cpu, addr, MMU_DATA_STORE, retaddr);
1251 }
1252
1253 /* Enforce qemu required alignment. */
1254 if (unlikely(addr & (size - 1))) {
1255 cpu_loop_exit_atomic(cpu, retaddr);
1256 }
1257
1258 ret = g2h(cpu, addr);
1259 set_helper_retaddr(retaddr);
1260 return ret;
1261 }
1262
1263 #include "atomic_common.c.inc"
1264
1265 /*
1266 * First set of functions passes in OI and RETADDR.
1267 * This makes them callable from other helpers.
1268 */
1269
1270 #define ATOMIC_NAME(X) \
1271 glue(glue(glue(cpu_atomic_ ## X, SUFFIX), END), _mmu)
1272 #define ATOMIC_MMU_CLEANUP do { clear_helper_retaddr(); } while (0)
1273
1274 #define DATA_SIZE 1
1275 #include "atomic_template.h"
1276
1277 #define DATA_SIZE 2
1278 #include "atomic_template.h"
1279
1280 #define DATA_SIZE 4
1281 #include "atomic_template.h"
1282
1283 #ifdef CONFIG_ATOMIC64
1284 #define DATA_SIZE 8
1285 #include "atomic_template.h"
1286 #endif
1287
1288 #if defined(CONFIG_ATOMIC128) || HAVE_CMPXCHG128
1289 #define DATA_SIZE 16
1290 #include "atomic_template.h"
1291 #endif
1292