xref: /qemu/system/physmem.c (revision d8d5ca40)
1 /*
2  * RAM allocation and memory access
3  *
4  *  Copyright (c) 2003 Fabrice Bellard
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "exec/page-vary.h"
22 #include "qapi/error.h"
23 
24 #include "qemu/cutils.h"
25 #include "qemu/cacheflush.h"
26 #include "qemu/hbitmap.h"
27 #include "qemu/madvise.h"
28 #include "qemu/lockable.h"
29 
30 #ifdef CONFIG_TCG
31 #include "hw/core/tcg-cpu-ops.h"
32 #endif /* CONFIG_TCG */
33 
34 #include "exec/exec-all.h"
35 #include "exec/page-protection.h"
36 #include "exec/target_page.h"
37 #include "hw/qdev-core.h"
38 #include "hw/qdev-properties.h"
39 #include "hw/boards.h"
40 #include "sysemu/xen.h"
41 #include "sysemu/kvm.h"
42 #include "sysemu/tcg.h"
43 #include "sysemu/qtest.h"
44 #include "qemu/timer.h"
45 #include "qemu/config-file.h"
46 #include "qemu/error-report.h"
47 #include "qemu/qemu-print.h"
48 #include "qemu/log.h"
49 #include "qemu/memalign.h"
50 #include "exec/memory.h"
51 #include "exec/ioport.h"
52 #include "sysemu/dma.h"
53 #include "sysemu/hostmem.h"
54 #include "sysemu/hw_accel.h"
55 #include "sysemu/xen-mapcache.h"
56 #include "trace.h"
57 
58 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
59 #include <linux/falloc.h>
60 #endif
61 
62 #include "qemu/rcu_queue.h"
63 #include "qemu/main-loop.h"
64 #include "exec/translate-all.h"
65 #include "sysemu/replay.h"
66 
67 #include "exec/memory-internal.h"
68 #include "exec/ram_addr.h"
69 
70 #include "qemu/pmem.h"
71 
72 #include "migration/vmstate.h"
73 
74 #include "qemu/range.h"
75 #ifndef _WIN32
76 #include "qemu/mmap-alloc.h"
77 #endif
78 
79 #include "monitor/monitor.h"
80 
81 #ifdef CONFIG_LIBDAXCTL
82 #include <daxctl/libdaxctl.h>
83 #endif
84 
85 //#define DEBUG_SUBPAGE
86 
87 /* ram_list is read under rcu_read_lock()/rcu_read_unlock().  Writes
88  * are protected by the ramlist lock.
89  */
90 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
91 
92 static MemoryRegion *system_memory;
93 static MemoryRegion *system_io;
94 
95 AddressSpace address_space_io;
96 AddressSpace address_space_memory;
97 
98 static MemoryRegion io_mem_unassigned;
99 
100 typedef struct PhysPageEntry PhysPageEntry;
101 
102 struct PhysPageEntry {
103     /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
104     uint32_t skip : 6;
105      /* index into phys_sections (!skip) or phys_map_nodes (skip) */
106     uint32_t ptr : 26;
107 };
108 
109 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
110 
111 /* Size of the L2 (and L3, etc) page tables.  */
112 #define ADDR_SPACE_BITS 64
113 
114 #define P_L2_BITS 9
115 #define P_L2_SIZE (1 << P_L2_BITS)
116 
117 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
118 
119 typedef PhysPageEntry Node[P_L2_SIZE];
120 
121 typedef struct PhysPageMap {
122     struct rcu_head rcu;
123 
124     unsigned sections_nb;
125     unsigned sections_nb_alloc;
126     unsigned nodes_nb;
127     unsigned nodes_nb_alloc;
128     Node *nodes;
129     MemoryRegionSection *sections;
130 } PhysPageMap;
131 
132 struct AddressSpaceDispatch {
133     MemoryRegionSection *mru_section;
134     /* This is a multi-level map on the physical address space.
135      * The bottom level has pointers to MemoryRegionSections.
136      */
137     PhysPageEntry phys_map;
138     PhysPageMap map;
139 };
140 
141 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
142 typedef struct subpage_t {
143     MemoryRegion iomem;
144     FlatView *fv;
145     hwaddr base;
146     uint16_t sub_section[];
147 } subpage_t;
148 
149 #define PHYS_SECTION_UNASSIGNED 0
150 
151 static void io_mem_init(void);
152 static void memory_map_init(void);
153 static void tcg_log_global_after_sync(MemoryListener *listener);
154 static void tcg_commit(MemoryListener *listener);
155 
156 /**
157  * CPUAddressSpace: all the information a CPU needs about an AddressSpace
158  * @cpu: the CPU whose AddressSpace this is
159  * @as: the AddressSpace itself
160  * @memory_dispatch: its dispatch pointer (cached, RCU protected)
161  * @tcg_as_listener: listener for tracking changes to the AddressSpace
162  */
163 typedef struct CPUAddressSpace {
164     CPUState *cpu;
165     AddressSpace *as;
166     struct AddressSpaceDispatch *memory_dispatch;
167     MemoryListener tcg_as_listener;
168 } CPUAddressSpace;
169 
170 struct DirtyBitmapSnapshot {
171     ram_addr_t start;
172     ram_addr_t end;
173     unsigned long dirty[];
174 };
175 
phys_map_node_reserve(PhysPageMap * map,unsigned nodes)176 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
177 {
178     static unsigned alloc_hint = 16;
179     if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
180         map->nodes_nb_alloc = MAX(alloc_hint, map->nodes_nb + nodes);
181         map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
182         alloc_hint = map->nodes_nb_alloc;
183     }
184 }
185 
phys_map_node_alloc(PhysPageMap * map,bool leaf)186 static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
187 {
188     unsigned i;
189     uint32_t ret;
190     PhysPageEntry e;
191     PhysPageEntry *p;
192 
193     ret = map->nodes_nb++;
194     p = map->nodes[ret];
195     assert(ret != PHYS_MAP_NODE_NIL);
196     assert(ret != map->nodes_nb_alloc);
197 
198     e.skip = leaf ? 0 : 1;
199     e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
200     for (i = 0; i < P_L2_SIZE; ++i) {
201         memcpy(&p[i], &e, sizeof(e));
202     }
203     return ret;
204 }
205 
phys_page_set_level(PhysPageMap * map,PhysPageEntry * lp,hwaddr * index,uint64_t * nb,uint16_t leaf,int level)206 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
207                                 hwaddr *index, uint64_t *nb, uint16_t leaf,
208                                 int level)
209 {
210     PhysPageEntry *p;
211     hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
212 
213     if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
214         lp->ptr = phys_map_node_alloc(map, level == 0);
215     }
216     p = map->nodes[lp->ptr];
217     lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
218 
219     while (*nb && lp < &p[P_L2_SIZE]) {
220         if ((*index & (step - 1)) == 0 && *nb >= step) {
221             lp->skip = 0;
222             lp->ptr = leaf;
223             *index += step;
224             *nb -= step;
225         } else {
226             phys_page_set_level(map, lp, index, nb, leaf, level - 1);
227         }
228         ++lp;
229     }
230 }
231 
phys_page_set(AddressSpaceDispatch * d,hwaddr index,uint64_t nb,uint16_t leaf)232 static void phys_page_set(AddressSpaceDispatch *d,
233                           hwaddr index, uint64_t nb,
234                           uint16_t leaf)
235 {
236     /* Wildly overreserve - it doesn't matter much. */
237     phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
238 
239     phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
240 }
241 
242 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
243  * and update our entry so we can skip it and go directly to the destination.
244  */
phys_page_compact(PhysPageEntry * lp,Node * nodes)245 static void phys_page_compact(PhysPageEntry *lp, Node *nodes)
246 {
247     unsigned valid_ptr = P_L2_SIZE;
248     int valid = 0;
249     PhysPageEntry *p;
250     int i;
251 
252     if (lp->ptr == PHYS_MAP_NODE_NIL) {
253         return;
254     }
255 
256     p = nodes[lp->ptr];
257     for (i = 0; i < P_L2_SIZE; i++) {
258         if (p[i].ptr == PHYS_MAP_NODE_NIL) {
259             continue;
260         }
261 
262         valid_ptr = i;
263         valid++;
264         if (p[i].skip) {
265             phys_page_compact(&p[i], nodes);
266         }
267     }
268 
269     /* We can only compress if there's only one child. */
270     if (valid != 1) {
271         return;
272     }
273 
274     assert(valid_ptr < P_L2_SIZE);
275 
276     /* Don't compress if it won't fit in the # of bits we have. */
277     if (P_L2_LEVELS >= (1 << 6) &&
278         lp->skip + p[valid_ptr].skip >= (1 << 6)) {
279         return;
280     }
281 
282     lp->ptr = p[valid_ptr].ptr;
283     if (!p[valid_ptr].skip) {
284         /* If our only child is a leaf, make this a leaf. */
285         /* By design, we should have made this node a leaf to begin with so we
286          * should never reach here.
287          * But since it's so simple to handle this, let's do it just in case we
288          * change this rule.
289          */
290         lp->skip = 0;
291     } else {
292         lp->skip += p[valid_ptr].skip;
293     }
294 }
295 
address_space_dispatch_compact(AddressSpaceDispatch * d)296 void address_space_dispatch_compact(AddressSpaceDispatch *d)
297 {
298     if (d->phys_map.skip) {
299         phys_page_compact(&d->phys_map, d->map.nodes);
300     }
301 }
302 
section_covers_addr(const MemoryRegionSection * section,hwaddr addr)303 static inline bool section_covers_addr(const MemoryRegionSection *section,
304                                        hwaddr addr)
305 {
306     /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
307      * the section must cover the entire address space.
308      */
309     return int128_gethi(section->size) ||
310            range_covers_byte(section->offset_within_address_space,
311                              int128_getlo(section->size), addr);
312 }
313 
phys_page_find(AddressSpaceDispatch * d,hwaddr addr)314 static MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr addr)
315 {
316     PhysPageEntry lp = d->phys_map, *p;
317     Node *nodes = d->map.nodes;
318     MemoryRegionSection *sections = d->map.sections;
319     hwaddr index = addr >> TARGET_PAGE_BITS;
320     int i;
321 
322     for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
323         if (lp.ptr == PHYS_MAP_NODE_NIL) {
324             return &sections[PHYS_SECTION_UNASSIGNED];
325         }
326         p = nodes[lp.ptr];
327         lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
328     }
329 
330     if (section_covers_addr(&sections[lp.ptr], addr)) {
331         return &sections[lp.ptr];
332     } else {
333         return &sections[PHYS_SECTION_UNASSIGNED];
334     }
335 }
336 
337 /* Called from RCU critical section */
address_space_lookup_region(AddressSpaceDispatch * d,hwaddr addr,bool resolve_subpage)338 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
339                                                         hwaddr addr,
340                                                         bool resolve_subpage)
341 {
342     MemoryRegionSection *section = qatomic_read(&d->mru_section);
343     subpage_t *subpage;
344 
345     if (!section || section == &d->map.sections[PHYS_SECTION_UNASSIGNED] ||
346         !section_covers_addr(section, addr)) {
347         section = phys_page_find(d, addr);
348         qatomic_set(&d->mru_section, section);
349     }
350     if (resolve_subpage && section->mr->subpage) {
351         subpage = container_of(section->mr, subpage_t, iomem);
352         section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
353     }
354     return section;
355 }
356 
357 /* Called from RCU critical section */
358 static MemoryRegionSection *
address_space_translate_internal(AddressSpaceDispatch * d,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool resolve_subpage)359 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
360                                  hwaddr *plen, bool resolve_subpage)
361 {
362     MemoryRegionSection *section;
363     MemoryRegion *mr;
364     Int128 diff;
365 
366     section = address_space_lookup_region(d, addr, resolve_subpage);
367     /* Compute offset within MemoryRegionSection */
368     addr -= section->offset_within_address_space;
369 
370     /* Compute offset within MemoryRegion */
371     *xlat = addr + section->offset_within_region;
372 
373     mr = section->mr;
374 
375     /* MMIO registers can be expected to perform full-width accesses based only
376      * on their address, without considering adjacent registers that could
377      * decode to completely different MemoryRegions.  When such registers
378      * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
379      * regions overlap wildly.  For this reason we cannot clamp the accesses
380      * here.
381      *
382      * If the length is small (as is the case for address_space_ldl/stl),
383      * everything works fine.  If the incoming length is large, however,
384      * the caller really has to do the clamping through memory_access_size.
385      */
386     if (memory_region_is_ram(mr)) {
387         diff = int128_sub(section->size, int128_make64(addr));
388         *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
389     }
390     return section;
391 }
392 
393 /**
394  * address_space_translate_iommu - translate an address through an IOMMU
395  * memory region and then through the target address space.
396  *
397  * @iommu_mr: the IOMMU memory region that we start the translation from
398  * @addr: the address to be translated through the MMU
399  * @xlat: the translated address offset within the destination memory region.
400  *        It cannot be %NULL.
401  * @plen_out: valid read/write length of the translated address. It
402  *            cannot be %NULL.
403  * @page_mask_out: page mask for the translated address. This
404  *            should only be meaningful for IOMMU translated
405  *            addresses, since there may be huge pages that this bit
406  *            would tell. It can be %NULL if we don't care about it.
407  * @is_write: whether the translation operation is for write
408  * @is_mmio: whether this can be MMIO, set true if it can
409  * @target_as: the address space targeted by the IOMMU
410  * @attrs: transaction attributes
411  *
412  * This function is called from RCU critical section.  It is the common
413  * part of flatview_do_translate and address_space_translate_cached.
414  */
address_space_translate_iommu(IOMMUMemoryRegion * iommu_mr,hwaddr * xlat,hwaddr * plen_out,hwaddr * page_mask_out,bool is_write,bool is_mmio,AddressSpace ** target_as,MemTxAttrs attrs)415 static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
416                                                          hwaddr *xlat,
417                                                          hwaddr *plen_out,
418                                                          hwaddr *page_mask_out,
419                                                          bool is_write,
420                                                          bool is_mmio,
421                                                          AddressSpace **target_as,
422                                                          MemTxAttrs attrs)
423 {
424     MemoryRegionSection *section;
425     hwaddr page_mask = (hwaddr)-1;
426 
427     do {
428         hwaddr addr = *xlat;
429         IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
430         int iommu_idx = 0;
431         IOMMUTLBEntry iotlb;
432 
433         if (imrc->attrs_to_index) {
434             iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
435         }
436 
437         iotlb = imrc->translate(iommu_mr, addr, is_write ?
438                                 IOMMU_WO : IOMMU_RO, iommu_idx);
439 
440         if (!(iotlb.perm & (1 << is_write))) {
441             goto unassigned;
442         }
443 
444         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
445                 | (addr & iotlb.addr_mask));
446         page_mask &= iotlb.addr_mask;
447         *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
448         *target_as = iotlb.target_as;
449 
450         section = address_space_translate_internal(
451                 address_space_to_dispatch(iotlb.target_as), addr, xlat,
452                 plen_out, is_mmio);
453 
454         iommu_mr = memory_region_get_iommu(section->mr);
455     } while (unlikely(iommu_mr));
456 
457     if (page_mask_out) {
458         *page_mask_out = page_mask;
459     }
460     return *section;
461 
462 unassigned:
463     return (MemoryRegionSection) { .mr = &io_mem_unassigned };
464 }
465 
466 /**
467  * flatview_do_translate - translate an address in FlatView
468  *
469  * @fv: the flat view that we want to translate on
470  * @addr: the address to be translated in above address space
471  * @xlat: the translated address offset within memory region. It
472  *        cannot be @NULL.
473  * @plen_out: valid read/write length of the translated address. It
474  *            can be @NULL when we don't care about it.
475  * @page_mask_out: page mask for the translated address. This
476  *            should only be meaningful for IOMMU translated
477  *            addresses, since there may be huge pages that this bit
478  *            would tell. It can be @NULL if we don't care about it.
479  * @is_write: whether the translation operation is for write
480  * @is_mmio: whether this can be MMIO, set true if it can
481  * @target_as: the address space targeted by the IOMMU
482  * @attrs: memory transaction attributes
483  *
484  * This function is called from RCU critical section
485  */
flatview_do_translate(FlatView * fv,hwaddr addr,hwaddr * xlat,hwaddr * plen_out,hwaddr * page_mask_out,bool is_write,bool is_mmio,AddressSpace ** target_as,MemTxAttrs attrs)486 static MemoryRegionSection flatview_do_translate(FlatView *fv,
487                                                  hwaddr addr,
488                                                  hwaddr *xlat,
489                                                  hwaddr *plen_out,
490                                                  hwaddr *page_mask_out,
491                                                  bool is_write,
492                                                  bool is_mmio,
493                                                  AddressSpace **target_as,
494                                                  MemTxAttrs attrs)
495 {
496     MemoryRegionSection *section;
497     IOMMUMemoryRegion *iommu_mr;
498     hwaddr plen = (hwaddr)(-1);
499 
500     if (!plen_out) {
501         plen_out = &plen;
502     }
503 
504     section = address_space_translate_internal(
505             flatview_to_dispatch(fv), addr, xlat,
506             plen_out, is_mmio);
507 
508     iommu_mr = memory_region_get_iommu(section->mr);
509     if (unlikely(iommu_mr)) {
510         return address_space_translate_iommu(iommu_mr, xlat,
511                                              plen_out, page_mask_out,
512                                              is_write, is_mmio,
513                                              target_as, attrs);
514     }
515     if (page_mask_out) {
516         /* Not behind an IOMMU, use default page size. */
517         *page_mask_out = ~TARGET_PAGE_MASK;
518     }
519 
520     return *section;
521 }
522 
523 /* Called from RCU critical section */
address_space_get_iotlb_entry(AddressSpace * as,hwaddr addr,bool is_write,MemTxAttrs attrs)524 IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
525                                             bool is_write, MemTxAttrs attrs)
526 {
527     MemoryRegionSection section;
528     hwaddr xlat, page_mask;
529 
530     /*
531      * This can never be MMIO, and we don't really care about plen,
532      * but page mask.
533      */
534     section = flatview_do_translate(address_space_to_flatview(as), addr, &xlat,
535                                     NULL, &page_mask, is_write, false, &as,
536                                     attrs);
537 
538     /* Illegal translation */
539     if (section.mr == &io_mem_unassigned) {
540         goto iotlb_fail;
541     }
542 
543     /* Convert memory region offset into address space offset */
544     xlat += section.offset_within_address_space -
545         section.offset_within_region;
546 
547     return (IOMMUTLBEntry) {
548         .target_as = as,
549         .iova = addr & ~page_mask,
550         .translated_addr = xlat & ~page_mask,
551         .addr_mask = page_mask,
552         /* IOTLBs are for DMAs, and DMA only allows on RAMs. */
553         .perm = IOMMU_RW,
554     };
555 
556 iotlb_fail:
557     return (IOMMUTLBEntry) {0};
558 }
559 
560 /* Called from RCU critical section */
flatview_translate(FlatView * fv,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool is_write,MemTxAttrs attrs)561 MemoryRegion *flatview_translate(FlatView *fv, hwaddr addr, hwaddr *xlat,
562                                  hwaddr *plen, bool is_write,
563                                  MemTxAttrs attrs)
564 {
565     MemoryRegion *mr;
566     MemoryRegionSection section;
567     AddressSpace *as = NULL;
568 
569     /* This can be MMIO, so setup MMIO bit. */
570     section = flatview_do_translate(fv, addr, xlat, plen, NULL,
571                                     is_write, true, &as, attrs);
572     mr = section.mr;
573 
574     if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
575         hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
576         *plen = MIN(page, *plen);
577     }
578 
579     return mr;
580 }
581 
582 typedef struct TCGIOMMUNotifier {
583     IOMMUNotifier n;
584     MemoryRegion *mr;
585     CPUState *cpu;
586     int iommu_idx;
587     bool active;
588 } TCGIOMMUNotifier;
589 
tcg_iommu_unmap_notify(IOMMUNotifier * n,IOMMUTLBEntry * iotlb)590 static void tcg_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
591 {
592     TCGIOMMUNotifier *notifier = container_of(n, TCGIOMMUNotifier, n);
593 
594     if (!notifier->active) {
595         return;
596     }
597     tlb_flush(notifier->cpu);
598     notifier->active = false;
599     /* We leave the notifier struct on the list to avoid reallocating it later.
600      * Generally the number of IOMMUs a CPU deals with will be small.
601      * In any case we can't unregister the iommu notifier from a notify
602      * callback.
603      */
604 }
605 
tcg_register_iommu_notifier(CPUState * cpu,IOMMUMemoryRegion * iommu_mr,int iommu_idx)606 static void tcg_register_iommu_notifier(CPUState *cpu,
607                                         IOMMUMemoryRegion *iommu_mr,
608                                         int iommu_idx)
609 {
610     /* Make sure this CPU has an IOMMU notifier registered for this
611      * IOMMU/IOMMU index combination, so that we can flush its TLB
612      * when the IOMMU tells us the mappings we've cached have changed.
613      */
614     MemoryRegion *mr = MEMORY_REGION(iommu_mr);
615     TCGIOMMUNotifier *notifier = NULL;
616     int i;
617 
618     for (i = 0; i < cpu->iommu_notifiers->len; i++) {
619         notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
620         if (notifier->mr == mr && notifier->iommu_idx == iommu_idx) {
621             break;
622         }
623     }
624     if (i == cpu->iommu_notifiers->len) {
625         /* Not found, add a new entry at the end of the array */
626         cpu->iommu_notifiers = g_array_set_size(cpu->iommu_notifiers, i + 1);
627         notifier = g_new0(TCGIOMMUNotifier, 1);
628         g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i) = notifier;
629 
630         notifier->mr = mr;
631         notifier->iommu_idx = iommu_idx;
632         notifier->cpu = cpu;
633         /* Rather than trying to register interest in the specific part
634          * of the iommu's address space that we've accessed and then
635          * expand it later as subsequent accesses touch more of it, we
636          * just register interest in the whole thing, on the assumption
637          * that iommu reconfiguration will be rare.
638          */
639         iommu_notifier_init(&notifier->n,
640                             tcg_iommu_unmap_notify,
641                             IOMMU_NOTIFIER_UNMAP,
642                             0,
643                             HWADDR_MAX,
644                             iommu_idx);
645         memory_region_register_iommu_notifier(notifier->mr, &notifier->n,
646                                               &error_fatal);
647     }
648 
649     if (!notifier->active) {
650         notifier->active = true;
651     }
652 }
653 
tcg_iommu_free_notifier_list(CPUState * cpu)654 void tcg_iommu_free_notifier_list(CPUState *cpu)
655 {
656     /* Destroy the CPU's notifier list */
657     int i;
658     TCGIOMMUNotifier *notifier;
659 
660     for (i = 0; i < cpu->iommu_notifiers->len; i++) {
661         notifier = g_array_index(cpu->iommu_notifiers, TCGIOMMUNotifier *, i);
662         memory_region_unregister_iommu_notifier(notifier->mr, &notifier->n);
663         g_free(notifier);
664     }
665     g_array_free(cpu->iommu_notifiers, true);
666 }
667 
tcg_iommu_init_notifier_list(CPUState * cpu)668 void tcg_iommu_init_notifier_list(CPUState *cpu)
669 {
670     cpu->iommu_notifiers = g_array_new(false, true, sizeof(TCGIOMMUNotifier *));
671 }
672 
673 /* Called from RCU critical section */
674 MemoryRegionSection *
address_space_translate_for_iotlb(CPUState * cpu,int asidx,hwaddr orig_addr,hwaddr * xlat,hwaddr * plen,MemTxAttrs attrs,int * prot)675 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr orig_addr,
676                                   hwaddr *xlat, hwaddr *plen,
677                                   MemTxAttrs attrs, int *prot)
678 {
679     MemoryRegionSection *section;
680     IOMMUMemoryRegion *iommu_mr;
681     IOMMUMemoryRegionClass *imrc;
682     IOMMUTLBEntry iotlb;
683     int iommu_idx;
684     hwaddr addr = orig_addr;
685     AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
686 
687     for (;;) {
688         section = address_space_translate_internal(d, addr, &addr, plen, false);
689 
690         iommu_mr = memory_region_get_iommu(section->mr);
691         if (!iommu_mr) {
692             break;
693         }
694 
695         imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
696 
697         iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
698         tcg_register_iommu_notifier(cpu, iommu_mr, iommu_idx);
699         /* We need all the permissions, so pass IOMMU_NONE so the IOMMU
700          * doesn't short-cut its translation table walk.
701          */
702         iotlb = imrc->translate(iommu_mr, addr, IOMMU_NONE, iommu_idx);
703         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
704                 | (addr & iotlb.addr_mask));
705         /* Update the caller's prot bits to remove permissions the IOMMU
706          * is giving us a failure response for. If we get down to no
707          * permissions left at all we can give up now.
708          */
709         if (!(iotlb.perm & IOMMU_RO)) {
710             *prot &= ~(PAGE_READ | PAGE_EXEC);
711         }
712         if (!(iotlb.perm & IOMMU_WO)) {
713             *prot &= ~PAGE_WRITE;
714         }
715 
716         if (!*prot) {
717             goto translate_fail;
718         }
719 
720         d = flatview_to_dispatch(address_space_to_flatview(iotlb.target_as));
721     }
722 
723     assert(!memory_region_is_iommu(section->mr));
724     *xlat = addr;
725     return section;
726 
727 translate_fail:
728     /*
729      * We should be given a page-aligned address -- certainly
730      * tlb_set_page_with_attrs() does so.  The page offset of xlat
731      * is used to index sections[], and PHYS_SECTION_UNASSIGNED = 0.
732      * The page portion of xlat will be logged by memory_region_access_valid()
733      * when this memory access is rejected, so use the original untranslated
734      * physical address.
735      */
736     assert((orig_addr & ~TARGET_PAGE_MASK) == 0);
737     *xlat = orig_addr;
738     return &d->map.sections[PHYS_SECTION_UNASSIGNED];
739 }
740 
cpu_address_space_init(CPUState * cpu,int asidx,const char * prefix,MemoryRegion * mr)741 void cpu_address_space_init(CPUState *cpu, int asidx,
742                             const char *prefix, MemoryRegion *mr)
743 {
744     CPUAddressSpace *newas;
745     AddressSpace *as = g_new0(AddressSpace, 1);
746     char *as_name;
747 
748     assert(mr);
749     as_name = g_strdup_printf("%s-%d", prefix, cpu->cpu_index);
750     address_space_init(as, mr, as_name);
751     g_free(as_name);
752 
753     /* Target code should have set num_ases before calling us */
754     assert(asidx < cpu->num_ases);
755 
756     if (asidx == 0) {
757         /* address space 0 gets the convenience alias */
758         cpu->as = as;
759     }
760 
761     /* KVM cannot currently support multiple address spaces. */
762     assert(asidx == 0 || !kvm_enabled());
763 
764     if (!cpu->cpu_ases) {
765         cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
766         cpu->cpu_ases_count = cpu->num_ases;
767     }
768 
769     newas = &cpu->cpu_ases[asidx];
770     newas->cpu = cpu;
771     newas->as = as;
772     if (tcg_enabled()) {
773         newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync;
774         newas->tcg_as_listener.commit = tcg_commit;
775         newas->tcg_as_listener.name = "tcg";
776         memory_listener_register(&newas->tcg_as_listener, as);
777     }
778 }
779 
cpu_address_space_destroy(CPUState * cpu,int asidx)780 void cpu_address_space_destroy(CPUState *cpu, int asidx)
781 {
782     CPUAddressSpace *cpuas;
783 
784     assert(cpu->cpu_ases);
785     assert(asidx >= 0 && asidx < cpu->num_ases);
786     /* KVM cannot currently support multiple address spaces. */
787     assert(asidx == 0 || !kvm_enabled());
788 
789     cpuas = &cpu->cpu_ases[asidx];
790     if (tcg_enabled()) {
791         memory_listener_unregister(&cpuas->tcg_as_listener);
792     }
793 
794     address_space_destroy(cpuas->as);
795     g_free_rcu(cpuas->as, rcu);
796 
797     if (asidx == 0) {
798         /* reset the convenience alias for address space 0 */
799         cpu->as = NULL;
800     }
801 
802     if (--cpu->cpu_ases_count == 0) {
803         g_free(cpu->cpu_ases);
804         cpu->cpu_ases = NULL;
805     }
806 }
807 
cpu_get_address_space(CPUState * cpu,int asidx)808 AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
809 {
810     /* Return the AddressSpace corresponding to the specified index */
811     return cpu->cpu_ases[asidx].as;
812 }
813 
814 /* Called from RCU critical section */
qemu_get_ram_block(ram_addr_t addr)815 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
816 {
817     RAMBlock *block;
818 
819     block = qatomic_rcu_read(&ram_list.mru_block);
820     if (block && addr - block->offset < block->max_length) {
821         return block;
822     }
823     RAMBLOCK_FOREACH(block) {
824         if (addr - block->offset < block->max_length) {
825             goto found;
826         }
827     }
828 
829     fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
830     abort();
831 
832 found:
833     /* It is safe to write mru_block outside the BQL.  This
834      * is what happens:
835      *
836      *     mru_block = xxx
837      *     rcu_read_unlock()
838      *                                        xxx removed from list
839      *                  rcu_read_lock()
840      *                  read mru_block
841      *                                        mru_block = NULL;
842      *                                        call_rcu(reclaim_ramblock, xxx);
843      *                  rcu_read_unlock()
844      *
845      * qatomic_rcu_set is not needed here.  The block was already published
846      * when it was placed into the list.  Here we're just making an extra
847      * copy of the pointer.
848      */
849     ram_list.mru_block = block;
850     return block;
851 }
852 
tlb_reset_dirty_range_all(ram_addr_t start,ram_addr_t length)853 void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
854 {
855     CPUState *cpu;
856     ram_addr_t start1;
857     RAMBlock *block;
858     ram_addr_t end;
859 
860     assert(tcg_enabled());
861     end = TARGET_PAGE_ALIGN(start + length);
862     start &= TARGET_PAGE_MASK;
863 
864     RCU_READ_LOCK_GUARD();
865     block = qemu_get_ram_block(start);
866     assert(block == qemu_get_ram_block(end - 1));
867     start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
868     CPU_FOREACH(cpu) {
869         tlb_reset_dirty(cpu, start1, length);
870     }
871 }
872 
873 /* Note: start and end must be within the same ram block.  */
cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,ram_addr_t length,unsigned client)874 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
875                                               ram_addr_t length,
876                                               unsigned client)
877 {
878     DirtyMemoryBlocks *blocks;
879     unsigned long end, page, start_page;
880     bool dirty = false;
881     RAMBlock *ramblock;
882     uint64_t mr_offset, mr_size;
883 
884     if (length == 0) {
885         return false;
886     }
887 
888     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
889     start_page = start >> TARGET_PAGE_BITS;
890     page = start_page;
891 
892     WITH_RCU_READ_LOCK_GUARD() {
893         blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
894         ramblock = qemu_get_ram_block(start);
895         /* Range sanity check on the ramblock */
896         assert(start >= ramblock->offset &&
897                start + length <= ramblock->offset + ramblock->used_length);
898 
899         while (page < end) {
900             unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
901             unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
902             unsigned long num = MIN(end - page,
903                                     DIRTY_MEMORY_BLOCK_SIZE - offset);
904 
905             dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
906                                                   offset, num);
907             page += num;
908         }
909 
910         mr_offset = (ram_addr_t)(start_page << TARGET_PAGE_BITS) - ramblock->offset;
911         mr_size = (end - start_page) << TARGET_PAGE_BITS;
912         memory_region_clear_dirty_bitmap(ramblock->mr, mr_offset, mr_size);
913     }
914 
915     if (dirty) {
916         cpu_physical_memory_dirty_bits_cleared(start, length);
917     }
918 
919     return dirty;
920 }
921 
cpu_physical_memory_snapshot_and_clear_dirty(MemoryRegion * mr,hwaddr offset,hwaddr length,unsigned client)922 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
923     (MemoryRegion *mr, hwaddr offset, hwaddr length, unsigned client)
924 {
925     DirtyMemoryBlocks *blocks;
926     ram_addr_t start, first, last;
927     unsigned long align = 1UL << (TARGET_PAGE_BITS + BITS_PER_LEVEL);
928     DirtyBitmapSnapshot *snap;
929     unsigned long page, end, dest;
930 
931     start = memory_region_get_ram_addr(mr);
932     /* We know we're only called for RAM MemoryRegions */
933     assert(start != RAM_ADDR_INVALID);
934     start += offset;
935 
936     first = QEMU_ALIGN_DOWN(start, align);
937     last  = QEMU_ALIGN_UP(start + length, align);
938 
939     snap = g_malloc0(sizeof(*snap) +
940                      ((last - first) >> (TARGET_PAGE_BITS + 3)));
941     snap->start = first;
942     snap->end   = last;
943 
944     page = first >> TARGET_PAGE_BITS;
945     end  = last  >> TARGET_PAGE_BITS;
946     dest = 0;
947 
948     WITH_RCU_READ_LOCK_GUARD() {
949         blocks = qatomic_rcu_read(&ram_list.dirty_memory[client]);
950 
951         while (page < end) {
952             unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
953             unsigned long ofs = page % DIRTY_MEMORY_BLOCK_SIZE;
954             unsigned long num = MIN(end - page,
955                                     DIRTY_MEMORY_BLOCK_SIZE - ofs);
956 
957             assert(QEMU_IS_ALIGNED(ofs, (1 << BITS_PER_LEVEL)));
958             assert(QEMU_IS_ALIGNED(num,    (1 << BITS_PER_LEVEL)));
959             ofs >>= BITS_PER_LEVEL;
960 
961             bitmap_copy_and_clear_atomic(snap->dirty + dest,
962                                          blocks->blocks[idx] + ofs,
963                                          num);
964             page += num;
965             dest += num >> BITS_PER_LEVEL;
966         }
967     }
968 
969     cpu_physical_memory_dirty_bits_cleared(start, length);
970 
971     memory_region_clear_dirty_bitmap(mr, offset, length);
972 
973     return snap;
974 }
975 
cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot * snap,ram_addr_t start,ram_addr_t length)976 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
977                                             ram_addr_t start,
978                                             ram_addr_t length)
979 {
980     unsigned long page, end;
981 
982     assert(start >= snap->start);
983     assert(start + length <= snap->end);
984 
985     end = TARGET_PAGE_ALIGN(start + length - snap->start) >> TARGET_PAGE_BITS;
986     page = (start - snap->start) >> TARGET_PAGE_BITS;
987 
988     while (page < end) {
989         if (test_bit(page, snap->dirty)) {
990             return true;
991         }
992         page++;
993     }
994     return false;
995 }
996 
997 /* Called from RCU critical section */
memory_region_section_get_iotlb(CPUState * cpu,MemoryRegionSection * section)998 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
999                                        MemoryRegionSection *section)
1000 {
1001     AddressSpaceDispatch *d = flatview_to_dispatch(section->fv);
1002     return section - d->map.sections;
1003 }
1004 
1005 static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
1006                             uint16_t section);
1007 static subpage_t *subpage_init(FlatView *fv, hwaddr base);
1008 
phys_section_add(PhysPageMap * map,MemoryRegionSection * section)1009 static uint16_t phys_section_add(PhysPageMap *map,
1010                                  MemoryRegionSection *section)
1011 {
1012     /* The physical section number is ORed with a page-aligned
1013      * pointer to produce the iotlb entries.  Thus it should
1014      * never overflow into the page-aligned value.
1015      */
1016     assert(map->sections_nb < TARGET_PAGE_SIZE);
1017 
1018     if (map->sections_nb == map->sections_nb_alloc) {
1019         map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
1020         map->sections = g_renew(MemoryRegionSection, map->sections,
1021                                 map->sections_nb_alloc);
1022     }
1023     map->sections[map->sections_nb] = *section;
1024     memory_region_ref(section->mr);
1025     return map->sections_nb++;
1026 }
1027 
phys_section_destroy(MemoryRegion * mr)1028 static void phys_section_destroy(MemoryRegion *mr)
1029 {
1030     bool have_sub_page = mr->subpage;
1031 
1032     memory_region_unref(mr);
1033 
1034     if (have_sub_page) {
1035         subpage_t *subpage = container_of(mr, subpage_t, iomem);
1036         object_unref(OBJECT(&subpage->iomem));
1037         g_free(subpage);
1038     }
1039 }
1040 
phys_sections_free(PhysPageMap * map)1041 static void phys_sections_free(PhysPageMap *map)
1042 {
1043     while (map->sections_nb > 0) {
1044         MemoryRegionSection *section = &map->sections[--map->sections_nb];
1045         phys_section_destroy(section->mr);
1046     }
1047     g_free(map->sections);
1048     g_free(map->nodes);
1049 }
1050 
register_subpage(FlatView * fv,MemoryRegionSection * section)1051 static void register_subpage(FlatView *fv, MemoryRegionSection *section)
1052 {
1053     AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1054     subpage_t *subpage;
1055     hwaddr base = section->offset_within_address_space
1056         & TARGET_PAGE_MASK;
1057     MemoryRegionSection *existing = phys_page_find(d, base);
1058     MemoryRegionSection subsection = {
1059         .offset_within_address_space = base,
1060         .size = int128_make64(TARGET_PAGE_SIZE),
1061     };
1062     hwaddr start, end;
1063 
1064     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
1065 
1066     if (!(existing->mr->subpage)) {
1067         subpage = subpage_init(fv, base);
1068         subsection.fv = fv;
1069         subsection.mr = &subpage->iomem;
1070         phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
1071                       phys_section_add(&d->map, &subsection));
1072     } else {
1073         subpage = container_of(existing->mr, subpage_t, iomem);
1074     }
1075     start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
1076     end = start + int128_get64(section->size) - 1;
1077     subpage_register(subpage, start, end,
1078                      phys_section_add(&d->map, section));
1079 }
1080 
1081 
register_multipage(FlatView * fv,MemoryRegionSection * section)1082 static void register_multipage(FlatView *fv,
1083                                MemoryRegionSection *section)
1084 {
1085     AddressSpaceDispatch *d = flatview_to_dispatch(fv);
1086     hwaddr start_addr = section->offset_within_address_space;
1087     uint16_t section_index = phys_section_add(&d->map, section);
1088     uint64_t num_pages = int128_get64(int128_rshift(section->size,
1089                                                     TARGET_PAGE_BITS));
1090 
1091     assert(num_pages);
1092     phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
1093 }
1094 
1095 /*
1096  * The range in *section* may look like this:
1097  *
1098  *      |s|PPPPPPP|s|
1099  *
1100  * where s stands for subpage and P for page.
1101  */
flatview_add_to_dispatch(FlatView * fv,MemoryRegionSection * section)1102 void flatview_add_to_dispatch(FlatView *fv, MemoryRegionSection *section)
1103 {
1104     MemoryRegionSection remain = *section;
1105     Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
1106 
1107     /* register first subpage */
1108     if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
1109         uint64_t left = TARGET_PAGE_ALIGN(remain.offset_within_address_space)
1110                         - remain.offset_within_address_space;
1111 
1112         MemoryRegionSection now = remain;
1113         now.size = int128_min(int128_make64(left), now.size);
1114         register_subpage(fv, &now);
1115         if (int128_eq(remain.size, now.size)) {
1116             return;
1117         }
1118         remain.size = int128_sub(remain.size, now.size);
1119         remain.offset_within_address_space += int128_get64(now.size);
1120         remain.offset_within_region += int128_get64(now.size);
1121     }
1122 
1123     /* register whole pages */
1124     if (int128_ge(remain.size, page_size)) {
1125         MemoryRegionSection now = remain;
1126         now.size = int128_and(now.size, int128_neg(page_size));
1127         register_multipage(fv, &now);
1128         if (int128_eq(remain.size, now.size)) {
1129             return;
1130         }
1131         remain.size = int128_sub(remain.size, now.size);
1132         remain.offset_within_address_space += int128_get64(now.size);
1133         remain.offset_within_region += int128_get64(now.size);
1134     }
1135 
1136     /* register last subpage */
1137     register_subpage(fv, &remain);
1138 }
1139 
qemu_flush_coalesced_mmio_buffer(void)1140 void qemu_flush_coalesced_mmio_buffer(void)
1141 {
1142     if (kvm_enabled())
1143         kvm_flush_coalesced_mmio_buffer();
1144 }
1145 
qemu_mutex_lock_ramlist(void)1146 void qemu_mutex_lock_ramlist(void)
1147 {
1148     qemu_mutex_lock(&ram_list.mutex);
1149 }
1150 
qemu_mutex_unlock_ramlist(void)1151 void qemu_mutex_unlock_ramlist(void)
1152 {
1153     qemu_mutex_unlock(&ram_list.mutex);
1154 }
1155 
ram_block_format(void)1156 GString *ram_block_format(void)
1157 {
1158     RAMBlock *block;
1159     char *psize;
1160     GString *buf = g_string_new("");
1161 
1162     RCU_READ_LOCK_GUARD();
1163     g_string_append_printf(buf, "%24s %8s  %18s %18s %18s %18s %3s\n",
1164                            "Block Name", "PSize", "Offset", "Used", "Total",
1165                            "HVA", "RO");
1166 
1167     RAMBLOCK_FOREACH(block) {
1168         psize = size_to_str(block->page_size);
1169         g_string_append_printf(buf, "%24s %8s  0x%016" PRIx64 " 0x%016" PRIx64
1170                                " 0x%016" PRIx64 " 0x%016" PRIx64 " %3s\n",
1171                                block->idstr, psize,
1172                                (uint64_t)block->offset,
1173                                (uint64_t)block->used_length,
1174                                (uint64_t)block->max_length,
1175                                (uint64_t)(uintptr_t)block->host,
1176                                block->mr->readonly ? "ro" : "rw");
1177 
1178         g_free(psize);
1179     }
1180 
1181     return buf;
1182 }
1183 
find_min_backend_pagesize(Object * obj,void * opaque)1184 static int find_min_backend_pagesize(Object *obj, void *opaque)
1185 {
1186     long *hpsize_min = opaque;
1187 
1188     if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1189         HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1190         long hpsize = host_memory_backend_pagesize(backend);
1191 
1192         if (host_memory_backend_is_mapped(backend) && (hpsize < *hpsize_min)) {
1193             *hpsize_min = hpsize;
1194         }
1195     }
1196 
1197     return 0;
1198 }
1199 
find_max_backend_pagesize(Object * obj,void * opaque)1200 static int find_max_backend_pagesize(Object *obj, void *opaque)
1201 {
1202     long *hpsize_max = opaque;
1203 
1204     if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
1205         HostMemoryBackend *backend = MEMORY_BACKEND(obj);
1206         long hpsize = host_memory_backend_pagesize(backend);
1207 
1208         if (host_memory_backend_is_mapped(backend) && (hpsize > *hpsize_max)) {
1209             *hpsize_max = hpsize;
1210         }
1211     }
1212 
1213     return 0;
1214 }
1215 
1216 /*
1217  * TODO: We assume right now that all mapped host memory backends are
1218  * used as RAM, however some might be used for different purposes.
1219  */
qemu_minrampagesize(void)1220 long qemu_minrampagesize(void)
1221 {
1222     long hpsize = LONG_MAX;
1223     Object *memdev_root = object_resolve_path("/objects", NULL);
1224 
1225     object_child_foreach(memdev_root, find_min_backend_pagesize, &hpsize);
1226     return hpsize;
1227 }
1228 
qemu_maxrampagesize(void)1229 long qemu_maxrampagesize(void)
1230 {
1231     long pagesize = 0;
1232     Object *memdev_root = object_resolve_path("/objects", NULL);
1233 
1234     object_child_foreach(memdev_root, find_max_backend_pagesize, &pagesize);
1235     return pagesize;
1236 }
1237 
1238 #ifdef CONFIG_POSIX
get_file_size(int fd)1239 static int64_t get_file_size(int fd)
1240 {
1241     int64_t size;
1242 #if defined(__linux__)
1243     struct stat st;
1244 
1245     if (fstat(fd, &st) < 0) {
1246         return -errno;
1247     }
1248 
1249     /* Special handling for devdax character devices */
1250     if (S_ISCHR(st.st_mode)) {
1251         g_autofree char *subsystem_path = NULL;
1252         g_autofree char *subsystem = NULL;
1253 
1254         subsystem_path = g_strdup_printf("/sys/dev/char/%d:%d/subsystem",
1255                                          major(st.st_rdev), minor(st.st_rdev));
1256         subsystem = g_file_read_link(subsystem_path, NULL);
1257 
1258         if (subsystem && g_str_has_suffix(subsystem, "/dax")) {
1259             g_autofree char *size_path = NULL;
1260             g_autofree char *size_str = NULL;
1261 
1262             size_path = g_strdup_printf("/sys/dev/char/%d:%d/size",
1263                                     major(st.st_rdev), minor(st.st_rdev));
1264 
1265             if (g_file_get_contents(size_path, &size_str, NULL, NULL)) {
1266                 return g_ascii_strtoll(size_str, NULL, 0);
1267             }
1268         }
1269     }
1270 #endif /* defined(__linux__) */
1271 
1272     /* st.st_size may be zero for special files yet lseek(2) works */
1273     size = lseek(fd, 0, SEEK_END);
1274     if (size < 0) {
1275         return -errno;
1276     }
1277     return size;
1278 }
1279 
get_file_align(int fd)1280 static int64_t get_file_align(int fd)
1281 {
1282     int64_t align = -1;
1283 #if defined(__linux__) && defined(CONFIG_LIBDAXCTL)
1284     struct stat st;
1285 
1286     if (fstat(fd, &st) < 0) {
1287         return -errno;
1288     }
1289 
1290     /* Special handling for devdax character devices */
1291     if (S_ISCHR(st.st_mode)) {
1292         g_autofree char *path = NULL;
1293         g_autofree char *rpath = NULL;
1294         struct daxctl_ctx *ctx;
1295         struct daxctl_region *region;
1296         int rc = 0;
1297 
1298         path = g_strdup_printf("/sys/dev/char/%d:%d",
1299                     major(st.st_rdev), minor(st.st_rdev));
1300         rpath = realpath(path, NULL);
1301         if (!rpath) {
1302             return -errno;
1303         }
1304 
1305         rc = daxctl_new(&ctx);
1306         if (rc) {
1307             return -1;
1308         }
1309 
1310         daxctl_region_foreach(ctx, region) {
1311             if (strstr(rpath, daxctl_region_get_path(region))) {
1312                 align = daxctl_region_get_align(region);
1313                 break;
1314             }
1315         }
1316         daxctl_unref(ctx);
1317     }
1318 #endif /* defined(__linux__) && defined(CONFIG_LIBDAXCTL) */
1319 
1320     return align;
1321 }
1322 
file_ram_open(const char * path,const char * region_name,bool readonly,bool * created)1323 static int file_ram_open(const char *path,
1324                          const char *region_name,
1325                          bool readonly,
1326                          bool *created)
1327 {
1328     char *filename;
1329     char *sanitized_name;
1330     char *c;
1331     int fd = -1;
1332 
1333     *created = false;
1334     for (;;) {
1335         fd = open(path, readonly ? O_RDONLY : O_RDWR);
1336         if (fd >= 0) {
1337             /*
1338              * open(O_RDONLY) won't fail with EISDIR. Check manually if we
1339              * opened a directory and fail similarly to how we fail ENOENT
1340              * in readonly mode. Note that mkstemp() would imply O_RDWR.
1341              */
1342             if (readonly) {
1343                 struct stat file_stat;
1344 
1345                 if (fstat(fd, &file_stat)) {
1346                     close(fd);
1347                     if (errno == EINTR) {
1348                         continue;
1349                     }
1350                     return -errno;
1351                 } else if (S_ISDIR(file_stat.st_mode)) {
1352                     close(fd);
1353                     return -EISDIR;
1354                 }
1355             }
1356             /* @path names an existing file, use it */
1357             break;
1358         }
1359         if (errno == ENOENT) {
1360             if (readonly) {
1361                 /* Refuse to create new, readonly files. */
1362                 return -ENOENT;
1363             }
1364             /* @path names a file that doesn't exist, create it */
1365             fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
1366             if (fd >= 0) {
1367                 *created = true;
1368                 break;
1369             }
1370         } else if (errno == EISDIR) {
1371             /* @path names a directory, create a file there */
1372             /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1373             sanitized_name = g_strdup(region_name);
1374             for (c = sanitized_name; *c != '\0'; c++) {
1375                 if (*c == '/') {
1376                     *c = '_';
1377                 }
1378             }
1379 
1380             filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1381                                        sanitized_name);
1382             g_free(sanitized_name);
1383 
1384             fd = mkstemp(filename);
1385             if (fd >= 0) {
1386                 unlink(filename);
1387                 g_free(filename);
1388                 break;
1389             }
1390             g_free(filename);
1391         }
1392         if (errno != EEXIST && errno != EINTR) {
1393             return -errno;
1394         }
1395         /*
1396          * Try again on EINTR and EEXIST.  The latter happens when
1397          * something else creates the file between our two open().
1398          */
1399     }
1400 
1401     return fd;
1402 }
1403 
file_ram_alloc(RAMBlock * block,ram_addr_t memory,int fd,bool truncate,off_t offset,Error ** errp)1404 static void *file_ram_alloc(RAMBlock *block,
1405                             ram_addr_t memory,
1406                             int fd,
1407                             bool truncate,
1408                             off_t offset,
1409                             Error **errp)
1410 {
1411     uint32_t qemu_map_flags;
1412     void *area;
1413 
1414     block->page_size = qemu_fd_getpagesize(fd);
1415     if (block->mr->align % block->page_size) {
1416         error_setg(errp, "alignment 0x%" PRIx64
1417                    " must be multiples of page size 0x%zx",
1418                    block->mr->align, block->page_size);
1419         return NULL;
1420     } else if (block->mr->align && !is_power_of_2(block->mr->align)) {
1421         error_setg(errp, "alignment 0x%" PRIx64
1422                    " must be a power of two", block->mr->align);
1423         return NULL;
1424     } else if (offset % block->page_size) {
1425         error_setg(errp, "offset 0x%" PRIx64
1426                    " must be multiples of page size 0x%zx",
1427                    offset, block->page_size);
1428         return NULL;
1429     }
1430     block->mr->align = MAX(block->page_size, block->mr->align);
1431 #if defined(__s390x__)
1432     if (kvm_enabled()) {
1433         block->mr->align = MAX(block->mr->align, QEMU_VMALLOC_ALIGN);
1434     }
1435 #endif
1436 
1437     if (memory < block->page_size) {
1438         error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
1439                    "or larger than page size 0x%zx",
1440                    memory, block->page_size);
1441         return NULL;
1442     }
1443 
1444     memory = ROUND_UP(memory, block->page_size);
1445 
1446     /*
1447      * ftruncate is not supported by hugetlbfs in older
1448      * hosts, so don't bother bailing out on errors.
1449      * If anything goes wrong with it under other filesystems,
1450      * mmap will fail.
1451      *
1452      * Do not truncate the non-empty backend file to avoid corrupting
1453      * the existing data in the file. Disabling shrinking is not
1454      * enough. For example, the current vNVDIMM implementation stores
1455      * the guest NVDIMM labels at the end of the backend file. If the
1456      * backend file is later extended, QEMU will not be able to find
1457      * those labels. Therefore, extending the non-empty backend file
1458      * is disabled as well.
1459      */
1460     if (truncate && ftruncate(fd, offset + memory)) {
1461         perror("ftruncate");
1462     }
1463 
1464     qemu_map_flags = (block->flags & RAM_READONLY) ? QEMU_MAP_READONLY : 0;
1465     qemu_map_flags |= (block->flags & RAM_SHARED) ? QEMU_MAP_SHARED : 0;
1466     qemu_map_flags |= (block->flags & RAM_PMEM) ? QEMU_MAP_SYNC : 0;
1467     qemu_map_flags |= (block->flags & RAM_NORESERVE) ? QEMU_MAP_NORESERVE : 0;
1468     area = qemu_ram_mmap(fd, memory, block->mr->align, qemu_map_flags, offset);
1469     if (area == MAP_FAILED) {
1470         error_setg_errno(errp, errno,
1471                          "unable to map backing store for guest RAM");
1472         return NULL;
1473     }
1474 
1475     block->fd = fd;
1476     block->fd_offset = offset;
1477     return area;
1478 }
1479 #endif
1480 
1481 /* Allocate space within the ram_addr_t space that governs the
1482  * dirty bitmaps.
1483  * Called with the ramlist lock held.
1484  */
find_ram_offset(ram_addr_t size)1485 static ram_addr_t find_ram_offset(ram_addr_t size)
1486 {
1487     RAMBlock *block, *next_block;
1488     ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1489 
1490     assert(size != 0); /* it would hand out same offset multiple times */
1491 
1492     if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
1493         return 0;
1494     }
1495 
1496     RAMBLOCK_FOREACH(block) {
1497         ram_addr_t candidate, next = RAM_ADDR_MAX;
1498 
1499         /* Align blocks to start on a 'long' in the bitmap
1500          * which makes the bitmap sync'ing take the fast path.
1501          */
1502         candidate = block->offset + block->max_length;
1503         candidate = ROUND_UP(candidate, BITS_PER_LONG << TARGET_PAGE_BITS);
1504 
1505         /* Search for the closest following block
1506          * and find the gap.
1507          */
1508         RAMBLOCK_FOREACH(next_block) {
1509             if (next_block->offset >= candidate) {
1510                 next = MIN(next, next_block->offset);
1511             }
1512         }
1513 
1514         /* If it fits remember our place and remember the size
1515          * of gap, but keep going so that we might find a smaller
1516          * gap to fill so avoiding fragmentation.
1517          */
1518         if (next - candidate >= size && next - candidate < mingap) {
1519             offset = candidate;
1520             mingap = next - candidate;
1521         }
1522 
1523         trace_find_ram_offset_loop(size, candidate, offset, next, mingap);
1524     }
1525 
1526     if (offset == RAM_ADDR_MAX) {
1527         fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1528                 (uint64_t)size);
1529         abort();
1530     }
1531 
1532     trace_find_ram_offset(size, offset);
1533 
1534     return offset;
1535 }
1536 
qemu_ram_setup_dump(void * addr,ram_addr_t size)1537 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1538 {
1539     int ret;
1540 
1541     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1542     if (!machine_dump_guest_core(current_machine)) {
1543         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1544         if (ret) {
1545             perror("qemu_madvise");
1546             fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1547                             "but dump-guest-core=off specified\n");
1548         }
1549     }
1550 }
1551 
qemu_ram_get_idstr(RAMBlock * rb)1552 const char *qemu_ram_get_idstr(RAMBlock *rb)
1553 {
1554     return rb->idstr;
1555 }
1556 
qemu_ram_get_host_addr(RAMBlock * rb)1557 void *qemu_ram_get_host_addr(RAMBlock *rb)
1558 {
1559     return rb->host;
1560 }
1561 
qemu_ram_get_offset(RAMBlock * rb)1562 ram_addr_t qemu_ram_get_offset(RAMBlock *rb)
1563 {
1564     return rb->offset;
1565 }
1566 
qemu_ram_get_used_length(RAMBlock * rb)1567 ram_addr_t qemu_ram_get_used_length(RAMBlock *rb)
1568 {
1569     return rb->used_length;
1570 }
1571 
qemu_ram_get_max_length(RAMBlock * rb)1572 ram_addr_t qemu_ram_get_max_length(RAMBlock *rb)
1573 {
1574     return rb->max_length;
1575 }
1576 
qemu_ram_is_shared(RAMBlock * rb)1577 bool qemu_ram_is_shared(RAMBlock *rb)
1578 {
1579     return rb->flags & RAM_SHARED;
1580 }
1581 
qemu_ram_is_noreserve(RAMBlock * rb)1582 bool qemu_ram_is_noreserve(RAMBlock *rb)
1583 {
1584     return rb->flags & RAM_NORESERVE;
1585 }
1586 
1587 /* Note: Only set at the start of postcopy */
qemu_ram_is_uf_zeroable(RAMBlock * rb)1588 bool qemu_ram_is_uf_zeroable(RAMBlock *rb)
1589 {
1590     return rb->flags & RAM_UF_ZEROPAGE;
1591 }
1592 
qemu_ram_set_uf_zeroable(RAMBlock * rb)1593 void qemu_ram_set_uf_zeroable(RAMBlock *rb)
1594 {
1595     rb->flags |= RAM_UF_ZEROPAGE;
1596 }
1597 
qemu_ram_is_migratable(RAMBlock * rb)1598 bool qemu_ram_is_migratable(RAMBlock *rb)
1599 {
1600     return rb->flags & RAM_MIGRATABLE;
1601 }
1602 
qemu_ram_set_migratable(RAMBlock * rb)1603 void qemu_ram_set_migratable(RAMBlock *rb)
1604 {
1605     rb->flags |= RAM_MIGRATABLE;
1606 }
1607 
qemu_ram_unset_migratable(RAMBlock * rb)1608 void qemu_ram_unset_migratable(RAMBlock *rb)
1609 {
1610     rb->flags &= ~RAM_MIGRATABLE;
1611 }
1612 
qemu_ram_is_named_file(RAMBlock * rb)1613 bool qemu_ram_is_named_file(RAMBlock *rb)
1614 {
1615     return rb->flags & RAM_NAMED_FILE;
1616 }
1617 
qemu_ram_get_fd(RAMBlock * rb)1618 int qemu_ram_get_fd(RAMBlock *rb)
1619 {
1620     return rb->fd;
1621 }
1622 
1623 /* Called with the BQL held.  */
qemu_ram_set_idstr(RAMBlock * new_block,const char * name,DeviceState * dev)1624 void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
1625 {
1626     RAMBlock *block;
1627 
1628     assert(new_block);
1629     assert(!new_block->idstr[0]);
1630 
1631     if (dev) {
1632         char *id = qdev_get_dev_path(dev);
1633         if (id) {
1634             snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1635             g_free(id);
1636         }
1637     }
1638     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1639 
1640     RCU_READ_LOCK_GUARD();
1641     RAMBLOCK_FOREACH(block) {
1642         if (block != new_block &&
1643             !strcmp(block->idstr, new_block->idstr)) {
1644             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1645                     new_block->idstr);
1646             abort();
1647         }
1648     }
1649 }
1650 
1651 /* Called with the BQL held.  */
qemu_ram_unset_idstr(RAMBlock * block)1652 void qemu_ram_unset_idstr(RAMBlock *block)
1653 {
1654     /* FIXME: arch_init.c assumes that this is not called throughout
1655      * migration.  Ignore the problem since hot-unplug during migration
1656      * does not work anyway.
1657      */
1658     if (block) {
1659         memset(block->idstr, 0, sizeof(block->idstr));
1660     }
1661 }
1662 
qemu_ram_pagesize(RAMBlock * rb)1663 size_t qemu_ram_pagesize(RAMBlock *rb)
1664 {
1665     return rb->page_size;
1666 }
1667 
1668 /* Returns the largest size of page in use */
qemu_ram_pagesize_largest(void)1669 size_t qemu_ram_pagesize_largest(void)
1670 {
1671     RAMBlock *block;
1672     size_t largest = 0;
1673 
1674     RAMBLOCK_FOREACH(block) {
1675         largest = MAX(largest, qemu_ram_pagesize(block));
1676     }
1677 
1678     return largest;
1679 }
1680 
memory_try_enable_merging(void * addr,size_t len)1681 static int memory_try_enable_merging(void *addr, size_t len)
1682 {
1683     if (!machine_mem_merge(current_machine)) {
1684         /* disabled by the user */
1685         return 0;
1686     }
1687 
1688     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1689 }
1690 
1691 /*
1692  * Resizing RAM while migrating can result in the migration being canceled.
1693  * Care has to be taken if the guest might have already detected the memory.
1694  *
1695  * As memory core doesn't know how is memory accessed, it is up to
1696  * resize callback to update device state and/or add assertions to detect
1697  * misuse, if necessary.
1698  */
qemu_ram_resize(RAMBlock * block,ram_addr_t newsize,Error ** errp)1699 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
1700 {
1701     const ram_addr_t oldsize = block->used_length;
1702     const ram_addr_t unaligned_size = newsize;
1703 
1704     assert(block);
1705 
1706     newsize = TARGET_PAGE_ALIGN(newsize);
1707     newsize = REAL_HOST_PAGE_ALIGN(newsize);
1708 
1709     if (block->used_length == newsize) {
1710         /*
1711          * We don't have to resize the ram block (which only knows aligned
1712          * sizes), however, we have to notify if the unaligned size changed.
1713          */
1714         if (unaligned_size != memory_region_size(block->mr)) {
1715             memory_region_set_size(block->mr, unaligned_size);
1716             if (block->resized) {
1717                 block->resized(block->idstr, unaligned_size, block->host);
1718             }
1719         }
1720         return 0;
1721     }
1722 
1723     if (!(block->flags & RAM_RESIZEABLE)) {
1724         error_setg_errno(errp, EINVAL,
1725                          "Size mismatch: %s: 0x" RAM_ADDR_FMT
1726                          " != 0x" RAM_ADDR_FMT, block->idstr,
1727                          newsize, block->used_length);
1728         return -EINVAL;
1729     }
1730 
1731     if (block->max_length < newsize) {
1732         error_setg_errno(errp, EINVAL,
1733                          "Size too large: %s: 0x" RAM_ADDR_FMT
1734                          " > 0x" RAM_ADDR_FMT, block->idstr,
1735                          newsize, block->max_length);
1736         return -EINVAL;
1737     }
1738 
1739     /* Notify before modifying the ram block and touching the bitmaps. */
1740     if (block->host) {
1741         ram_block_notify_resize(block->host, oldsize, newsize);
1742     }
1743 
1744     cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
1745     block->used_length = newsize;
1746     cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
1747                                         DIRTY_CLIENTS_ALL);
1748     memory_region_set_size(block->mr, unaligned_size);
1749     if (block->resized) {
1750         block->resized(block->idstr, unaligned_size, block->host);
1751     }
1752     return 0;
1753 }
1754 
1755 /*
1756  * Trigger sync on the given ram block for range [start, start + length]
1757  * with the backing store if one is available.
1758  * Otherwise no-op.
1759  * @Note: this is supposed to be a synchronous op.
1760  */
qemu_ram_msync(RAMBlock * block,ram_addr_t start,ram_addr_t length)1761 void qemu_ram_msync(RAMBlock *block, ram_addr_t start, ram_addr_t length)
1762 {
1763     /* The requested range should fit in within the block range */
1764     g_assert((start + length) <= block->used_length);
1765 
1766 #ifdef CONFIG_LIBPMEM
1767     /* The lack of support for pmem should not block the sync */
1768     if (ramblock_is_pmem(block)) {
1769         void *addr = ramblock_ptr(block, start);
1770         pmem_persist(addr, length);
1771         return;
1772     }
1773 #endif
1774     if (block->fd >= 0) {
1775         /**
1776          * Case there is no support for PMEM or the memory has not been
1777          * specified as persistent (or is not one) - use the msync.
1778          * Less optimal but still achieves the same goal
1779          */
1780         void *addr = ramblock_ptr(block, start);
1781         if (qemu_msync(addr, length, block->fd)) {
1782             warn_report("%s: failed to sync memory range: start: "
1783                     RAM_ADDR_FMT " length: " RAM_ADDR_FMT,
1784                     __func__, start, length);
1785         }
1786     }
1787 }
1788 
1789 /* Called with ram_list.mutex held */
dirty_memory_extend(ram_addr_t new_ram_size)1790 static void dirty_memory_extend(ram_addr_t new_ram_size)
1791 {
1792     unsigned int old_num_blocks = ram_list.num_dirty_blocks;
1793     unsigned int new_num_blocks = DIV_ROUND_UP(new_ram_size,
1794                                                DIRTY_MEMORY_BLOCK_SIZE);
1795     int i;
1796 
1797     /* Only need to extend if block count increased */
1798     if (new_num_blocks <= old_num_blocks) {
1799         return;
1800     }
1801 
1802     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1803         DirtyMemoryBlocks *old_blocks;
1804         DirtyMemoryBlocks *new_blocks;
1805         int j;
1806 
1807         old_blocks = qatomic_rcu_read(&ram_list.dirty_memory[i]);
1808         new_blocks = g_malloc(sizeof(*new_blocks) +
1809                               sizeof(new_blocks->blocks[0]) * new_num_blocks);
1810 
1811         if (old_num_blocks) {
1812             memcpy(new_blocks->blocks, old_blocks->blocks,
1813                    old_num_blocks * sizeof(old_blocks->blocks[0]));
1814         }
1815 
1816         for (j = old_num_blocks; j < new_num_blocks; j++) {
1817             new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
1818         }
1819 
1820         qatomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
1821 
1822         if (old_blocks) {
1823             g_free_rcu(old_blocks, rcu);
1824         }
1825     }
1826 
1827     ram_list.num_dirty_blocks = new_num_blocks;
1828 }
1829 
ram_block_add(RAMBlock * new_block,Error ** errp)1830 static void ram_block_add(RAMBlock *new_block, Error **errp)
1831 {
1832     const bool noreserve = qemu_ram_is_noreserve(new_block);
1833     const bool shared = qemu_ram_is_shared(new_block);
1834     RAMBlock *block;
1835     RAMBlock *last_block = NULL;
1836     bool free_on_error = false;
1837     ram_addr_t ram_size;
1838     Error *err = NULL;
1839 
1840     qemu_mutex_lock_ramlist();
1841     new_block->offset = find_ram_offset(new_block->max_length);
1842 
1843     if (!new_block->host) {
1844         if (xen_enabled()) {
1845             xen_ram_alloc(new_block->offset, new_block->max_length,
1846                           new_block->mr, &err);
1847             if (err) {
1848                 error_propagate(errp, err);
1849                 qemu_mutex_unlock_ramlist();
1850                 return;
1851             }
1852         } else {
1853             new_block->host = qemu_anon_ram_alloc(new_block->max_length,
1854                                                   &new_block->mr->align,
1855                                                   shared, noreserve);
1856             if (!new_block->host) {
1857                 error_setg_errno(errp, errno,
1858                                  "cannot set up guest memory '%s'",
1859                                  memory_region_name(new_block->mr));
1860                 qemu_mutex_unlock_ramlist();
1861                 return;
1862             }
1863             memory_try_enable_merging(new_block->host, new_block->max_length);
1864             free_on_error = true;
1865         }
1866     }
1867 
1868     if (new_block->flags & RAM_GUEST_MEMFD) {
1869         int ret;
1870 
1871         assert(kvm_enabled());
1872         assert(new_block->guest_memfd < 0);
1873 
1874         ret = ram_block_discard_require(true);
1875         if (ret < 0) {
1876             error_setg_errno(errp, -ret,
1877                              "cannot set up private guest memory: discard currently blocked");
1878             error_append_hint(errp, "Are you using assigned devices?\n");
1879             goto out_free;
1880         }
1881 
1882         new_block->guest_memfd = kvm_create_guest_memfd(new_block->max_length,
1883                                                         0, errp);
1884         if (new_block->guest_memfd < 0) {
1885             qemu_mutex_unlock_ramlist();
1886             goto out_free;
1887         }
1888     }
1889 
1890     ram_size = (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS;
1891     dirty_memory_extend(ram_size);
1892     /* Keep the list sorted from biggest to smallest block.  Unlike QTAILQ,
1893      * QLIST (which has an RCU-friendly variant) does not have insertion at
1894      * tail, so save the last element in last_block.
1895      */
1896     RAMBLOCK_FOREACH(block) {
1897         last_block = block;
1898         if (block->max_length < new_block->max_length) {
1899             break;
1900         }
1901     }
1902     if (block) {
1903         QLIST_INSERT_BEFORE_RCU(block, new_block, next);
1904     } else if (last_block) {
1905         QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
1906     } else { /* list is empty */
1907         QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
1908     }
1909     ram_list.mru_block = NULL;
1910 
1911     /* Write list before version */
1912     smp_wmb();
1913     ram_list.version++;
1914     qemu_mutex_unlock_ramlist();
1915 
1916     cpu_physical_memory_set_dirty_range(new_block->offset,
1917                                         new_block->used_length,
1918                                         DIRTY_CLIENTS_ALL);
1919 
1920     if (new_block->host) {
1921         qemu_ram_setup_dump(new_block->host, new_block->max_length);
1922         qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
1923         /*
1924          * MADV_DONTFORK is also needed by KVM in absence of synchronous MMU
1925          * Configure it unless the machine is a qtest server, in which case
1926          * KVM is not used and it may be forked (eg for fuzzing purposes).
1927          */
1928         if (!qtest_enabled()) {
1929             qemu_madvise(new_block->host, new_block->max_length,
1930                          QEMU_MADV_DONTFORK);
1931         }
1932         ram_block_notify_add(new_block->host, new_block->used_length,
1933                              new_block->max_length);
1934     }
1935     return;
1936 
1937 out_free:
1938     if (free_on_error) {
1939         qemu_anon_ram_free(new_block->host, new_block->max_length);
1940         new_block->host = NULL;
1941     }
1942 }
1943 
1944 #ifdef CONFIG_POSIX
qemu_ram_alloc_from_fd(ram_addr_t size,MemoryRegion * mr,uint32_t ram_flags,int fd,off_t offset,Error ** errp)1945 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
1946                                  uint32_t ram_flags, int fd, off_t offset,
1947                                  Error **errp)
1948 {
1949     RAMBlock *new_block;
1950     Error *local_err = NULL;
1951     int64_t file_size, file_align;
1952 
1953     /* Just support these ram flags by now. */
1954     assert((ram_flags & ~(RAM_SHARED | RAM_PMEM | RAM_NORESERVE |
1955                           RAM_PROTECTED | RAM_NAMED_FILE | RAM_READONLY |
1956                           RAM_READONLY_FD | RAM_GUEST_MEMFD)) == 0);
1957 
1958     if (xen_enabled()) {
1959         error_setg(errp, "-mem-path not supported with Xen");
1960         return NULL;
1961     }
1962 
1963     if (kvm_enabled() && !kvm_has_sync_mmu()) {
1964         error_setg(errp,
1965                    "host lacks kvm mmu notifiers, -mem-path unsupported");
1966         return NULL;
1967     }
1968 
1969     size = TARGET_PAGE_ALIGN(size);
1970     size = REAL_HOST_PAGE_ALIGN(size);
1971 
1972     file_size = get_file_size(fd);
1973     if (file_size > offset && file_size < (offset + size)) {
1974         error_setg(errp, "backing store size 0x%" PRIx64
1975                    " does not match 'size' option 0x" RAM_ADDR_FMT,
1976                    file_size, size);
1977         return NULL;
1978     }
1979 
1980     file_align = get_file_align(fd);
1981     if (file_align > 0 && file_align > mr->align) {
1982         error_setg(errp, "backing store align 0x%" PRIx64
1983                    " is larger than 'align' option 0x%" PRIx64,
1984                    file_align, mr->align);
1985         return NULL;
1986     }
1987 
1988     new_block = g_malloc0(sizeof(*new_block));
1989     new_block->mr = mr;
1990     new_block->used_length = size;
1991     new_block->max_length = size;
1992     new_block->flags = ram_flags;
1993     new_block->guest_memfd = -1;
1994     new_block->host = file_ram_alloc(new_block, size, fd, !file_size, offset,
1995                                      errp);
1996     if (!new_block->host) {
1997         g_free(new_block);
1998         return NULL;
1999     }
2000 
2001     ram_block_add(new_block, &local_err);
2002     if (local_err) {
2003         g_free(new_block);
2004         error_propagate(errp, local_err);
2005         return NULL;
2006     }
2007     return new_block;
2008 
2009 }
2010 
2011 
qemu_ram_alloc_from_file(ram_addr_t size,MemoryRegion * mr,uint32_t ram_flags,const char * mem_path,off_t offset,Error ** errp)2012 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
2013                                    uint32_t ram_flags, const char *mem_path,
2014                                    off_t offset, Error **errp)
2015 {
2016     int fd;
2017     bool created;
2018     RAMBlock *block;
2019 
2020     fd = file_ram_open(mem_path, memory_region_name(mr),
2021                        !!(ram_flags & RAM_READONLY_FD), &created);
2022     if (fd < 0) {
2023         error_setg_errno(errp, -fd, "can't open backing store %s for guest RAM",
2024                          mem_path);
2025         if (!(ram_flags & RAM_READONLY_FD) && !(ram_flags & RAM_SHARED) &&
2026             fd == -EACCES) {
2027             /*
2028              * If we can open the file R/O (note: will never create a new file)
2029              * and we are dealing with a private mapping, there are still ways
2030              * to consume such files and get RAM instead of ROM.
2031              */
2032             fd = file_ram_open(mem_path, memory_region_name(mr), true,
2033                                &created);
2034             if (fd < 0) {
2035                 return NULL;
2036             }
2037             assert(!created);
2038             close(fd);
2039             error_append_hint(errp, "Consider opening the backing store"
2040                 " read-only but still creating writable RAM using"
2041                 " '-object memory-backend-file,readonly=on,rom=off...'"
2042                 " (see \"VM templating\" documentation)\n");
2043         }
2044         return NULL;
2045     }
2046 
2047     block = qemu_ram_alloc_from_fd(size, mr, ram_flags, fd, offset, errp);
2048     if (!block) {
2049         if (created) {
2050             unlink(mem_path);
2051         }
2052         close(fd);
2053         return NULL;
2054     }
2055 
2056     return block;
2057 }
2058 #endif
2059 
2060 static
qemu_ram_alloc_internal(ram_addr_t size,ram_addr_t max_size,void (* resized)(const char *,uint64_t length,void * host),void * host,uint32_t ram_flags,MemoryRegion * mr,Error ** errp)2061 RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
2062                                   void (*resized)(const char*,
2063                                                   uint64_t length,
2064                                                   void *host),
2065                                   void *host, uint32_t ram_flags,
2066                                   MemoryRegion *mr, Error **errp)
2067 {
2068     RAMBlock *new_block;
2069     Error *local_err = NULL;
2070     int align;
2071 
2072     assert((ram_flags & ~(RAM_SHARED | RAM_RESIZEABLE | RAM_PREALLOC |
2073                           RAM_NORESERVE | RAM_GUEST_MEMFD)) == 0);
2074     assert(!host ^ (ram_flags & RAM_PREALLOC));
2075 
2076     align = qemu_real_host_page_size();
2077     align = MAX(align, TARGET_PAGE_SIZE);
2078     size = ROUND_UP(size, align);
2079     max_size = ROUND_UP(max_size, align);
2080 
2081     new_block = g_malloc0(sizeof(*new_block));
2082     new_block->mr = mr;
2083     new_block->resized = resized;
2084     new_block->used_length = size;
2085     new_block->max_length = max_size;
2086     assert(max_size >= size);
2087     new_block->fd = -1;
2088     new_block->guest_memfd = -1;
2089     new_block->page_size = qemu_real_host_page_size();
2090     new_block->host = host;
2091     new_block->flags = ram_flags;
2092     ram_block_add(new_block, &local_err);
2093     if (local_err) {
2094         g_free(new_block);
2095         error_propagate(errp, local_err);
2096         return NULL;
2097     }
2098     return new_block;
2099 }
2100 
qemu_ram_alloc_from_ptr(ram_addr_t size,void * host,MemoryRegion * mr,Error ** errp)2101 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
2102                                    MemoryRegion *mr, Error **errp)
2103 {
2104     return qemu_ram_alloc_internal(size, size, NULL, host, RAM_PREALLOC, mr,
2105                                    errp);
2106 }
2107 
qemu_ram_alloc(ram_addr_t size,uint32_t ram_flags,MemoryRegion * mr,Error ** errp)2108 RAMBlock *qemu_ram_alloc(ram_addr_t size, uint32_t ram_flags,
2109                          MemoryRegion *mr, Error **errp)
2110 {
2111     assert((ram_flags & ~(RAM_SHARED | RAM_NORESERVE | RAM_GUEST_MEMFD)) == 0);
2112     return qemu_ram_alloc_internal(size, size, NULL, NULL, ram_flags, mr, errp);
2113 }
2114 
qemu_ram_alloc_resizeable(ram_addr_t size,ram_addr_t maxsz,void (* resized)(const char *,uint64_t length,void * host),MemoryRegion * mr,Error ** errp)2115 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
2116                                      void (*resized)(const char*,
2117                                                      uint64_t length,
2118                                                      void *host),
2119                                      MemoryRegion *mr, Error **errp)
2120 {
2121     return qemu_ram_alloc_internal(size, maxsz, resized, NULL,
2122                                    RAM_RESIZEABLE, mr, errp);
2123 }
2124 
reclaim_ramblock(RAMBlock * block)2125 static void reclaim_ramblock(RAMBlock *block)
2126 {
2127     if (block->flags & RAM_PREALLOC) {
2128         ;
2129     } else if (xen_enabled()) {
2130         xen_invalidate_map_cache_entry(block->host);
2131 #ifndef _WIN32
2132     } else if (block->fd >= 0) {
2133         qemu_ram_munmap(block->fd, block->host, block->max_length);
2134         close(block->fd);
2135 #endif
2136     } else {
2137         qemu_anon_ram_free(block->host, block->max_length);
2138     }
2139 
2140     if (block->guest_memfd >= 0) {
2141         close(block->guest_memfd);
2142         ram_block_discard_require(false);
2143     }
2144 
2145     g_free(block);
2146 }
2147 
qemu_ram_free(RAMBlock * block)2148 void qemu_ram_free(RAMBlock *block)
2149 {
2150     if (!block) {
2151         return;
2152     }
2153 
2154     if (block->host) {
2155         ram_block_notify_remove(block->host, block->used_length,
2156                                 block->max_length);
2157     }
2158 
2159     qemu_mutex_lock_ramlist();
2160     QLIST_REMOVE_RCU(block, next);
2161     ram_list.mru_block = NULL;
2162     /* Write list before version */
2163     smp_wmb();
2164     ram_list.version++;
2165     call_rcu(block, reclaim_ramblock, rcu);
2166     qemu_mutex_unlock_ramlist();
2167 }
2168 
2169 #ifndef _WIN32
qemu_ram_remap(ram_addr_t addr,ram_addr_t length)2170 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2171 {
2172     RAMBlock *block;
2173     ram_addr_t offset;
2174     int flags;
2175     void *area, *vaddr;
2176     int prot;
2177 
2178     RAMBLOCK_FOREACH(block) {
2179         offset = addr - block->offset;
2180         if (offset < block->max_length) {
2181             vaddr = ramblock_ptr(block, offset);
2182             if (block->flags & RAM_PREALLOC) {
2183                 ;
2184             } else if (xen_enabled()) {
2185                 abort();
2186             } else {
2187                 flags = MAP_FIXED;
2188                 flags |= block->flags & RAM_SHARED ?
2189                          MAP_SHARED : MAP_PRIVATE;
2190                 flags |= block->flags & RAM_NORESERVE ? MAP_NORESERVE : 0;
2191                 prot = PROT_READ;
2192                 prot |= block->flags & RAM_READONLY ? 0 : PROT_WRITE;
2193                 if (block->fd >= 0) {
2194                     area = mmap(vaddr, length, prot, flags, block->fd,
2195                                 offset + block->fd_offset);
2196                 } else {
2197                     flags |= MAP_ANONYMOUS;
2198                     area = mmap(vaddr, length, prot, flags, -1, 0);
2199                 }
2200                 if (area != vaddr) {
2201                     error_report("Could not remap addr: "
2202                                  RAM_ADDR_FMT "@" RAM_ADDR_FMT "",
2203                                  length, addr);
2204                     exit(1);
2205                 }
2206                 memory_try_enable_merging(vaddr, length);
2207                 qemu_ram_setup_dump(vaddr, length);
2208             }
2209         }
2210     }
2211 }
2212 #endif /* !_WIN32 */
2213 
2214 /*
2215  * Return a host pointer to guest's ram.
2216  * For Xen, foreign mappings get created if they don't already exist.
2217  *
2218  * @block: block for the RAM to lookup (optional and may be NULL).
2219  * @addr: address within the memory region.
2220  * @size: pointer to requested size (optional and may be NULL).
2221  *        size may get modified and return a value smaller than
2222  *        what was requested.
2223  * @lock: wether to lock the mapping in xen-mapcache until invalidated.
2224  * @is_write: hint wether to map RW or RO in the xen-mapcache.
2225  *            (optional and may always be set to true).
2226  *
2227  * Called within RCU critical section.
2228  */
qemu_ram_ptr_length(RAMBlock * block,ram_addr_t addr,hwaddr * size,bool lock,bool is_write)2229 static void *qemu_ram_ptr_length(RAMBlock *block, ram_addr_t addr,
2230                                  hwaddr *size, bool lock,
2231                                  bool is_write)
2232 {
2233     hwaddr len = 0;
2234 
2235     if (size && *size == 0) {
2236         return NULL;
2237     }
2238 
2239     if (block == NULL) {
2240         block = qemu_get_ram_block(addr);
2241         addr -= block->offset;
2242     }
2243     if (size) {
2244         *size = MIN(*size, block->max_length - addr);
2245         len = *size;
2246     }
2247 
2248     if (xen_enabled() && block->host == NULL) {
2249         /* We need to check if the requested address is in the RAM
2250          * because we don't want to map the entire memory in QEMU.
2251          * In that case just map the requested area.
2252          */
2253         if (xen_mr_is_memory(block->mr)) {
2254             return xen_map_cache(block->mr, block->offset + addr,
2255                                  len, block->offset,
2256                                  lock, lock, is_write);
2257         }
2258 
2259         block->host = xen_map_cache(block->mr, block->offset,
2260                                     block->max_length,
2261                                     block->offset,
2262                                     1, lock, is_write);
2263     }
2264 
2265     return ramblock_ptr(block, addr);
2266 }
2267 
2268 /*
2269  * Return a host pointer to ram allocated with qemu_ram_alloc.
2270  * This should not be used for general purpose DMA.  Use address_space_map
2271  * or address_space_rw instead. For local memory (e.g. video ram) that the
2272  * device owns, use memory_region_get_ram_ptr.
2273  *
2274  * Called within RCU critical section.
2275  */
qemu_map_ram_ptr(RAMBlock * ram_block,ram_addr_t addr)2276 void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
2277 {
2278     return qemu_ram_ptr_length(ram_block, addr, NULL, false, true);
2279 }
2280 
2281 /* Return the offset of a hostpointer within a ramblock */
qemu_ram_block_host_offset(RAMBlock * rb,void * host)2282 ram_addr_t qemu_ram_block_host_offset(RAMBlock *rb, void *host)
2283 {
2284     ram_addr_t res = (uint8_t *)host - (uint8_t *)rb->host;
2285     assert((uintptr_t)host >= (uintptr_t)rb->host);
2286     assert(res < rb->max_length);
2287 
2288     return res;
2289 }
2290 
qemu_ram_block_from_host(void * ptr,bool round_offset,ram_addr_t * offset)2291 RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
2292                                    ram_addr_t *offset)
2293 {
2294     RAMBlock *block;
2295     uint8_t *host = ptr;
2296 
2297     if (xen_enabled()) {
2298         ram_addr_t ram_addr;
2299         RCU_READ_LOCK_GUARD();
2300         ram_addr = xen_ram_addr_from_mapcache(ptr);
2301         if (ram_addr == RAM_ADDR_INVALID) {
2302             return NULL;
2303         }
2304 
2305         block = qemu_get_ram_block(ram_addr);
2306         if (block) {
2307             *offset = ram_addr - block->offset;
2308         }
2309         return block;
2310     }
2311 
2312     RCU_READ_LOCK_GUARD();
2313     block = qatomic_rcu_read(&ram_list.mru_block);
2314     if (block && block->host && host - block->host < block->max_length) {
2315         goto found;
2316     }
2317 
2318     RAMBLOCK_FOREACH(block) {
2319         /* This case append when the block is not mapped. */
2320         if (block->host == NULL) {
2321             continue;
2322         }
2323         if (host - block->host < block->max_length) {
2324             goto found;
2325         }
2326     }
2327 
2328     return NULL;
2329 
2330 found:
2331     *offset = (host - block->host);
2332     if (round_offset) {
2333         *offset &= TARGET_PAGE_MASK;
2334     }
2335     return block;
2336 }
2337 
2338 /*
2339  * Finds the named RAMBlock
2340  *
2341  * name: The name of RAMBlock to find
2342  *
2343  * Returns: RAMBlock (or NULL if not found)
2344  */
qemu_ram_block_by_name(const char * name)2345 RAMBlock *qemu_ram_block_by_name(const char *name)
2346 {
2347     RAMBlock *block;
2348 
2349     RAMBLOCK_FOREACH(block) {
2350         if (!strcmp(name, block->idstr)) {
2351             return block;
2352         }
2353     }
2354 
2355     return NULL;
2356 }
2357 
2358 /*
2359  * Some of the system routines need to translate from a host pointer
2360  * (typically a TLB entry) back to a ram offset.
2361  */
qemu_ram_addr_from_host(void * ptr)2362 ram_addr_t qemu_ram_addr_from_host(void *ptr)
2363 {
2364     RAMBlock *block;
2365     ram_addr_t offset;
2366 
2367     block = qemu_ram_block_from_host(ptr, false, &offset);
2368     if (!block) {
2369         return RAM_ADDR_INVALID;
2370     }
2371 
2372     return block->offset + offset;
2373 }
2374 
qemu_ram_addr_from_host_nofail(void * ptr)2375 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
2376 {
2377     ram_addr_t ram_addr;
2378 
2379     ram_addr = qemu_ram_addr_from_host(ptr);
2380     if (ram_addr == RAM_ADDR_INVALID) {
2381         error_report("Bad ram pointer %p", ptr);
2382         abort();
2383     }
2384     return ram_addr;
2385 }
2386 
2387 static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
2388                                  MemTxAttrs attrs, void *buf, hwaddr len);
2389 static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
2390                                   const void *buf, hwaddr len);
2391 static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
2392                                   bool is_write, MemTxAttrs attrs);
2393 
subpage_read(void * opaque,hwaddr addr,uint64_t * data,unsigned len,MemTxAttrs attrs)2394 static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
2395                                 unsigned len, MemTxAttrs attrs)
2396 {
2397     subpage_t *subpage = opaque;
2398     uint8_t buf[8];
2399     MemTxResult res;
2400 
2401 #if defined(DEBUG_SUBPAGE)
2402     printf("%s: subpage %p len %u addr " HWADDR_FMT_plx "\n", __func__,
2403            subpage, len, addr);
2404 #endif
2405     res = flatview_read(subpage->fv, addr + subpage->base, attrs, buf, len);
2406     if (res) {
2407         return res;
2408     }
2409     *data = ldn_p(buf, len);
2410     return MEMTX_OK;
2411 }
2412 
subpage_write(void * opaque,hwaddr addr,uint64_t value,unsigned len,MemTxAttrs attrs)2413 static MemTxResult subpage_write(void *opaque, hwaddr addr,
2414                                  uint64_t value, unsigned len, MemTxAttrs attrs)
2415 {
2416     subpage_t *subpage = opaque;
2417     uint8_t buf[8];
2418 
2419 #if defined(DEBUG_SUBPAGE)
2420     printf("%s: subpage %p len %u addr " HWADDR_FMT_plx
2421            " value %"PRIx64"\n",
2422            __func__, subpage, len, addr, value);
2423 #endif
2424     stn_p(buf, len, value);
2425     return flatview_write(subpage->fv, addr + subpage->base, attrs, buf, len);
2426 }
2427 
subpage_accepts(void * opaque,hwaddr addr,unsigned len,bool is_write,MemTxAttrs attrs)2428 static bool subpage_accepts(void *opaque, hwaddr addr,
2429                             unsigned len, bool is_write,
2430                             MemTxAttrs attrs)
2431 {
2432     subpage_t *subpage = opaque;
2433 #if defined(DEBUG_SUBPAGE)
2434     printf("%s: subpage %p %c len %u addr " HWADDR_FMT_plx "\n",
2435            __func__, subpage, is_write ? 'w' : 'r', len, addr);
2436 #endif
2437 
2438     return flatview_access_valid(subpage->fv, addr + subpage->base,
2439                                  len, is_write, attrs);
2440 }
2441 
2442 static const MemoryRegionOps subpage_ops = {
2443     .read_with_attrs = subpage_read,
2444     .write_with_attrs = subpage_write,
2445     .impl.min_access_size = 1,
2446     .impl.max_access_size = 8,
2447     .valid.min_access_size = 1,
2448     .valid.max_access_size = 8,
2449     .valid.accepts = subpage_accepts,
2450     .endianness = DEVICE_NATIVE_ENDIAN,
2451 };
2452 
subpage_register(subpage_t * mmio,uint32_t start,uint32_t end,uint16_t section)2453 static int subpage_register(subpage_t *mmio, uint32_t start, uint32_t end,
2454                             uint16_t section)
2455 {
2456     int idx, eidx;
2457 
2458     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2459         return -1;
2460     idx = SUBPAGE_IDX(start);
2461     eidx = SUBPAGE_IDX(end);
2462 #if defined(DEBUG_SUBPAGE)
2463     printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
2464            __func__, mmio, start, end, idx, eidx, section);
2465 #endif
2466     for (; idx <= eidx; idx++) {
2467         mmio->sub_section[idx] = section;
2468     }
2469 
2470     return 0;
2471 }
2472 
subpage_init(FlatView * fv,hwaddr base)2473 static subpage_t *subpage_init(FlatView *fv, hwaddr base)
2474 {
2475     subpage_t *mmio;
2476 
2477     /* mmio->sub_section is set to PHYS_SECTION_UNASSIGNED with g_malloc0 */
2478     mmio = g_malloc0(sizeof(subpage_t) + TARGET_PAGE_SIZE * sizeof(uint16_t));
2479     mmio->fv = fv;
2480     mmio->base = base;
2481     memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
2482                           NULL, TARGET_PAGE_SIZE);
2483     mmio->iomem.subpage = true;
2484 #if defined(DEBUG_SUBPAGE)
2485     printf("%s: %p base " HWADDR_FMT_plx " len %08x\n", __func__,
2486            mmio, base, TARGET_PAGE_SIZE);
2487 #endif
2488 
2489     return mmio;
2490 }
2491 
dummy_section(PhysPageMap * map,FlatView * fv,MemoryRegion * mr)2492 static uint16_t dummy_section(PhysPageMap *map, FlatView *fv, MemoryRegion *mr)
2493 {
2494     assert(fv);
2495     MemoryRegionSection section = {
2496         .fv = fv,
2497         .mr = mr,
2498         .offset_within_address_space = 0,
2499         .offset_within_region = 0,
2500         .size = int128_2_64(),
2501     };
2502 
2503     return phys_section_add(map, &section);
2504 }
2505 
iotlb_to_section(CPUState * cpu,hwaddr index,MemTxAttrs attrs)2506 MemoryRegionSection *iotlb_to_section(CPUState *cpu,
2507                                       hwaddr index, MemTxAttrs attrs)
2508 {
2509     int asidx = cpu_asidx_from_attrs(cpu, attrs);
2510     CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
2511     AddressSpaceDispatch *d = cpuas->memory_dispatch;
2512     int section_index = index & ~TARGET_PAGE_MASK;
2513     MemoryRegionSection *ret;
2514 
2515     assert(section_index < d->map.sections_nb);
2516     ret = d->map.sections + section_index;
2517     assert(ret->mr);
2518     assert(ret->mr->ops);
2519 
2520     return ret;
2521 }
2522 
io_mem_init(void)2523 static void io_mem_init(void)
2524 {
2525     memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
2526                           NULL, UINT64_MAX);
2527 }
2528 
address_space_dispatch_new(FlatView * fv)2529 AddressSpaceDispatch *address_space_dispatch_new(FlatView *fv)
2530 {
2531     AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
2532     uint16_t n;
2533 
2534     n = dummy_section(&d->map, fv, &io_mem_unassigned);
2535     assert(n == PHYS_SECTION_UNASSIGNED);
2536 
2537     d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
2538 
2539     return d;
2540 }
2541 
address_space_dispatch_free(AddressSpaceDispatch * d)2542 void address_space_dispatch_free(AddressSpaceDispatch *d)
2543 {
2544     phys_sections_free(&d->map);
2545     g_free(d);
2546 }
2547 
do_nothing(CPUState * cpu,run_on_cpu_data d)2548 static void do_nothing(CPUState *cpu, run_on_cpu_data d)
2549 {
2550 }
2551 
tcg_log_global_after_sync(MemoryListener * listener)2552 static void tcg_log_global_after_sync(MemoryListener *listener)
2553 {
2554     CPUAddressSpace *cpuas;
2555 
2556     /* Wait for the CPU to end the current TB.  This avoids the following
2557      * incorrect race:
2558      *
2559      *      vCPU                         migration
2560      *      ----------------------       -------------------------
2561      *      TLB check -> slow path
2562      *        notdirty_mem_write
2563      *          write to RAM
2564      *          mark dirty
2565      *                                   clear dirty flag
2566      *      TLB check -> fast path
2567      *                                   read memory
2568      *        write to RAM
2569      *
2570      * by pushing the migration thread's memory read after the vCPU thread has
2571      * written the memory.
2572      */
2573     if (replay_mode == REPLAY_MODE_NONE) {
2574         /*
2575          * VGA can make calls to this function while updating the screen.
2576          * In record/replay mode this causes a deadlock, because
2577          * run_on_cpu waits for rr mutex. Therefore no races are possible
2578          * in this case and no need for making run_on_cpu when
2579          * record/replay is enabled.
2580          */
2581         cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2582         run_on_cpu(cpuas->cpu, do_nothing, RUN_ON_CPU_NULL);
2583     }
2584 }
2585 
tcg_commit_cpu(CPUState * cpu,run_on_cpu_data data)2586 static void tcg_commit_cpu(CPUState *cpu, run_on_cpu_data data)
2587 {
2588     CPUAddressSpace *cpuas = data.host_ptr;
2589 
2590     cpuas->memory_dispatch = address_space_to_dispatch(cpuas->as);
2591     tlb_flush(cpu);
2592 }
2593 
tcg_commit(MemoryListener * listener)2594 static void tcg_commit(MemoryListener *listener)
2595 {
2596     CPUAddressSpace *cpuas;
2597     CPUState *cpu;
2598 
2599     assert(tcg_enabled());
2600     /* since each CPU stores ram addresses in its TLB cache, we must
2601        reset the modified entries */
2602     cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
2603     cpu = cpuas->cpu;
2604 
2605     /*
2606      * Defer changes to as->memory_dispatch until the cpu is quiescent.
2607      * Otherwise we race between (1) other cpu threads and (2) ongoing
2608      * i/o for the current cpu thread, with data cached by mmu_lookup().
2609      *
2610      * In addition, queueing the work function will kick the cpu back to
2611      * the main loop, which will end the RCU critical section and reclaim
2612      * the memory data structures.
2613      *
2614      * That said, the listener is also called during realize, before
2615      * all of the tcg machinery for run-on is initialized: thus halt_cond.
2616      */
2617     if (cpu->halt_cond) {
2618         async_run_on_cpu(cpu, tcg_commit_cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2619     } else {
2620         tcg_commit_cpu(cpu, RUN_ON_CPU_HOST_PTR(cpuas));
2621     }
2622 }
2623 
memory_map_init(void)2624 static void memory_map_init(void)
2625 {
2626     system_memory = g_malloc(sizeof(*system_memory));
2627 
2628     memory_region_init(system_memory, NULL, "system", UINT64_MAX);
2629     address_space_init(&address_space_memory, system_memory, "memory");
2630 
2631     system_io = g_malloc(sizeof(*system_io));
2632     memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
2633                           65536);
2634     address_space_init(&address_space_io, system_io, "I/O");
2635 }
2636 
get_system_memory(void)2637 MemoryRegion *get_system_memory(void)
2638 {
2639     return system_memory;
2640 }
2641 
get_system_io(void)2642 MemoryRegion *get_system_io(void)
2643 {
2644     return system_io;
2645 }
2646 
invalidate_and_set_dirty(MemoryRegion * mr,hwaddr addr,hwaddr length)2647 static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
2648                                      hwaddr length)
2649 {
2650     uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
2651     ram_addr_t ramaddr = memory_region_get_ram_addr(mr);
2652 
2653     /* We know we're only called for RAM MemoryRegions */
2654     assert(ramaddr != RAM_ADDR_INVALID);
2655     addr += ramaddr;
2656 
2657     /* No early return if dirty_log_mask is or becomes 0, because
2658      * cpu_physical_memory_set_dirty_range will still call
2659      * xen_modified_memory.
2660      */
2661     if (dirty_log_mask) {
2662         dirty_log_mask =
2663             cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
2664     }
2665     if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
2666         assert(tcg_enabled());
2667         tb_invalidate_phys_range(addr, addr + length - 1);
2668         dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
2669     }
2670     cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
2671 }
2672 
memory_region_flush_rom_device(MemoryRegion * mr,hwaddr addr,hwaddr size)2673 void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size)
2674 {
2675     /*
2676      * In principle this function would work on other memory region types too,
2677      * but the ROM device use case is the only one where this operation is
2678      * necessary.  Other memory regions should use the
2679      * address_space_read/write() APIs.
2680      */
2681     assert(memory_region_is_romd(mr));
2682 
2683     invalidate_and_set_dirty(mr, addr, size);
2684 }
2685 
memory_access_size(MemoryRegion * mr,unsigned l,hwaddr addr)2686 int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
2687 {
2688     unsigned access_size_max = mr->ops->valid.max_access_size;
2689 
2690     /* Regions are assumed to support 1-4 byte accesses unless
2691        otherwise specified.  */
2692     if (access_size_max == 0) {
2693         access_size_max = 4;
2694     }
2695 
2696     /* Bound the maximum access by the alignment of the address.  */
2697     if (!mr->ops->impl.unaligned) {
2698         unsigned align_size_max = addr & -addr;
2699         if (align_size_max != 0 && align_size_max < access_size_max) {
2700             access_size_max = align_size_max;
2701         }
2702     }
2703 
2704     /* Don't attempt accesses larger than the maximum.  */
2705     if (l > access_size_max) {
2706         l = access_size_max;
2707     }
2708     l = pow2floor(l);
2709 
2710     return l;
2711 }
2712 
prepare_mmio_access(MemoryRegion * mr)2713 bool prepare_mmio_access(MemoryRegion *mr)
2714 {
2715     bool release_lock = false;
2716 
2717     if (!bql_locked()) {
2718         bql_lock();
2719         release_lock = true;
2720     }
2721     if (mr->flush_coalesced_mmio) {
2722         qemu_flush_coalesced_mmio_buffer();
2723     }
2724 
2725     return release_lock;
2726 }
2727 
2728 /**
2729  * flatview_access_allowed
2730  * @mr: #MemoryRegion to be accessed
2731  * @attrs: memory transaction attributes
2732  * @addr: address within that memory region
2733  * @len: the number of bytes to access
2734  *
2735  * Check if a memory transaction is allowed.
2736  *
2737  * Returns: true if transaction is allowed, false if denied.
2738  */
flatview_access_allowed(MemoryRegion * mr,MemTxAttrs attrs,hwaddr addr,hwaddr len)2739 static bool flatview_access_allowed(MemoryRegion *mr, MemTxAttrs attrs,
2740                                     hwaddr addr, hwaddr len)
2741 {
2742     if (likely(!attrs.memory)) {
2743         return true;
2744     }
2745     if (memory_region_is_ram(mr)) {
2746         return true;
2747     }
2748     qemu_log_mask(LOG_GUEST_ERROR,
2749                   "Invalid access to non-RAM device at "
2750                   "addr 0x%" HWADDR_PRIX ", size %" HWADDR_PRIu ", "
2751                   "region '%s'\n", addr, len, memory_region_name(mr));
2752     return false;
2753 }
2754 
flatview_write_continue_step(MemTxAttrs attrs,const uint8_t * buf,hwaddr len,hwaddr mr_addr,hwaddr * l,MemoryRegion * mr)2755 static MemTxResult flatview_write_continue_step(MemTxAttrs attrs,
2756                                                 const uint8_t *buf,
2757                                                 hwaddr len, hwaddr mr_addr,
2758                                                 hwaddr *l, MemoryRegion *mr)
2759 {
2760     if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
2761         return MEMTX_ACCESS_ERROR;
2762     }
2763 
2764     if (!memory_access_is_direct(mr, true)) {
2765         uint64_t val;
2766         MemTxResult result;
2767         bool release_lock = prepare_mmio_access(mr);
2768 
2769         *l = memory_access_size(mr, *l, mr_addr);
2770         /*
2771          * XXX: could force current_cpu to NULL to avoid
2772          * potential bugs
2773          */
2774 
2775         /*
2776          * Assure Coverity (and ourselves) that we are not going to OVERRUN
2777          * the buffer by following ldn_he_p().
2778          */
2779 #ifdef QEMU_STATIC_ANALYSIS
2780         assert((*l == 1 && len >= 1) ||
2781                (*l == 2 && len >= 2) ||
2782                (*l == 4 && len >= 4) ||
2783                (*l == 8 && len >= 8));
2784 #endif
2785         val = ldn_he_p(buf, *l);
2786         result = memory_region_dispatch_write(mr, mr_addr, val,
2787                                               size_memop(*l), attrs);
2788         if (release_lock) {
2789             bql_unlock();
2790         }
2791 
2792         return result;
2793     } else {
2794         /* RAM case */
2795         uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
2796                                                false, true);
2797 
2798         memmove(ram_ptr, buf, *l);
2799         invalidate_and_set_dirty(mr, mr_addr, *l);
2800 
2801         return MEMTX_OK;
2802     }
2803 }
2804 
2805 /* Called within RCU critical section.  */
flatview_write_continue(FlatView * fv,hwaddr addr,MemTxAttrs attrs,const void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)2806 static MemTxResult flatview_write_continue(FlatView *fv, hwaddr addr,
2807                                            MemTxAttrs attrs,
2808                                            const void *ptr,
2809                                            hwaddr len, hwaddr mr_addr,
2810                                            hwaddr l, MemoryRegion *mr)
2811 {
2812     MemTxResult result = MEMTX_OK;
2813     const uint8_t *buf = ptr;
2814 
2815     for (;;) {
2816         result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
2817                                                mr);
2818 
2819         len -= l;
2820         buf += l;
2821         addr += l;
2822 
2823         if (!len) {
2824             break;
2825         }
2826 
2827         l = len;
2828         mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
2829     }
2830 
2831     return result;
2832 }
2833 
2834 /* Called from RCU critical section.  */
flatview_write(FlatView * fv,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)2835 static MemTxResult flatview_write(FlatView *fv, hwaddr addr, MemTxAttrs attrs,
2836                                   const void *buf, hwaddr len)
2837 {
2838     hwaddr l;
2839     hwaddr mr_addr;
2840     MemoryRegion *mr;
2841 
2842     l = len;
2843     mr = flatview_translate(fv, addr, &mr_addr, &l, true, attrs);
2844     if (!flatview_access_allowed(mr, attrs, addr, len)) {
2845         return MEMTX_ACCESS_ERROR;
2846     }
2847     return flatview_write_continue(fv, addr, attrs, buf, len,
2848                                    mr_addr, l, mr);
2849 }
2850 
flatview_read_continue_step(MemTxAttrs attrs,uint8_t * buf,hwaddr len,hwaddr mr_addr,hwaddr * l,MemoryRegion * mr)2851 static MemTxResult flatview_read_continue_step(MemTxAttrs attrs, uint8_t *buf,
2852                                                hwaddr len, hwaddr mr_addr,
2853                                                hwaddr *l,
2854                                                MemoryRegion *mr)
2855 {
2856     if (!flatview_access_allowed(mr, attrs, mr_addr, *l)) {
2857         return MEMTX_ACCESS_ERROR;
2858     }
2859 
2860     if (!memory_access_is_direct(mr, false)) {
2861         /* I/O case */
2862         uint64_t val;
2863         MemTxResult result;
2864         bool release_lock = prepare_mmio_access(mr);
2865 
2866         *l = memory_access_size(mr, *l, mr_addr);
2867         result = memory_region_dispatch_read(mr, mr_addr, &val, size_memop(*l),
2868                                              attrs);
2869 
2870         /*
2871          * Assure Coverity (and ourselves) that we are not going to OVERRUN
2872          * the buffer by following stn_he_p().
2873          */
2874 #ifdef QEMU_STATIC_ANALYSIS
2875         assert((*l == 1 && len >= 1) ||
2876                (*l == 2 && len >= 2) ||
2877                (*l == 4 && len >= 4) ||
2878                (*l == 8 && len >= 8));
2879 #endif
2880         stn_he_p(buf, *l, val);
2881 
2882         if (release_lock) {
2883             bql_unlock();
2884         }
2885         return result;
2886     } else {
2887         /* RAM case */
2888         uint8_t *ram_ptr = qemu_ram_ptr_length(mr->ram_block, mr_addr, l,
2889                                                false, false);
2890 
2891         memcpy(buf, ram_ptr, *l);
2892 
2893         return MEMTX_OK;
2894     }
2895 }
2896 
2897 /* Called within RCU critical section.  */
flatview_read_continue(FlatView * fv,hwaddr addr,MemTxAttrs attrs,void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)2898 MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
2899                                    MemTxAttrs attrs, void *ptr,
2900                                    hwaddr len, hwaddr mr_addr, hwaddr l,
2901                                    MemoryRegion *mr)
2902 {
2903     MemTxResult result = MEMTX_OK;
2904     uint8_t *buf = ptr;
2905 
2906     fuzz_dma_read_cb(addr, len, mr);
2907     for (;;) {
2908         result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
2909 
2910         len -= l;
2911         buf += l;
2912         addr += l;
2913 
2914         if (!len) {
2915             break;
2916         }
2917 
2918         l = len;
2919         mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
2920     }
2921 
2922     return result;
2923 }
2924 
2925 /* Called from RCU critical section.  */
flatview_read(FlatView * fv,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len)2926 static MemTxResult flatview_read(FlatView *fv, hwaddr addr,
2927                                  MemTxAttrs attrs, void *buf, hwaddr len)
2928 {
2929     hwaddr l;
2930     hwaddr mr_addr;
2931     MemoryRegion *mr;
2932 
2933     l = len;
2934     mr = flatview_translate(fv, addr, &mr_addr, &l, false, attrs);
2935     if (!flatview_access_allowed(mr, attrs, addr, len)) {
2936         return MEMTX_ACCESS_ERROR;
2937     }
2938     return flatview_read_continue(fv, addr, attrs, buf, len,
2939                                   mr_addr, l, mr);
2940 }
2941 
address_space_read_full(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len)2942 MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
2943                                     MemTxAttrs attrs, void *buf, hwaddr len)
2944 {
2945     MemTxResult result = MEMTX_OK;
2946     FlatView *fv;
2947 
2948     if (len > 0) {
2949         RCU_READ_LOCK_GUARD();
2950         fv = address_space_to_flatview(as);
2951         result = flatview_read(fv, addr, attrs, buf, len);
2952     }
2953 
2954     return result;
2955 }
2956 
address_space_write(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)2957 MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
2958                                 MemTxAttrs attrs,
2959                                 const void *buf, hwaddr len)
2960 {
2961     MemTxResult result = MEMTX_OK;
2962     FlatView *fv;
2963 
2964     if (len > 0) {
2965         RCU_READ_LOCK_GUARD();
2966         fv = address_space_to_flatview(as);
2967         result = flatview_write(fv, addr, attrs, buf, len);
2968     }
2969 
2970     return result;
2971 }
2972 
address_space_rw(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,void * buf,hwaddr len,bool is_write)2973 MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
2974                              void *buf, hwaddr len, bool is_write)
2975 {
2976     if (is_write) {
2977         return address_space_write(as, addr, attrs, buf, len);
2978     } else {
2979         return address_space_read_full(as, addr, attrs, buf, len);
2980     }
2981 }
2982 
address_space_set(AddressSpace * as,hwaddr addr,uint8_t c,hwaddr len,MemTxAttrs attrs)2983 MemTxResult address_space_set(AddressSpace *as, hwaddr addr,
2984                               uint8_t c, hwaddr len, MemTxAttrs attrs)
2985 {
2986 #define FILLBUF_SIZE 512
2987     uint8_t fillbuf[FILLBUF_SIZE];
2988     int l;
2989     MemTxResult error = MEMTX_OK;
2990 
2991     memset(fillbuf, c, FILLBUF_SIZE);
2992     while (len > 0) {
2993         l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
2994         error |= address_space_write(as, addr, attrs, fillbuf, l);
2995         len -= l;
2996         addr += l;
2997     }
2998 
2999     return error;
3000 }
3001 
cpu_physical_memory_rw(hwaddr addr,void * buf,hwaddr len,bool is_write)3002 void cpu_physical_memory_rw(hwaddr addr, void *buf,
3003                             hwaddr len, bool is_write)
3004 {
3005     address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
3006                      buf, len, is_write);
3007 }
3008 
3009 enum write_rom_type {
3010     WRITE_DATA,
3011     FLUSH_CACHE,
3012 };
3013 
address_space_write_rom_internal(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * ptr,hwaddr len,enum write_rom_type type)3014 static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
3015                                                            hwaddr addr,
3016                                                            MemTxAttrs attrs,
3017                                                            const void *ptr,
3018                                                            hwaddr len,
3019                                                            enum write_rom_type type)
3020 {
3021     hwaddr l;
3022     uint8_t *ram_ptr;
3023     hwaddr addr1;
3024     MemoryRegion *mr;
3025     const uint8_t *buf = ptr;
3026 
3027     RCU_READ_LOCK_GUARD();
3028     while (len > 0) {
3029         l = len;
3030         mr = address_space_translate(as, addr, &addr1, &l, true, attrs);
3031 
3032         if (!(memory_region_is_ram(mr) ||
3033               memory_region_is_romd(mr))) {
3034             l = memory_access_size(mr, l, addr1);
3035         } else {
3036             /* ROM/RAM case */
3037             ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
3038             switch (type) {
3039             case WRITE_DATA:
3040                 memcpy(ram_ptr, buf, l);
3041                 invalidate_and_set_dirty(mr, addr1, l);
3042                 break;
3043             case FLUSH_CACHE:
3044                 flush_idcache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr, l);
3045                 break;
3046             }
3047         }
3048         len -= l;
3049         buf += l;
3050         addr += l;
3051     }
3052     return MEMTX_OK;
3053 }
3054 
3055 /* used for ROM loading : can write in RAM and ROM */
address_space_write_rom(AddressSpace * as,hwaddr addr,MemTxAttrs attrs,const void * buf,hwaddr len)3056 MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
3057                                     MemTxAttrs attrs,
3058                                     const void *buf, hwaddr len)
3059 {
3060     return address_space_write_rom_internal(as, addr, attrs,
3061                                             buf, len, WRITE_DATA);
3062 }
3063 
cpu_flush_icache_range(hwaddr start,hwaddr len)3064 void cpu_flush_icache_range(hwaddr start, hwaddr len)
3065 {
3066     /*
3067      * This function should do the same thing as an icache flush that was
3068      * triggered from within the guest. For TCG we are always cache coherent,
3069      * so there is no need to flush anything. For KVM / Xen we need to flush
3070      * the host's instruction cache at least.
3071      */
3072     if (tcg_enabled()) {
3073         return;
3074     }
3075 
3076     address_space_write_rom_internal(&address_space_memory,
3077                                      start, MEMTXATTRS_UNSPECIFIED,
3078                                      NULL, len, FLUSH_CACHE);
3079 }
3080 
3081 /*
3082  * A magic value stored in the first 8 bytes of the bounce buffer struct. Used
3083  * to detect illegal pointers passed to address_space_unmap.
3084  */
3085 #define BOUNCE_BUFFER_MAGIC 0xb4017ceb4ffe12ed
3086 
3087 typedef struct {
3088     uint64_t magic;
3089     MemoryRegion *mr;
3090     hwaddr addr;
3091     size_t len;
3092     uint8_t buffer[];
3093 } BounceBuffer;
3094 
3095 static void
address_space_unregister_map_client_do(AddressSpaceMapClient * client)3096 address_space_unregister_map_client_do(AddressSpaceMapClient *client)
3097 {
3098     QLIST_REMOVE(client, link);
3099     g_free(client);
3100 }
3101 
address_space_notify_map_clients_locked(AddressSpace * as)3102 static void address_space_notify_map_clients_locked(AddressSpace *as)
3103 {
3104     AddressSpaceMapClient *client;
3105 
3106     while (!QLIST_EMPTY(&as->map_client_list)) {
3107         client = QLIST_FIRST(&as->map_client_list);
3108         qemu_bh_schedule(client->bh);
3109         address_space_unregister_map_client_do(client);
3110     }
3111 }
3112 
address_space_register_map_client(AddressSpace * as,QEMUBH * bh)3113 void address_space_register_map_client(AddressSpace *as, QEMUBH *bh)
3114 {
3115     AddressSpaceMapClient *client = g_malloc(sizeof(*client));
3116 
3117     QEMU_LOCK_GUARD(&as->map_client_list_lock);
3118     client->bh = bh;
3119     QLIST_INSERT_HEAD(&as->map_client_list, client, link);
3120     /* Write map_client_list before reading bounce_buffer_size. */
3121     smp_mb();
3122     if (qatomic_read(&as->bounce_buffer_size) < as->max_bounce_buffer_size) {
3123         address_space_notify_map_clients_locked(as);
3124     }
3125 }
3126 
cpu_exec_init_all(void)3127 void cpu_exec_init_all(void)
3128 {
3129     qemu_mutex_init(&ram_list.mutex);
3130     /* The data structures we set up here depend on knowing the page size,
3131      * so no more changes can be made after this point.
3132      * In an ideal world, nothing we did before we had finished the
3133      * machine setup would care about the target page size, and we could
3134      * do this much later, rather than requiring board models to state
3135      * up front what their requirements are.
3136      */
3137     finalize_target_page_bits();
3138     io_mem_init();
3139     memory_map_init();
3140 }
3141 
address_space_unregister_map_client(AddressSpace * as,QEMUBH * bh)3142 void address_space_unregister_map_client(AddressSpace *as, QEMUBH *bh)
3143 {
3144     AddressSpaceMapClient *client;
3145 
3146     QEMU_LOCK_GUARD(&as->map_client_list_lock);
3147     QLIST_FOREACH(client, &as->map_client_list, link) {
3148         if (client->bh == bh) {
3149             address_space_unregister_map_client_do(client);
3150             break;
3151         }
3152     }
3153 }
3154 
address_space_notify_map_clients(AddressSpace * as)3155 static void address_space_notify_map_clients(AddressSpace *as)
3156 {
3157     QEMU_LOCK_GUARD(&as->map_client_list_lock);
3158     address_space_notify_map_clients_locked(as);
3159 }
3160 
flatview_access_valid(FlatView * fv,hwaddr addr,hwaddr len,bool is_write,MemTxAttrs attrs)3161 static bool flatview_access_valid(FlatView *fv, hwaddr addr, hwaddr len,
3162                                   bool is_write, MemTxAttrs attrs)
3163 {
3164     MemoryRegion *mr;
3165     hwaddr l, xlat;
3166 
3167     while (len > 0) {
3168         l = len;
3169         mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3170         if (!memory_access_is_direct(mr, is_write)) {
3171             l = memory_access_size(mr, l, addr);
3172             if (!memory_region_access_valid(mr, xlat, l, is_write, attrs)) {
3173                 return false;
3174             }
3175         }
3176 
3177         len -= l;
3178         addr += l;
3179     }
3180     return true;
3181 }
3182 
address_space_access_valid(AddressSpace * as,hwaddr addr,hwaddr len,bool is_write,MemTxAttrs attrs)3183 bool address_space_access_valid(AddressSpace *as, hwaddr addr,
3184                                 hwaddr len, bool is_write,
3185                                 MemTxAttrs attrs)
3186 {
3187     FlatView *fv;
3188 
3189     RCU_READ_LOCK_GUARD();
3190     fv = address_space_to_flatview(as);
3191     return flatview_access_valid(fv, addr, len, is_write, attrs);
3192 }
3193 
3194 static hwaddr
flatview_extend_translation(FlatView * fv,hwaddr addr,hwaddr target_len,MemoryRegion * mr,hwaddr base,hwaddr len,bool is_write,MemTxAttrs attrs)3195 flatview_extend_translation(FlatView *fv, hwaddr addr,
3196                             hwaddr target_len,
3197                             MemoryRegion *mr, hwaddr base, hwaddr len,
3198                             bool is_write, MemTxAttrs attrs)
3199 {
3200     hwaddr done = 0;
3201     hwaddr xlat;
3202     MemoryRegion *this_mr;
3203 
3204     for (;;) {
3205         target_len -= len;
3206         addr += len;
3207         done += len;
3208         if (target_len == 0) {
3209             return done;
3210         }
3211 
3212         len = target_len;
3213         this_mr = flatview_translate(fv, addr, &xlat,
3214                                      &len, is_write, attrs);
3215         if (this_mr != mr || xlat != base + done) {
3216             return done;
3217         }
3218     }
3219 }
3220 
3221 /* Map a physical memory region into a host virtual address.
3222  * May map a subset of the requested range, given by and returned in *plen.
3223  * May return NULL if resources needed to perform the mapping are exhausted.
3224  * Use only for reads OR writes - not for read-modify-write operations.
3225  * Use address_space_register_map_client() to know when retrying the map
3226  * operation is likely to succeed.
3227  */
address_space_map(AddressSpace * as,hwaddr addr,hwaddr * plen,bool is_write,MemTxAttrs attrs)3228 void *address_space_map(AddressSpace *as,
3229                         hwaddr addr,
3230                         hwaddr *plen,
3231                         bool is_write,
3232                         MemTxAttrs attrs)
3233 {
3234     hwaddr len = *plen;
3235     hwaddr l, xlat;
3236     MemoryRegion *mr;
3237     FlatView *fv;
3238 
3239     trace_address_space_map(as, addr, len, is_write, *(uint32_t *) &attrs);
3240 
3241     if (len == 0) {
3242         return NULL;
3243     }
3244 
3245     l = len;
3246     RCU_READ_LOCK_GUARD();
3247     fv = address_space_to_flatview(as);
3248     mr = flatview_translate(fv, addr, &xlat, &l, is_write, attrs);
3249 
3250     if (!memory_access_is_direct(mr, is_write)) {
3251         size_t used = qatomic_read(&as->bounce_buffer_size);
3252         for (;;) {
3253             hwaddr alloc = MIN(as->max_bounce_buffer_size - used, l);
3254             size_t new_size = used + alloc;
3255             size_t actual =
3256                 qatomic_cmpxchg(&as->bounce_buffer_size, used, new_size);
3257             if (actual == used) {
3258                 l = alloc;
3259                 break;
3260             }
3261             used = actual;
3262         }
3263 
3264         if (l == 0) {
3265             *plen = 0;
3266             return NULL;
3267         }
3268 
3269         BounceBuffer *bounce = g_malloc0(l + sizeof(BounceBuffer));
3270         bounce->magic = BOUNCE_BUFFER_MAGIC;
3271         memory_region_ref(mr);
3272         bounce->mr = mr;
3273         bounce->addr = addr;
3274         bounce->len = l;
3275 
3276         if (!is_write) {
3277             flatview_read(fv, addr, attrs,
3278                           bounce->buffer, l);
3279         }
3280 
3281         *plen = l;
3282         return bounce->buffer;
3283     }
3284 
3285     memory_region_ref(mr);
3286     *plen = flatview_extend_translation(fv, addr, len, mr, xlat,
3287                                         l, is_write, attrs);
3288     fuzz_dma_read_cb(addr, *plen, mr);
3289     return qemu_ram_ptr_length(mr->ram_block, xlat, plen, true, is_write);
3290 }
3291 
3292 /* Unmaps a memory region previously mapped by address_space_map().
3293  * Will also mark the memory as dirty if is_write is true.  access_len gives
3294  * the amount of memory that was actually read or written by the caller.
3295  */
address_space_unmap(AddressSpace * as,void * buffer,hwaddr len,bool is_write,hwaddr access_len)3296 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
3297                          bool is_write, hwaddr access_len)
3298 {
3299     MemoryRegion *mr;
3300     ram_addr_t addr1;
3301 
3302     mr = memory_region_from_host(buffer, &addr1);
3303     if (mr != NULL) {
3304         if (is_write) {
3305             invalidate_and_set_dirty(mr, addr1, access_len);
3306         }
3307         if (xen_enabled()) {
3308             xen_invalidate_map_cache_entry(buffer);
3309         }
3310         memory_region_unref(mr);
3311         return;
3312     }
3313 
3314 
3315     BounceBuffer *bounce = container_of(buffer, BounceBuffer, buffer);
3316     assert(bounce->magic == BOUNCE_BUFFER_MAGIC);
3317 
3318     if (is_write) {
3319         address_space_write(as, bounce->addr, MEMTXATTRS_UNSPECIFIED,
3320                             bounce->buffer, access_len);
3321     }
3322 
3323     qatomic_sub(&as->bounce_buffer_size, bounce->len);
3324     bounce->magic = ~BOUNCE_BUFFER_MAGIC;
3325     memory_region_unref(bounce->mr);
3326     g_free(bounce);
3327     /* Write bounce_buffer_size before reading map_client_list. */
3328     smp_mb();
3329     address_space_notify_map_clients(as);
3330 }
3331 
cpu_physical_memory_map(hwaddr addr,hwaddr * plen,bool is_write)3332 void *cpu_physical_memory_map(hwaddr addr,
3333                               hwaddr *plen,
3334                               bool is_write)
3335 {
3336     return address_space_map(&address_space_memory, addr, plen, is_write,
3337                              MEMTXATTRS_UNSPECIFIED);
3338 }
3339 
cpu_physical_memory_unmap(void * buffer,hwaddr len,bool is_write,hwaddr access_len)3340 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
3341                                bool is_write, hwaddr access_len)
3342 {
3343     return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
3344 }
3345 
3346 #define ARG1_DECL                AddressSpace *as
3347 #define ARG1                     as
3348 #define SUFFIX
3349 #define TRANSLATE(...)           address_space_translate(as, __VA_ARGS__)
3350 #define RCU_READ_LOCK(...)       rcu_read_lock()
3351 #define RCU_READ_UNLOCK(...)     rcu_read_unlock()
3352 #include "memory_ldst.c.inc"
3353 
address_space_cache_init(MemoryRegionCache * cache,AddressSpace * as,hwaddr addr,hwaddr len,bool is_write)3354 int64_t address_space_cache_init(MemoryRegionCache *cache,
3355                                  AddressSpace *as,
3356                                  hwaddr addr,
3357                                  hwaddr len,
3358                                  bool is_write)
3359 {
3360     AddressSpaceDispatch *d;
3361     hwaddr l;
3362     MemoryRegion *mr;
3363     Int128 diff;
3364 
3365     assert(len > 0);
3366 
3367     l = len;
3368     cache->fv = address_space_get_flatview(as);
3369     d = flatview_to_dispatch(cache->fv);
3370     cache->mrs = *address_space_translate_internal(d, addr, &cache->xlat, &l, true);
3371 
3372     /*
3373      * cache->xlat is now relative to cache->mrs.mr, not to the section itself.
3374      * Take that into account to compute how many bytes are there between
3375      * cache->xlat and the end of the section.
3376      */
3377     diff = int128_sub(cache->mrs.size,
3378                       int128_make64(cache->xlat - cache->mrs.offset_within_region));
3379     l = int128_get64(int128_min(diff, int128_make64(l)));
3380 
3381     mr = cache->mrs.mr;
3382     memory_region_ref(mr);
3383     if (memory_access_is_direct(mr, is_write)) {
3384         /* We don't care about the memory attributes here as we're only
3385          * doing this if we found actual RAM, which behaves the same
3386          * regardless of attributes; so UNSPECIFIED is fine.
3387          */
3388         l = flatview_extend_translation(cache->fv, addr, len, mr,
3389                                         cache->xlat, l, is_write,
3390                                         MEMTXATTRS_UNSPECIFIED);
3391         cache->ptr = qemu_ram_ptr_length(mr->ram_block, cache->xlat, &l, true,
3392                                          is_write);
3393     } else {
3394         cache->ptr = NULL;
3395     }
3396 
3397     cache->len = l;
3398     cache->is_write = is_write;
3399     return l;
3400 }
3401 
address_space_cache_invalidate(MemoryRegionCache * cache,hwaddr addr,hwaddr access_len)3402 void address_space_cache_invalidate(MemoryRegionCache *cache,
3403                                     hwaddr addr,
3404                                     hwaddr access_len)
3405 {
3406     assert(cache->is_write);
3407     if (likely(cache->ptr)) {
3408         invalidate_and_set_dirty(cache->mrs.mr, addr + cache->xlat, access_len);
3409     }
3410 }
3411 
address_space_cache_destroy(MemoryRegionCache * cache)3412 void address_space_cache_destroy(MemoryRegionCache *cache)
3413 {
3414     if (!cache->mrs.mr) {
3415         return;
3416     }
3417 
3418     if (xen_enabled()) {
3419         xen_invalidate_map_cache_entry(cache->ptr);
3420     }
3421     memory_region_unref(cache->mrs.mr);
3422     flatview_unref(cache->fv);
3423     cache->mrs.mr = NULL;
3424     cache->fv = NULL;
3425 }
3426 
3427 /* Called from RCU critical section.  This function has the same
3428  * semantics as address_space_translate, but it only works on a
3429  * predefined range of a MemoryRegion that was mapped with
3430  * address_space_cache_init.
3431  */
address_space_translate_cached(MemoryRegionCache * cache,hwaddr addr,hwaddr * xlat,hwaddr * plen,bool is_write,MemTxAttrs attrs)3432 static inline MemoryRegion *address_space_translate_cached(
3433     MemoryRegionCache *cache, hwaddr addr, hwaddr *xlat,
3434     hwaddr *plen, bool is_write, MemTxAttrs attrs)
3435 {
3436     MemoryRegionSection section;
3437     MemoryRegion *mr;
3438     IOMMUMemoryRegion *iommu_mr;
3439     AddressSpace *target_as;
3440 
3441     assert(!cache->ptr);
3442     *xlat = addr + cache->xlat;
3443 
3444     mr = cache->mrs.mr;
3445     iommu_mr = memory_region_get_iommu(mr);
3446     if (!iommu_mr) {
3447         /* MMIO region.  */
3448         return mr;
3449     }
3450 
3451     section = address_space_translate_iommu(iommu_mr, xlat, plen,
3452                                             NULL, is_write, true,
3453                                             &target_as, attrs);
3454     return section.mr;
3455 }
3456 
3457 /* Called within RCU critical section.  */
address_space_write_continue_cached(MemTxAttrs attrs,const void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)3458 static MemTxResult address_space_write_continue_cached(MemTxAttrs attrs,
3459                                                        const void *ptr,
3460                                                        hwaddr len,
3461                                                        hwaddr mr_addr,
3462                                                        hwaddr l,
3463                                                        MemoryRegion *mr)
3464 {
3465     MemTxResult result = MEMTX_OK;
3466     const uint8_t *buf = ptr;
3467 
3468     for (;;) {
3469         result |= flatview_write_continue_step(attrs, buf, len, mr_addr, &l,
3470                                                mr);
3471 
3472         len -= l;
3473         buf += l;
3474         mr_addr += l;
3475 
3476         if (!len) {
3477             break;
3478         }
3479 
3480         l = len;
3481     }
3482 
3483     return result;
3484 }
3485 
3486 /* Called within RCU critical section.  */
address_space_read_continue_cached(MemTxAttrs attrs,void * ptr,hwaddr len,hwaddr mr_addr,hwaddr l,MemoryRegion * mr)3487 static MemTxResult address_space_read_continue_cached(MemTxAttrs attrs,
3488                                                       void *ptr, hwaddr len,
3489                                                       hwaddr mr_addr, hwaddr l,
3490                                                       MemoryRegion *mr)
3491 {
3492     MemTxResult result = MEMTX_OK;
3493     uint8_t *buf = ptr;
3494 
3495     for (;;) {
3496         result |= flatview_read_continue_step(attrs, buf, len, mr_addr, &l, mr);
3497         len -= l;
3498         buf += l;
3499         mr_addr += l;
3500 
3501         if (!len) {
3502             break;
3503         }
3504         l = len;
3505     }
3506 
3507     return result;
3508 }
3509 
3510 /* Called from RCU critical section. address_space_read_cached uses this
3511  * out of line function when the target is an MMIO or IOMMU region.
3512  */
3513 MemTxResult
address_space_read_cached_slow(MemoryRegionCache * cache,hwaddr addr,void * buf,hwaddr len)3514 address_space_read_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3515                                    void *buf, hwaddr len)
3516 {
3517     hwaddr mr_addr, l;
3518     MemoryRegion *mr;
3519 
3520     l = len;
3521     mr = address_space_translate_cached(cache, addr, &mr_addr, &l, false,
3522                                         MEMTXATTRS_UNSPECIFIED);
3523     return address_space_read_continue_cached(MEMTXATTRS_UNSPECIFIED,
3524                                               buf, len, mr_addr, l, mr);
3525 }
3526 
3527 /* Called from RCU critical section. address_space_write_cached uses this
3528  * out of line function when the target is an MMIO or IOMMU region.
3529  */
3530 MemTxResult
address_space_write_cached_slow(MemoryRegionCache * cache,hwaddr addr,const void * buf,hwaddr len)3531 address_space_write_cached_slow(MemoryRegionCache *cache, hwaddr addr,
3532                                     const void *buf, hwaddr len)
3533 {
3534     hwaddr mr_addr, l;
3535     MemoryRegion *mr;
3536 
3537     l = len;
3538     mr = address_space_translate_cached(cache, addr, &mr_addr, &l, true,
3539                                         MEMTXATTRS_UNSPECIFIED);
3540     return address_space_write_continue_cached(MEMTXATTRS_UNSPECIFIED,
3541                                                buf, len, mr_addr, l, mr);
3542 }
3543 
3544 #define ARG1_DECL                MemoryRegionCache *cache
3545 #define ARG1                     cache
3546 #define SUFFIX                   _cached_slow
3547 #define TRANSLATE(...)           address_space_translate_cached(cache, __VA_ARGS__)
3548 #define RCU_READ_LOCK()          ((void)0)
3549 #define RCU_READ_UNLOCK()        ((void)0)
3550 #include "memory_ldst.c.inc"
3551 
3552 /* virtual memory access for debug (includes writing to ROM) */
cpu_memory_rw_debug(CPUState * cpu,vaddr addr,void * ptr,size_t len,bool is_write)3553 int cpu_memory_rw_debug(CPUState *cpu, vaddr addr,
3554                         void *ptr, size_t len, bool is_write)
3555 {
3556     hwaddr phys_addr;
3557     vaddr l, page;
3558     uint8_t *buf = ptr;
3559 
3560     cpu_synchronize_state(cpu);
3561     while (len > 0) {
3562         int asidx;
3563         MemTxAttrs attrs;
3564         MemTxResult res;
3565 
3566         page = addr & TARGET_PAGE_MASK;
3567         phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
3568         asidx = cpu_asidx_from_attrs(cpu, attrs);
3569         /* if no physical page mapped, return an error */
3570         if (phys_addr == -1)
3571             return -1;
3572         l = (page + TARGET_PAGE_SIZE) - addr;
3573         if (l > len)
3574             l = len;
3575         phys_addr += (addr & ~TARGET_PAGE_MASK);
3576         if (is_write) {
3577             res = address_space_write_rom(cpu->cpu_ases[asidx].as, phys_addr,
3578                                           attrs, buf, l);
3579         } else {
3580             res = address_space_read(cpu->cpu_ases[asidx].as, phys_addr,
3581                                      attrs, buf, l);
3582         }
3583         if (res != MEMTX_OK) {
3584             return -1;
3585         }
3586         len -= l;
3587         buf += l;
3588         addr += l;
3589     }
3590     return 0;
3591 }
3592 
cpu_physical_memory_is_io(hwaddr phys_addr)3593 bool cpu_physical_memory_is_io(hwaddr phys_addr)
3594 {
3595     MemoryRegion*mr;
3596     hwaddr l = 1;
3597 
3598     RCU_READ_LOCK_GUARD();
3599     mr = address_space_translate(&address_space_memory,
3600                                  phys_addr, &phys_addr, &l, false,
3601                                  MEMTXATTRS_UNSPECIFIED);
3602 
3603     return !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
3604 }
3605 
qemu_ram_foreach_block(RAMBlockIterFunc func,void * opaque)3606 int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
3607 {
3608     RAMBlock *block;
3609     int ret = 0;
3610 
3611     RCU_READ_LOCK_GUARD();
3612     RAMBLOCK_FOREACH(block) {
3613         ret = func(block, opaque);
3614         if (ret) {
3615             break;
3616         }
3617     }
3618     return ret;
3619 }
3620 
3621 /*
3622  * Unmap pages of memory from start to start+length such that
3623  * they a) read as 0, b) Trigger whatever fault mechanism
3624  * the OS provides for postcopy.
3625  * The pages must be unmapped by the end of the function.
3626  * Returns: 0 on success, none-0 on failure
3627  *
3628  */
ram_block_discard_range(RAMBlock * rb,uint64_t start,size_t length)3629 int ram_block_discard_range(RAMBlock *rb, uint64_t start, size_t length)
3630 {
3631     int ret = -1;
3632 
3633     uint8_t *host_startaddr = rb->host + start;
3634 
3635     if (!QEMU_PTR_IS_ALIGNED(host_startaddr, rb->page_size)) {
3636         error_report("%s: Unaligned start address: %p",
3637                      __func__, host_startaddr);
3638         goto err;
3639     }
3640 
3641     if ((start + length) <= rb->max_length) {
3642         bool need_madvise, need_fallocate;
3643         if (!QEMU_IS_ALIGNED(length, rb->page_size)) {
3644             error_report("%s: Unaligned length: %zx", __func__, length);
3645             goto err;
3646         }
3647 
3648         errno = ENOTSUP; /* If we are missing MADVISE etc */
3649 
3650         /* The logic here is messy;
3651          *    madvise DONTNEED fails for hugepages
3652          *    fallocate works on hugepages and shmem
3653          *    shared anonymous memory requires madvise REMOVE
3654          */
3655         need_madvise = (rb->page_size == qemu_real_host_page_size());
3656         need_fallocate = rb->fd != -1;
3657         if (need_fallocate) {
3658             /* For a file, this causes the area of the file to be zero'd
3659              * if read, and for hugetlbfs also causes it to be unmapped
3660              * so a userfault will trigger.
3661              */
3662 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
3663             /*
3664              * fallocate() will fail with readonly files. Let's print a
3665              * proper error message.
3666              */
3667             if (rb->flags & RAM_READONLY_FD) {
3668                 error_report("%s: Discarding RAM with readonly files is not"
3669                              " supported", __func__);
3670                 goto err;
3671 
3672             }
3673             /*
3674              * We'll discard data from the actual file, even though we only
3675              * have a MAP_PRIVATE mapping, possibly messing with other
3676              * MAP_PRIVATE/MAP_SHARED mappings. There is no easy way to
3677              * change that behavior whithout violating the promised
3678              * semantics of ram_block_discard_range().
3679              *
3680              * Only warn, because it works as long as nobody else uses that
3681              * file.
3682              */
3683             if (!qemu_ram_is_shared(rb)) {
3684                 warn_report_once("%s: Discarding RAM"
3685                                  " in private file mappings is possibly"
3686                                  " dangerous, because it will modify the"
3687                                  " underlying file and will affect other"
3688                                  " users of the file", __func__);
3689             }
3690 
3691             ret = fallocate(rb->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3692                             start, length);
3693             if (ret) {
3694                 ret = -errno;
3695                 error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)",
3696                              __func__, rb->idstr, start, length, ret);
3697                 goto err;
3698             }
3699 #else
3700             ret = -ENOSYS;
3701             error_report("%s: fallocate not available/file"
3702                          "%s:%" PRIx64 " +%zx (%d)",
3703                          __func__, rb->idstr, start, length, ret);
3704             goto err;
3705 #endif
3706         }
3707         if (need_madvise) {
3708             /* For normal RAM this causes it to be unmapped,
3709              * for shared memory it causes the local mapping to disappear
3710              * and to fall back on the file contents (which we just
3711              * fallocate'd away).
3712              */
3713 #if defined(CONFIG_MADVISE)
3714             if (qemu_ram_is_shared(rb) && rb->fd < 0) {
3715                 ret = madvise(host_startaddr, length, QEMU_MADV_REMOVE);
3716             } else {
3717                 ret = madvise(host_startaddr, length, QEMU_MADV_DONTNEED);
3718             }
3719             if (ret) {
3720                 ret = -errno;
3721                 error_report("%s: Failed to discard range "
3722                              "%s:%" PRIx64 " +%zx (%d)",
3723                              __func__, rb->idstr, start, length, ret);
3724                 goto err;
3725             }
3726 #else
3727             ret = -ENOSYS;
3728             error_report("%s: MADVISE not available %s:%" PRIx64 " +%zx (%d)",
3729                          __func__, rb->idstr, start, length, ret);
3730             goto err;
3731 #endif
3732         }
3733         trace_ram_block_discard_range(rb->idstr, host_startaddr, length,
3734                                       need_madvise, need_fallocate, ret);
3735     } else {
3736         error_report("%s: Overrun block '%s' (%" PRIu64 "/%zx/" RAM_ADDR_FMT")",
3737                      __func__, rb->idstr, start, length, rb->max_length);
3738     }
3739 
3740 err:
3741     return ret;
3742 }
3743 
ram_block_discard_guest_memfd_range(RAMBlock * rb,uint64_t start,size_t length)3744 int ram_block_discard_guest_memfd_range(RAMBlock *rb, uint64_t start,
3745                                         size_t length)
3746 {
3747     int ret = -1;
3748 
3749 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
3750     ret = fallocate(rb->guest_memfd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
3751                     start, length);
3752 
3753     if (ret) {
3754         ret = -errno;
3755         error_report("%s: Failed to fallocate %s:%" PRIx64 " +%zx (%d)",
3756                      __func__, rb->idstr, start, length, ret);
3757     }
3758 #else
3759     ret = -ENOSYS;
3760     error_report("%s: fallocate not available %s:%" PRIx64 " +%zx (%d)",
3761                  __func__, rb->idstr, start, length, ret);
3762 #endif
3763 
3764     return ret;
3765 }
3766 
ramblock_is_pmem(RAMBlock * rb)3767 bool ramblock_is_pmem(RAMBlock *rb)
3768 {
3769     return rb->flags & RAM_PMEM;
3770 }
3771 
mtree_print_phys_entries(int start,int end,int skip,int ptr)3772 static void mtree_print_phys_entries(int start, int end, int skip, int ptr)
3773 {
3774     if (start == end - 1) {
3775         qemu_printf("\t%3d      ", start);
3776     } else {
3777         qemu_printf("\t%3d..%-3d ", start, end - 1);
3778     }
3779     qemu_printf(" skip=%d ", skip);
3780     if (ptr == PHYS_MAP_NODE_NIL) {
3781         qemu_printf(" ptr=NIL");
3782     } else if (!skip) {
3783         qemu_printf(" ptr=#%d", ptr);
3784     } else {
3785         qemu_printf(" ptr=[%d]", ptr);
3786     }
3787     qemu_printf("\n");
3788 }
3789 
3790 #define MR_SIZE(size) (int128_nz(size) ? (hwaddr)int128_get64( \
3791                            int128_sub((size), int128_one())) : 0)
3792 
mtree_print_dispatch(AddressSpaceDispatch * d,MemoryRegion * root)3793 void mtree_print_dispatch(AddressSpaceDispatch *d, MemoryRegion *root)
3794 {
3795     int i;
3796 
3797     qemu_printf("  Dispatch\n");
3798     qemu_printf("    Physical sections\n");
3799 
3800     for (i = 0; i < d->map.sections_nb; ++i) {
3801         MemoryRegionSection *s = d->map.sections + i;
3802         const char *names[] = { " [unassigned]", " [not dirty]",
3803                                 " [ROM]", " [watch]" };
3804 
3805         qemu_printf("      #%d @" HWADDR_FMT_plx ".." HWADDR_FMT_plx
3806                     " %s%s%s%s%s",
3807             i,
3808             s->offset_within_address_space,
3809             s->offset_within_address_space + MR_SIZE(s->size),
3810             s->mr->name ? s->mr->name : "(noname)",
3811             i < ARRAY_SIZE(names) ? names[i] : "",
3812             s->mr == root ? " [ROOT]" : "",
3813             s == d->mru_section ? " [MRU]" : "",
3814             s->mr->is_iommu ? " [iommu]" : "");
3815 
3816         if (s->mr->alias) {
3817             qemu_printf(" alias=%s", s->mr->alias->name ?
3818                     s->mr->alias->name : "noname");
3819         }
3820         qemu_printf("\n");
3821     }
3822 
3823     qemu_printf("    Nodes (%d bits per level, %d levels) ptr=[%d] skip=%d\n",
3824                P_L2_BITS, P_L2_LEVELS, d->phys_map.ptr, d->phys_map.skip);
3825     for (i = 0; i < d->map.nodes_nb; ++i) {
3826         int j, jprev;
3827         PhysPageEntry prev;
3828         Node *n = d->map.nodes + i;
3829 
3830         qemu_printf("      [%d]\n", i);
3831 
3832         for (j = 0, jprev = 0, prev = *n[0]; j < ARRAY_SIZE(*n); ++j) {
3833             PhysPageEntry *pe = *n + j;
3834 
3835             if (pe->ptr == prev.ptr && pe->skip == prev.skip) {
3836                 continue;
3837             }
3838 
3839             mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
3840 
3841             jprev = j;
3842             prev = *pe;
3843         }
3844 
3845         if (jprev != ARRAY_SIZE(*n)) {
3846             mtree_print_phys_entries(jprev, j, prev.skip, prev.ptr);
3847         }
3848     }
3849 }
3850 
3851 /* Require any discards to work. */
3852 static unsigned int ram_block_discard_required_cnt;
3853 /* Require only coordinated discards to work. */
3854 static unsigned int ram_block_coordinated_discard_required_cnt;
3855 /* Disable any discards. */
3856 static unsigned int ram_block_discard_disabled_cnt;
3857 /* Disable only uncoordinated discards. */
3858 static unsigned int ram_block_uncoordinated_discard_disabled_cnt;
3859 static QemuMutex ram_block_discard_disable_mutex;
3860 
ram_block_discard_disable_mutex_lock(void)3861 static void ram_block_discard_disable_mutex_lock(void)
3862 {
3863     static gsize initialized;
3864 
3865     if (g_once_init_enter(&initialized)) {
3866         qemu_mutex_init(&ram_block_discard_disable_mutex);
3867         g_once_init_leave(&initialized, 1);
3868     }
3869     qemu_mutex_lock(&ram_block_discard_disable_mutex);
3870 }
3871 
ram_block_discard_disable_mutex_unlock(void)3872 static void ram_block_discard_disable_mutex_unlock(void)
3873 {
3874     qemu_mutex_unlock(&ram_block_discard_disable_mutex);
3875 }
3876 
ram_block_discard_disable(bool state)3877 int ram_block_discard_disable(bool state)
3878 {
3879     int ret = 0;
3880 
3881     ram_block_discard_disable_mutex_lock();
3882     if (!state) {
3883         ram_block_discard_disabled_cnt--;
3884     } else if (ram_block_discard_required_cnt ||
3885                ram_block_coordinated_discard_required_cnt) {
3886         ret = -EBUSY;
3887     } else {
3888         ram_block_discard_disabled_cnt++;
3889     }
3890     ram_block_discard_disable_mutex_unlock();
3891     return ret;
3892 }
3893 
ram_block_uncoordinated_discard_disable(bool state)3894 int ram_block_uncoordinated_discard_disable(bool state)
3895 {
3896     int ret = 0;
3897 
3898     ram_block_discard_disable_mutex_lock();
3899     if (!state) {
3900         ram_block_uncoordinated_discard_disabled_cnt--;
3901     } else if (ram_block_discard_required_cnt) {
3902         ret = -EBUSY;
3903     } else {
3904         ram_block_uncoordinated_discard_disabled_cnt++;
3905     }
3906     ram_block_discard_disable_mutex_unlock();
3907     return ret;
3908 }
3909 
ram_block_discard_require(bool state)3910 int ram_block_discard_require(bool state)
3911 {
3912     int ret = 0;
3913 
3914     ram_block_discard_disable_mutex_lock();
3915     if (!state) {
3916         ram_block_discard_required_cnt--;
3917     } else if (ram_block_discard_disabled_cnt ||
3918                ram_block_uncoordinated_discard_disabled_cnt) {
3919         ret = -EBUSY;
3920     } else {
3921         ram_block_discard_required_cnt++;
3922     }
3923     ram_block_discard_disable_mutex_unlock();
3924     return ret;
3925 }
3926 
ram_block_coordinated_discard_require(bool state)3927 int ram_block_coordinated_discard_require(bool state)
3928 {
3929     int ret = 0;
3930 
3931     ram_block_discard_disable_mutex_lock();
3932     if (!state) {
3933         ram_block_coordinated_discard_required_cnt--;
3934     } else if (ram_block_discard_disabled_cnt) {
3935         ret = -EBUSY;
3936     } else {
3937         ram_block_coordinated_discard_required_cnt++;
3938     }
3939     ram_block_discard_disable_mutex_unlock();
3940     return ret;
3941 }
3942 
ram_block_discard_is_disabled(void)3943 bool ram_block_discard_is_disabled(void)
3944 {
3945     return qatomic_read(&ram_block_discard_disabled_cnt) ||
3946            qatomic_read(&ram_block_uncoordinated_discard_disabled_cnt);
3947 }
3948 
ram_block_discard_is_required(void)3949 bool ram_block_discard_is_required(void)
3950 {
3951     return qatomic_read(&ram_block_discard_required_cnt) ||
3952            qatomic_read(&ram_block_coordinated_discard_required_cnt);
3953 }
3954