1 /* 2 * Declarations for cpu physical memory functions 3 * 4 * Copyright 2011 Red Hat, Inc. and/or its affiliates 5 * 6 * Authors: 7 * Avi Kivity <avi@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or 10 * later. See the COPYING file in the top-level directory. 11 * 12 */ 13 14 /* 15 * This header is for use by exec.c and memory.c ONLY. Do not include it. 16 * The functions declared here will be removed soon. 17 */ 18 19 #ifndef RAM_ADDR_H 20 #define RAM_ADDR_H 21 22 #ifndef CONFIG_USER_ONLY 23 #include "hw/xen/xen.h" 24 25 struct RAMBlock { 26 struct rcu_head rcu; 27 struct MemoryRegion *mr; 28 uint8_t *host; 29 ram_addr_t offset; 30 ram_addr_t used_length; 31 ram_addr_t max_length; 32 void (*resized)(const char*, uint64_t length, void *host); 33 uint32_t flags; 34 /* Protected by iothread lock. */ 35 char idstr[256]; 36 /* RCU-enabled, writes protected by the ramlist lock */ 37 QLIST_ENTRY(RAMBlock) next; 38 int fd; 39 }; 40 41 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset) 42 { 43 return (b && b->host && offset < b->used_length) ? true : false; 44 } 45 46 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset) 47 { 48 assert(offset_in_ramblock(block, offset)); 49 return (char *)block->host + offset; 50 } 51 52 /* The dirty memory bitmap is split into fixed-size blocks to allow growth 53 * under RCU. The bitmap for a block can be accessed as follows: 54 * 55 * rcu_read_lock(); 56 * 57 * DirtyMemoryBlocks *blocks = 58 * atomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]); 59 * 60 * ram_addr_t idx = (addr >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; 61 * unsigned long *block = blocks.blocks[idx]; 62 * ...access block bitmap... 63 * 64 * rcu_read_unlock(); 65 * 66 * Remember to check for the end of the block when accessing a range of 67 * addresses. Move on to the next block if you reach the end. 68 * 69 * Organization into blocks allows dirty memory to grow (but not shrink) under 70 * RCU. When adding new RAMBlocks requires the dirty memory to grow, a new 71 * DirtyMemoryBlocks array is allocated with pointers to existing blocks kept 72 * the same. Other threads can safely access existing blocks while dirty 73 * memory is being grown. When no threads are using the old DirtyMemoryBlocks 74 * anymore it is freed by RCU (but the underlying blocks stay because they are 75 * pointed to from the new DirtyMemoryBlocks). 76 */ 77 #define DIRTY_MEMORY_BLOCK_SIZE ((ram_addr_t)256 * 1024 * 8) 78 typedef struct { 79 struct rcu_head rcu; 80 unsigned long *blocks[]; 81 } DirtyMemoryBlocks; 82 83 typedef struct RAMList { 84 QemuMutex mutex; 85 RAMBlock *mru_block; 86 /* RCU-enabled, writes protected by the ramlist lock. */ 87 QLIST_HEAD(, RAMBlock) blocks; 88 DirtyMemoryBlocks *dirty_memory[DIRTY_MEMORY_NUM]; 89 uint32_t version; 90 } RAMList; 91 extern RAMList ram_list; 92 93 ram_addr_t last_ram_offset(void); 94 void qemu_mutex_lock_ramlist(void); 95 void qemu_mutex_unlock_ramlist(void); 96 97 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr, 98 bool share, const char *mem_path, 99 Error **errp); 100 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host, 101 MemoryRegion *mr, Error **errp); 102 RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp); 103 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size, 104 void (*resized)(const char*, 105 uint64_t length, 106 void *host), 107 MemoryRegion *mr, Error **errp); 108 void qemu_ram_free(RAMBlock *block); 109 110 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp); 111 112 #define DIRTY_CLIENTS_ALL ((1 << DIRTY_MEMORY_NUM) - 1) 113 #define DIRTY_CLIENTS_NOCODE (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE)) 114 115 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start, 116 ram_addr_t length, 117 unsigned client) 118 { 119 DirtyMemoryBlocks *blocks; 120 unsigned long end, page; 121 unsigned long idx, offset, base; 122 bool dirty = false; 123 124 assert(client < DIRTY_MEMORY_NUM); 125 126 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 127 page = start >> TARGET_PAGE_BITS; 128 129 rcu_read_lock(); 130 131 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); 132 133 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 134 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 135 base = page - offset; 136 while (page < end) { 137 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 138 unsigned long num = next - base; 139 unsigned long found = find_next_bit(blocks->blocks[idx], num, offset); 140 if (found < num) { 141 dirty = true; 142 break; 143 } 144 145 page = next; 146 idx++; 147 offset = 0; 148 base += DIRTY_MEMORY_BLOCK_SIZE; 149 } 150 151 rcu_read_unlock(); 152 153 return dirty; 154 } 155 156 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start, 157 ram_addr_t length, 158 unsigned client) 159 { 160 DirtyMemoryBlocks *blocks; 161 unsigned long end, page; 162 unsigned long idx, offset, base; 163 bool dirty = true; 164 165 assert(client < DIRTY_MEMORY_NUM); 166 167 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 168 page = start >> TARGET_PAGE_BITS; 169 170 rcu_read_lock(); 171 172 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); 173 174 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 175 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 176 base = page - offset; 177 while (page < end) { 178 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 179 unsigned long num = next - base; 180 unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset); 181 if (found < num) { 182 dirty = false; 183 break; 184 } 185 186 page = next; 187 idx++; 188 offset = 0; 189 base += DIRTY_MEMORY_BLOCK_SIZE; 190 } 191 192 rcu_read_unlock(); 193 194 return dirty; 195 } 196 197 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr, 198 unsigned client) 199 { 200 return cpu_physical_memory_get_dirty(addr, 1, client); 201 } 202 203 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr) 204 { 205 bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA); 206 bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE); 207 bool migration = 208 cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION); 209 return !(vga && code && migration); 210 } 211 212 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start, 213 ram_addr_t length, 214 uint8_t mask) 215 { 216 uint8_t ret = 0; 217 218 if (mask & (1 << DIRTY_MEMORY_VGA) && 219 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) { 220 ret |= (1 << DIRTY_MEMORY_VGA); 221 } 222 if (mask & (1 << DIRTY_MEMORY_CODE) && 223 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) { 224 ret |= (1 << DIRTY_MEMORY_CODE); 225 } 226 if (mask & (1 << DIRTY_MEMORY_MIGRATION) && 227 !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) { 228 ret |= (1 << DIRTY_MEMORY_MIGRATION); 229 } 230 return ret; 231 } 232 233 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr, 234 unsigned client) 235 { 236 unsigned long page, idx, offset; 237 DirtyMemoryBlocks *blocks; 238 239 assert(client < DIRTY_MEMORY_NUM); 240 241 page = addr >> TARGET_PAGE_BITS; 242 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 243 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 244 245 rcu_read_lock(); 246 247 blocks = atomic_rcu_read(&ram_list.dirty_memory[client]); 248 249 set_bit_atomic(offset, blocks->blocks[idx]); 250 251 rcu_read_unlock(); 252 } 253 254 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start, 255 ram_addr_t length, 256 uint8_t mask) 257 { 258 DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM]; 259 unsigned long end, page; 260 unsigned long idx, offset, base; 261 int i; 262 263 if (!mask && !xen_enabled()) { 264 return; 265 } 266 267 end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS; 268 page = start >> TARGET_PAGE_BITS; 269 270 rcu_read_lock(); 271 272 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 273 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]); 274 } 275 276 idx = page / DIRTY_MEMORY_BLOCK_SIZE; 277 offset = page % DIRTY_MEMORY_BLOCK_SIZE; 278 base = page - offset; 279 while (page < end) { 280 unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE); 281 282 if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) { 283 bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx], 284 offset, next - page); 285 } 286 if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) { 287 bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx], 288 offset, next - page); 289 } 290 if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) { 291 bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx], 292 offset, next - page); 293 } 294 295 page = next; 296 idx++; 297 offset = 0; 298 base += DIRTY_MEMORY_BLOCK_SIZE; 299 } 300 301 rcu_read_unlock(); 302 303 xen_modified_memory(start, length); 304 } 305 306 #if !defined(_WIN32) 307 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap, 308 ram_addr_t start, 309 ram_addr_t pages) 310 { 311 unsigned long i, j; 312 unsigned long page_number, c; 313 hwaddr addr; 314 ram_addr_t ram_addr; 315 unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS; 316 unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE; 317 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 318 319 /* start address is aligned at the start of a word? */ 320 if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) && 321 (hpratio == 1)) { 322 unsigned long **blocks[DIRTY_MEMORY_NUM]; 323 unsigned long idx; 324 unsigned long offset; 325 long k; 326 long nr = BITS_TO_LONGS(pages); 327 328 idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE; 329 offset = BIT_WORD((start >> TARGET_PAGE_BITS) % 330 DIRTY_MEMORY_BLOCK_SIZE); 331 332 rcu_read_lock(); 333 334 for (i = 0; i < DIRTY_MEMORY_NUM; i++) { 335 blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks; 336 } 337 338 for (k = 0; k < nr; k++) { 339 if (bitmap[k]) { 340 unsigned long temp = leul_to_cpu(bitmap[k]); 341 342 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp); 343 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp); 344 if (tcg_enabled()) { 345 atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp); 346 } 347 } 348 349 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { 350 offset = 0; 351 idx++; 352 } 353 } 354 355 rcu_read_unlock(); 356 357 xen_modified_memory(start, pages << TARGET_PAGE_BITS); 358 } else { 359 uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE; 360 /* 361 * bitmap-traveling is faster than memory-traveling (for addr...) 362 * especially when most of the memory is not dirty. 363 */ 364 for (i = 0; i < len; i++) { 365 if (bitmap[i] != 0) { 366 c = leul_to_cpu(bitmap[i]); 367 do { 368 j = ctzl(c); 369 c &= ~(1ul << j); 370 page_number = (i * HOST_LONG_BITS + j) * hpratio; 371 addr = page_number * TARGET_PAGE_SIZE; 372 ram_addr = start + addr; 373 cpu_physical_memory_set_dirty_range(ram_addr, 374 TARGET_PAGE_SIZE * hpratio, clients); 375 } while (c != 0); 376 } 377 } 378 } 379 } 380 #endif /* not _WIN32 */ 381 382 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start, 383 ram_addr_t length, 384 unsigned client); 385 386 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start, 387 ram_addr_t length) 388 { 389 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION); 390 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA); 391 cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE); 392 } 393 394 395 static inline 396 uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest, 397 ram_addr_t start, 398 ram_addr_t length) 399 { 400 ram_addr_t addr; 401 unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS); 402 uint64_t num_dirty = 0; 403 404 /* start address is aligned at the start of a word? */ 405 if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) { 406 int k; 407 int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS); 408 unsigned long * const *src; 409 unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE; 410 unsigned long offset = BIT_WORD((page * BITS_PER_LONG) % 411 DIRTY_MEMORY_BLOCK_SIZE); 412 413 rcu_read_lock(); 414 415 src = atomic_rcu_read( 416 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks; 417 418 for (k = page; k < page + nr; k++) { 419 if (src[idx][offset]) { 420 unsigned long bits = atomic_xchg(&src[idx][offset], 0); 421 unsigned long new_dirty; 422 new_dirty = ~dest[k]; 423 dest[k] |= bits; 424 new_dirty &= bits; 425 num_dirty += ctpopl(new_dirty); 426 } 427 428 if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) { 429 offset = 0; 430 idx++; 431 } 432 } 433 434 rcu_read_unlock(); 435 } else { 436 for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) { 437 if (cpu_physical_memory_test_and_clear_dirty( 438 start + addr, 439 TARGET_PAGE_SIZE, 440 DIRTY_MEMORY_MIGRATION)) { 441 long k = (start + addr) >> TARGET_PAGE_BITS; 442 if (!test_and_set_bit(k, dest)) { 443 num_dirty++; 444 } 445 } 446 } 447 } 448 449 return num_dirty; 450 } 451 452 void migration_bitmap_extend(ram_addr_t old, ram_addr_t new); 453 #endif 454 #endif 455