1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Procedures for maintaining information about logical memory blocks. 4 * 5 * Peter Bergner, IBM Corp. June 2001. 6 * Copyright (C) 2001 Peter Bergner. 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/slab.h> 11 #include <linux/init.h> 12 #include <linux/bitops.h> 13 #include <linux/poison.h> 14 #include <linux/pfn.h> 15 #include <linux/debugfs.h> 16 #include <linux/kmemleak.h> 17 #include <linux/seq_file.h> 18 #include <linux/memblock.h> 19 20 #include <asm/sections.h> 21 #include <linux/io.h> 22 23 #include "internal.h" 24 25 #define INIT_MEMBLOCK_REGIONS 128 26 #define INIT_PHYSMEM_REGIONS 4 27 28 #ifndef INIT_MEMBLOCK_RESERVED_REGIONS 29 # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS 30 #endif 31 32 #ifndef INIT_MEMBLOCK_MEMORY_REGIONS 33 #define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS 34 #endif 35 36 /** 37 * DOC: memblock overview 38 * 39 * Memblock is a method of managing memory regions during the early 40 * boot period when the usual kernel memory allocators are not up and 41 * running. 42 * 43 * Memblock views the system memory as collections of contiguous 44 * regions. There are several types of these collections: 45 * 46 * * ``memory`` - describes the physical memory available to the 47 * kernel; this may differ from the actual physical memory installed 48 * in the system, for instance when the memory is restricted with 49 * ``mem=`` command line parameter 50 * * ``reserved`` - describes the regions that were allocated 51 * * ``physmem`` - describes the actual physical memory available during 52 * boot regardless of the possible restrictions and memory hot(un)plug; 53 * the ``physmem`` type is only available on some architectures. 54 * 55 * Each region is represented by struct memblock_region that 56 * defines the region extents, its attributes and NUMA node id on NUMA 57 * systems. Every memory type is described by the struct memblock_type 58 * which contains an array of memory regions along with 59 * the allocator metadata. The "memory" and "reserved" types are nicely 60 * wrapped with struct memblock. This structure is statically 61 * initialized at build time. The region arrays are initially sized to 62 * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and 63 * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array 64 * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS. 65 * The memblock_allow_resize() enables automatic resizing of the region 66 * arrays during addition of new regions. This feature should be used 67 * with care so that memory allocated for the region array will not 68 * overlap with areas that should be reserved, for example initrd. 69 * 70 * The early architecture setup should tell memblock what the physical 71 * memory layout is by using memblock_add() or memblock_add_node() 72 * functions. The first function does not assign the region to a NUMA 73 * node and it is appropriate for UMA systems. Yet, it is possible to 74 * use it on NUMA systems as well and assign the region to a NUMA node 75 * later in the setup process using memblock_set_node(). The 76 * memblock_add_node() performs such an assignment directly. 77 * 78 * Once memblock is setup the memory can be allocated using one of the 79 * API variants: 80 * 81 * * memblock_phys_alloc*() - these functions return the **physical** 82 * address of the allocated memory 83 * * memblock_alloc*() - these functions return the **virtual** address 84 * of the allocated memory. 85 * 86 * Note, that both API variants use implicit assumptions about allowed 87 * memory ranges and the fallback methods. Consult the documentation 88 * of memblock_alloc_internal() and memblock_alloc_range_nid() 89 * functions for more elaborate description. 90 * 91 * As the system boot progresses, the architecture specific mem_init() 92 * function frees all the memory to the buddy page allocator. 93 * 94 * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the 95 * memblock data structures (except "physmem") will be discarded after the 96 * system initialization completes. 97 */ 98 99 #ifndef CONFIG_NUMA 100 struct pglist_data __refdata contig_page_data; 101 EXPORT_SYMBOL(contig_page_data); 102 #endif 103 104 unsigned long max_low_pfn; 105 unsigned long min_low_pfn; 106 unsigned long max_pfn; 107 unsigned long long max_possible_pfn; 108 109 static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock; 110 static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock; 111 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 112 static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS]; 113 #endif 114 115 struct memblock memblock __initdata_memblock = { 116 .memory.regions = memblock_memory_init_regions, 117 .memory.cnt = 1, /* empty dummy entry */ 118 .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS, 119 .memory.name = "memory", 120 121 .reserved.regions = memblock_reserved_init_regions, 122 .reserved.cnt = 1, /* empty dummy entry */ 123 .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS, 124 .reserved.name = "reserved", 125 126 .bottom_up = false, 127 .current_limit = MEMBLOCK_ALLOC_ANYWHERE, 128 }; 129 130 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 131 struct memblock_type physmem = { 132 .regions = memblock_physmem_init_regions, 133 .cnt = 1, /* empty dummy entry */ 134 .max = INIT_PHYSMEM_REGIONS, 135 .name = "physmem", 136 }; 137 #endif 138 139 /* 140 * keep a pointer to &memblock.memory in the text section to use it in 141 * __next_mem_range() and its helpers. 142 * For architectures that do not keep memblock data after init, this 143 * pointer will be reset to NULL at memblock_discard() 144 */ 145 static __refdata struct memblock_type *memblock_memory = &memblock.memory; 146 147 #define for_each_memblock_type(i, memblock_type, rgn) \ 148 for (i = 0, rgn = &memblock_type->regions[0]; \ 149 i < memblock_type->cnt; \ 150 i++, rgn = &memblock_type->regions[i]) 151 152 #define memblock_dbg(fmt, ...) \ 153 do { \ 154 if (memblock_debug) \ 155 pr_info(fmt, ##__VA_ARGS__); \ 156 } while (0) 157 158 static int memblock_debug __initdata_memblock; 159 static bool system_has_some_mirror __initdata_memblock = false; 160 static int memblock_can_resize __initdata_memblock; 161 static int memblock_memory_in_slab __initdata_memblock = 0; 162 static int memblock_reserved_in_slab __initdata_memblock = 0; 163 164 static enum memblock_flags __init_memblock choose_memblock_flags(void) 165 { 166 return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE; 167 } 168 169 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */ 170 static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size) 171 { 172 return *size = min(*size, PHYS_ADDR_MAX - base); 173 } 174 175 /* 176 * Address comparison utilities 177 */ 178 static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1, 179 phys_addr_t base2, phys_addr_t size2) 180 { 181 return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); 182 } 183 184 bool __init_memblock memblock_overlaps_region(struct memblock_type *type, 185 phys_addr_t base, phys_addr_t size) 186 { 187 unsigned long i; 188 189 memblock_cap_size(base, &size); 190 191 for (i = 0; i < type->cnt; i++) 192 if (memblock_addrs_overlap(base, size, type->regions[i].base, 193 type->regions[i].size)) 194 break; 195 return i < type->cnt; 196 } 197 198 /** 199 * __memblock_find_range_bottom_up - find free area utility in bottom-up 200 * @start: start of candidate range 201 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 202 * %MEMBLOCK_ALLOC_ACCESSIBLE 203 * @size: size of free area to find 204 * @align: alignment of free area to find 205 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 206 * @flags: pick from blocks based on memory attributes 207 * 208 * Utility called from memblock_find_in_range_node(), find free area bottom-up. 209 * 210 * Return: 211 * Found address on success, 0 on failure. 212 */ 213 static phys_addr_t __init_memblock 214 __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end, 215 phys_addr_t size, phys_addr_t align, int nid, 216 enum memblock_flags flags) 217 { 218 phys_addr_t this_start, this_end, cand; 219 u64 i; 220 221 for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) { 222 this_start = clamp(this_start, start, end); 223 this_end = clamp(this_end, start, end); 224 225 cand = round_up(this_start, align); 226 if (cand < this_end && this_end - cand >= size) 227 return cand; 228 } 229 230 return 0; 231 } 232 233 /** 234 * __memblock_find_range_top_down - find free area utility, in top-down 235 * @start: start of candidate range 236 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 237 * %MEMBLOCK_ALLOC_ACCESSIBLE 238 * @size: size of free area to find 239 * @align: alignment of free area to find 240 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 241 * @flags: pick from blocks based on memory attributes 242 * 243 * Utility called from memblock_find_in_range_node(), find free area top-down. 244 * 245 * Return: 246 * Found address on success, 0 on failure. 247 */ 248 static phys_addr_t __init_memblock 249 __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end, 250 phys_addr_t size, phys_addr_t align, int nid, 251 enum memblock_flags flags) 252 { 253 phys_addr_t this_start, this_end, cand; 254 u64 i; 255 256 for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end, 257 NULL) { 258 this_start = clamp(this_start, start, end); 259 this_end = clamp(this_end, start, end); 260 261 if (this_end < size) 262 continue; 263 264 cand = round_down(this_end - size, align); 265 if (cand >= this_start) 266 return cand; 267 } 268 269 return 0; 270 } 271 272 /** 273 * memblock_find_in_range_node - find free area in given range and node 274 * @size: size of free area to find 275 * @align: alignment of free area to find 276 * @start: start of candidate range 277 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 278 * %MEMBLOCK_ALLOC_ACCESSIBLE 279 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 280 * @flags: pick from blocks based on memory attributes 281 * 282 * Find @size free area aligned to @align in the specified range and node. 283 * 284 * Return: 285 * Found address on success, 0 on failure. 286 */ 287 static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size, 288 phys_addr_t align, phys_addr_t start, 289 phys_addr_t end, int nid, 290 enum memblock_flags flags) 291 { 292 /* pump up @end */ 293 if (end == MEMBLOCK_ALLOC_ACCESSIBLE || 294 end == MEMBLOCK_ALLOC_NOLEAKTRACE) 295 end = memblock.current_limit; 296 297 /* avoid allocating the first page */ 298 start = max_t(phys_addr_t, start, PAGE_SIZE); 299 end = max(start, end); 300 301 if (memblock_bottom_up()) 302 return __memblock_find_range_bottom_up(start, end, size, align, 303 nid, flags); 304 else 305 return __memblock_find_range_top_down(start, end, size, align, 306 nid, flags); 307 } 308 309 /** 310 * memblock_find_in_range - find free area in given range 311 * @start: start of candidate range 312 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or 313 * %MEMBLOCK_ALLOC_ACCESSIBLE 314 * @size: size of free area to find 315 * @align: alignment of free area to find 316 * 317 * Find @size free area aligned to @align in the specified range. 318 * 319 * Return: 320 * Found address on success, 0 on failure. 321 */ 322 static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start, 323 phys_addr_t end, phys_addr_t size, 324 phys_addr_t align) 325 { 326 phys_addr_t ret; 327 enum memblock_flags flags = choose_memblock_flags(); 328 329 again: 330 ret = memblock_find_in_range_node(size, align, start, end, 331 NUMA_NO_NODE, flags); 332 333 if (!ret && (flags & MEMBLOCK_MIRROR)) { 334 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 335 &size); 336 flags &= ~MEMBLOCK_MIRROR; 337 goto again; 338 } 339 340 return ret; 341 } 342 343 static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r) 344 { 345 type->total_size -= type->regions[r].size; 346 memmove(&type->regions[r], &type->regions[r + 1], 347 (type->cnt - (r + 1)) * sizeof(type->regions[r])); 348 type->cnt--; 349 350 /* Special case for empty arrays */ 351 if (type->cnt == 0) { 352 WARN_ON(type->total_size != 0); 353 type->cnt = 1; 354 type->regions[0].base = 0; 355 type->regions[0].size = 0; 356 type->regions[0].flags = 0; 357 memblock_set_region_node(&type->regions[0], MAX_NUMNODES); 358 } 359 } 360 361 #ifndef CONFIG_ARCH_KEEP_MEMBLOCK 362 /** 363 * memblock_discard - discard memory and reserved arrays if they were allocated 364 */ 365 void __init memblock_discard(void) 366 { 367 phys_addr_t addr, size; 368 369 if (memblock.reserved.regions != memblock_reserved_init_regions) { 370 addr = __pa(memblock.reserved.regions); 371 size = PAGE_ALIGN(sizeof(struct memblock_region) * 372 memblock.reserved.max); 373 if (memblock_reserved_in_slab) 374 kfree(memblock.reserved.regions); 375 else 376 memblock_free_late(addr, size); 377 } 378 379 if (memblock.memory.regions != memblock_memory_init_regions) { 380 addr = __pa(memblock.memory.regions); 381 size = PAGE_ALIGN(sizeof(struct memblock_region) * 382 memblock.memory.max); 383 if (memblock_memory_in_slab) 384 kfree(memblock.memory.regions); 385 else 386 memblock_free_late(addr, size); 387 } 388 389 memblock_memory = NULL; 390 } 391 #endif 392 393 /** 394 * memblock_double_array - double the size of the memblock regions array 395 * @type: memblock type of the regions array being doubled 396 * @new_area_start: starting address of memory range to avoid overlap with 397 * @new_area_size: size of memory range to avoid overlap with 398 * 399 * Double the size of the @type regions array. If memblock is being used to 400 * allocate memory for a new reserved regions array and there is a previously 401 * allocated memory range [@new_area_start, @new_area_start + @new_area_size] 402 * waiting to be reserved, ensure the memory used by the new array does 403 * not overlap. 404 * 405 * Return: 406 * 0 on success, -1 on failure. 407 */ 408 static int __init_memblock memblock_double_array(struct memblock_type *type, 409 phys_addr_t new_area_start, 410 phys_addr_t new_area_size) 411 { 412 struct memblock_region *new_array, *old_array; 413 phys_addr_t old_alloc_size, new_alloc_size; 414 phys_addr_t old_size, new_size, addr, new_end; 415 int use_slab = slab_is_available(); 416 int *in_slab; 417 418 /* We don't allow resizing until we know about the reserved regions 419 * of memory that aren't suitable for allocation 420 */ 421 if (!memblock_can_resize) 422 return -1; 423 424 /* Calculate new doubled size */ 425 old_size = type->max * sizeof(struct memblock_region); 426 new_size = old_size << 1; 427 /* 428 * We need to allocated new one align to PAGE_SIZE, 429 * so we can free them completely later. 430 */ 431 old_alloc_size = PAGE_ALIGN(old_size); 432 new_alloc_size = PAGE_ALIGN(new_size); 433 434 /* Retrieve the slab flag */ 435 if (type == &memblock.memory) 436 in_slab = &memblock_memory_in_slab; 437 else 438 in_slab = &memblock_reserved_in_slab; 439 440 /* Try to find some space for it */ 441 if (use_slab) { 442 new_array = kmalloc(new_size, GFP_KERNEL); 443 addr = new_array ? __pa(new_array) : 0; 444 } else { 445 /* only exclude range when trying to double reserved.regions */ 446 if (type != &memblock.reserved) 447 new_area_start = new_area_size = 0; 448 449 addr = memblock_find_in_range(new_area_start + new_area_size, 450 memblock.current_limit, 451 new_alloc_size, PAGE_SIZE); 452 if (!addr && new_area_size) 453 addr = memblock_find_in_range(0, 454 min(new_area_start, memblock.current_limit), 455 new_alloc_size, PAGE_SIZE); 456 457 new_array = addr ? __va(addr) : NULL; 458 } 459 if (!addr) { 460 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n", 461 type->name, type->max, type->max * 2); 462 return -1; 463 } 464 465 new_end = addr + new_size - 1; 466 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]", 467 type->name, type->max * 2, &addr, &new_end); 468 469 /* 470 * Found space, we now need to move the array over before we add the 471 * reserved region since it may be our reserved array itself that is 472 * full. 473 */ 474 memcpy(new_array, type->regions, old_size); 475 memset(new_array + type->max, 0, old_size); 476 old_array = type->regions; 477 type->regions = new_array; 478 type->max <<= 1; 479 480 /* Free old array. We needn't free it if the array is the static one */ 481 if (*in_slab) 482 kfree(old_array); 483 else if (old_array != memblock_memory_init_regions && 484 old_array != memblock_reserved_init_regions) 485 memblock_free(old_array, old_alloc_size); 486 487 /* 488 * Reserve the new array if that comes from the memblock. Otherwise, we 489 * needn't do it 490 */ 491 if (!use_slab) 492 BUG_ON(memblock_reserve(addr, new_alloc_size)); 493 494 /* Update slab flag */ 495 *in_slab = use_slab; 496 497 return 0; 498 } 499 500 /** 501 * memblock_merge_regions - merge neighboring compatible regions 502 * @type: memblock type to scan 503 * @start_rgn: start scanning from (@start_rgn - 1) 504 * @end_rgn: end scanning at (@end_rgn - 1) 505 * Scan @type and merge neighboring compatible regions in [@start_rgn - 1, @end_rgn) 506 */ 507 static void __init_memblock memblock_merge_regions(struct memblock_type *type, 508 unsigned long start_rgn, 509 unsigned long end_rgn) 510 { 511 int i = 0; 512 if (start_rgn) 513 i = start_rgn - 1; 514 end_rgn = min(end_rgn, type->cnt - 1); 515 while (i < end_rgn) { 516 struct memblock_region *this = &type->regions[i]; 517 struct memblock_region *next = &type->regions[i + 1]; 518 519 if (this->base + this->size != next->base || 520 memblock_get_region_node(this) != 521 memblock_get_region_node(next) || 522 this->flags != next->flags) { 523 BUG_ON(this->base + this->size > next->base); 524 i++; 525 continue; 526 } 527 528 this->size += next->size; 529 /* move forward from next + 1, index of which is i + 2 */ 530 memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next)); 531 type->cnt--; 532 end_rgn--; 533 } 534 } 535 536 /** 537 * memblock_insert_region - insert new memblock region 538 * @type: memblock type to insert into 539 * @idx: index for the insertion point 540 * @base: base address of the new region 541 * @size: size of the new region 542 * @nid: node id of the new region 543 * @flags: flags of the new region 544 * 545 * Insert new memblock region [@base, @base + @size) into @type at @idx. 546 * @type must already have extra room to accommodate the new region. 547 */ 548 static void __init_memblock memblock_insert_region(struct memblock_type *type, 549 int idx, phys_addr_t base, 550 phys_addr_t size, 551 int nid, 552 enum memblock_flags flags) 553 { 554 struct memblock_region *rgn = &type->regions[idx]; 555 556 BUG_ON(type->cnt >= type->max); 557 memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn)); 558 rgn->base = base; 559 rgn->size = size; 560 rgn->flags = flags; 561 memblock_set_region_node(rgn, nid); 562 type->cnt++; 563 type->total_size += size; 564 } 565 566 /** 567 * memblock_add_range - add new memblock region 568 * @type: memblock type to add new region into 569 * @base: base address of the new region 570 * @size: size of the new region 571 * @nid: nid of the new region 572 * @flags: flags of the new region 573 * 574 * Add new memblock region [@base, @base + @size) into @type. The new region 575 * is allowed to overlap with existing ones - overlaps don't affect already 576 * existing regions. @type is guaranteed to be minimal (all neighbouring 577 * compatible regions are merged) after the addition. 578 * 579 * Return: 580 * 0 on success, -errno on failure. 581 */ 582 static int __init_memblock memblock_add_range(struct memblock_type *type, 583 phys_addr_t base, phys_addr_t size, 584 int nid, enum memblock_flags flags) 585 { 586 bool insert = false; 587 phys_addr_t obase = base; 588 phys_addr_t end = base + memblock_cap_size(base, &size); 589 int idx, nr_new, start_rgn = -1, end_rgn; 590 struct memblock_region *rgn; 591 592 if (!size) 593 return 0; 594 595 /* special case for empty array */ 596 if (type->regions[0].size == 0) { 597 WARN_ON(type->cnt != 1 || type->total_size); 598 type->regions[0].base = base; 599 type->regions[0].size = size; 600 type->regions[0].flags = flags; 601 memblock_set_region_node(&type->regions[0], nid); 602 type->total_size = size; 603 return 0; 604 } 605 606 /* 607 * The worst case is when new range overlaps all existing regions, 608 * then we'll need type->cnt + 1 empty regions in @type. So if 609 * type->cnt * 2 + 1 is less than or equal to type->max, we know 610 * that there is enough empty regions in @type, and we can insert 611 * regions directly. 612 */ 613 if (type->cnt * 2 + 1 <= type->max) 614 insert = true; 615 616 repeat: 617 /* 618 * The following is executed twice. Once with %false @insert and 619 * then with %true. The first counts the number of regions needed 620 * to accommodate the new area. The second actually inserts them. 621 */ 622 base = obase; 623 nr_new = 0; 624 625 for_each_memblock_type(idx, type, rgn) { 626 phys_addr_t rbase = rgn->base; 627 phys_addr_t rend = rbase + rgn->size; 628 629 if (rbase >= end) 630 break; 631 if (rend <= base) 632 continue; 633 /* 634 * @rgn overlaps. If it separates the lower part of new 635 * area, insert that portion. 636 */ 637 if (rbase > base) { 638 #ifdef CONFIG_NUMA 639 WARN_ON(nid != memblock_get_region_node(rgn)); 640 #endif 641 WARN_ON(flags != rgn->flags); 642 nr_new++; 643 if (insert) { 644 if (start_rgn == -1) 645 start_rgn = idx; 646 end_rgn = idx + 1; 647 memblock_insert_region(type, idx++, base, 648 rbase - base, nid, 649 flags); 650 } 651 } 652 /* area below @rend is dealt with, forget about it */ 653 base = min(rend, end); 654 } 655 656 /* insert the remaining portion */ 657 if (base < end) { 658 nr_new++; 659 if (insert) { 660 if (start_rgn == -1) 661 start_rgn = idx; 662 end_rgn = idx + 1; 663 memblock_insert_region(type, idx, base, end - base, 664 nid, flags); 665 } 666 } 667 668 if (!nr_new) 669 return 0; 670 671 /* 672 * If this was the first round, resize array and repeat for actual 673 * insertions; otherwise, merge and return. 674 */ 675 if (!insert) { 676 while (type->cnt + nr_new > type->max) 677 if (memblock_double_array(type, obase, size) < 0) 678 return -ENOMEM; 679 insert = true; 680 goto repeat; 681 } else { 682 memblock_merge_regions(type, start_rgn, end_rgn); 683 return 0; 684 } 685 } 686 687 /** 688 * memblock_add_node - add new memblock region within a NUMA node 689 * @base: base address of the new region 690 * @size: size of the new region 691 * @nid: nid of the new region 692 * @flags: flags of the new region 693 * 694 * Add new memblock region [@base, @base + @size) to the "memory" 695 * type. See memblock_add_range() description for mode details 696 * 697 * Return: 698 * 0 on success, -errno on failure. 699 */ 700 int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size, 701 int nid, enum memblock_flags flags) 702 { 703 phys_addr_t end = base + size - 1; 704 705 memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__, 706 &base, &end, nid, flags, (void *)_RET_IP_); 707 708 return memblock_add_range(&memblock.memory, base, size, nid, flags); 709 } 710 711 /** 712 * memblock_add - add new memblock region 713 * @base: base address of the new region 714 * @size: size of the new region 715 * 716 * Add new memblock region [@base, @base + @size) to the "memory" 717 * type. See memblock_add_range() description for mode details 718 * 719 * Return: 720 * 0 on success, -errno on failure. 721 */ 722 int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size) 723 { 724 phys_addr_t end = base + size - 1; 725 726 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 727 &base, &end, (void *)_RET_IP_); 728 729 return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0); 730 } 731 732 /** 733 * memblock_isolate_range - isolate given range into disjoint memblocks 734 * @type: memblock type to isolate range for 735 * @base: base of range to isolate 736 * @size: size of range to isolate 737 * @start_rgn: out parameter for the start of isolated region 738 * @end_rgn: out parameter for the end of isolated region 739 * 740 * Walk @type and ensure that regions don't cross the boundaries defined by 741 * [@base, @base + @size). Crossing regions are split at the boundaries, 742 * which may create at most two more regions. The index of the first 743 * region inside the range is returned in *@start_rgn and end in *@end_rgn. 744 * 745 * Return: 746 * 0 on success, -errno on failure. 747 */ 748 static int __init_memblock memblock_isolate_range(struct memblock_type *type, 749 phys_addr_t base, phys_addr_t size, 750 int *start_rgn, int *end_rgn) 751 { 752 phys_addr_t end = base + memblock_cap_size(base, &size); 753 int idx; 754 struct memblock_region *rgn; 755 756 *start_rgn = *end_rgn = 0; 757 758 if (!size) 759 return 0; 760 761 /* we'll create at most two more regions */ 762 while (type->cnt + 2 > type->max) 763 if (memblock_double_array(type, base, size) < 0) 764 return -ENOMEM; 765 766 for_each_memblock_type(idx, type, rgn) { 767 phys_addr_t rbase = rgn->base; 768 phys_addr_t rend = rbase + rgn->size; 769 770 if (rbase >= end) 771 break; 772 if (rend <= base) 773 continue; 774 775 if (rbase < base) { 776 /* 777 * @rgn intersects from below. Split and continue 778 * to process the next region - the new top half. 779 */ 780 rgn->base = base; 781 rgn->size -= base - rbase; 782 type->total_size -= base - rbase; 783 memblock_insert_region(type, idx, rbase, base - rbase, 784 memblock_get_region_node(rgn), 785 rgn->flags); 786 } else if (rend > end) { 787 /* 788 * @rgn intersects from above. Split and redo the 789 * current region - the new bottom half. 790 */ 791 rgn->base = end; 792 rgn->size -= end - rbase; 793 type->total_size -= end - rbase; 794 memblock_insert_region(type, idx--, rbase, end - rbase, 795 memblock_get_region_node(rgn), 796 rgn->flags); 797 } else { 798 /* @rgn is fully contained, record it */ 799 if (!*end_rgn) 800 *start_rgn = idx; 801 *end_rgn = idx + 1; 802 } 803 } 804 805 return 0; 806 } 807 808 static int __init_memblock memblock_remove_range(struct memblock_type *type, 809 phys_addr_t base, phys_addr_t size) 810 { 811 int start_rgn, end_rgn; 812 int i, ret; 813 814 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 815 if (ret) 816 return ret; 817 818 for (i = end_rgn - 1; i >= start_rgn; i--) 819 memblock_remove_region(type, i); 820 return 0; 821 } 822 823 int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size) 824 { 825 phys_addr_t end = base + size - 1; 826 827 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 828 &base, &end, (void *)_RET_IP_); 829 830 return memblock_remove_range(&memblock.memory, base, size); 831 } 832 833 /** 834 * memblock_free - free boot memory allocation 835 * @ptr: starting address of the boot memory allocation 836 * @size: size of the boot memory block in bytes 837 * 838 * Free boot memory block previously allocated by memblock_alloc_xx() API. 839 * The freeing memory will not be released to the buddy allocator. 840 */ 841 void __init_memblock memblock_free(void *ptr, size_t size) 842 { 843 if (ptr) 844 memblock_phys_free(__pa(ptr), size); 845 } 846 847 /** 848 * memblock_phys_free - free boot memory block 849 * @base: phys starting address of the boot memory block 850 * @size: size of the boot memory block in bytes 851 * 852 * Free boot memory block previously allocated by memblock_phys_alloc_xx() API. 853 * The freeing memory will not be released to the buddy allocator. 854 */ 855 int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size) 856 { 857 phys_addr_t end = base + size - 1; 858 859 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 860 &base, &end, (void *)_RET_IP_); 861 862 kmemleak_free_part_phys(base, size); 863 return memblock_remove_range(&memblock.reserved, base, size); 864 } 865 866 int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size) 867 { 868 phys_addr_t end = base + size - 1; 869 870 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 871 &base, &end, (void *)_RET_IP_); 872 873 return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0); 874 } 875 876 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 877 int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size) 878 { 879 phys_addr_t end = base + size - 1; 880 881 memblock_dbg("%s: [%pa-%pa] %pS\n", __func__, 882 &base, &end, (void *)_RET_IP_); 883 884 return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0); 885 } 886 #endif 887 888 /** 889 * memblock_setclr_flag - set or clear flag for a memory region 890 * @base: base address of the region 891 * @size: size of the region 892 * @set: set or clear the flag 893 * @flag: the flag to update 894 * 895 * This function isolates region [@base, @base + @size), and sets/clears flag 896 * 897 * Return: 0 on success, -errno on failure. 898 */ 899 static int __init_memblock memblock_setclr_flag(phys_addr_t base, 900 phys_addr_t size, int set, int flag) 901 { 902 struct memblock_type *type = &memblock.memory; 903 int i, ret, start_rgn, end_rgn; 904 905 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 906 if (ret) 907 return ret; 908 909 for (i = start_rgn; i < end_rgn; i++) { 910 struct memblock_region *r = &type->regions[i]; 911 912 if (set) 913 r->flags |= flag; 914 else 915 r->flags &= ~flag; 916 } 917 918 memblock_merge_regions(type, start_rgn, end_rgn); 919 return 0; 920 } 921 922 /** 923 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG. 924 * @base: the base phys addr of the region 925 * @size: the size of the region 926 * 927 * Return: 0 on success, -errno on failure. 928 */ 929 int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size) 930 { 931 return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG); 932 } 933 934 /** 935 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region. 936 * @base: the base phys addr of the region 937 * @size: the size of the region 938 * 939 * Return: 0 on success, -errno on failure. 940 */ 941 int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size) 942 { 943 return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG); 944 } 945 946 /** 947 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR. 948 * @base: the base phys addr of the region 949 * @size: the size of the region 950 * 951 * Return: 0 on success, -errno on failure. 952 */ 953 int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size) 954 { 955 if (!mirrored_kernelcore) 956 return 0; 957 958 system_has_some_mirror = true; 959 960 return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR); 961 } 962 963 /** 964 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP. 965 * @base: the base phys addr of the region 966 * @size: the size of the region 967 * 968 * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the 969 * direct mapping of the physical memory. These regions will still be 970 * covered by the memory map. The struct page representing NOMAP memory 971 * frames in the memory map will be PageReserved() 972 * 973 * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from 974 * memblock, the caller must inform kmemleak to ignore that memory 975 * 976 * Return: 0 on success, -errno on failure. 977 */ 978 int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size) 979 { 980 return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP); 981 } 982 983 /** 984 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region. 985 * @base: the base phys addr of the region 986 * @size: the size of the region 987 * 988 * Return: 0 on success, -errno on failure. 989 */ 990 int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size) 991 { 992 return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP); 993 } 994 995 static bool should_skip_region(struct memblock_type *type, 996 struct memblock_region *m, 997 int nid, int flags) 998 { 999 int m_nid = memblock_get_region_node(m); 1000 1001 /* we never skip regions when iterating memblock.reserved or physmem */ 1002 if (type != memblock_memory) 1003 return false; 1004 1005 /* only memory regions are associated with nodes, check it */ 1006 if (nid != NUMA_NO_NODE && nid != m_nid) 1007 return true; 1008 1009 /* skip hotpluggable memory regions if needed */ 1010 if (movable_node_is_enabled() && memblock_is_hotpluggable(m) && 1011 !(flags & MEMBLOCK_HOTPLUG)) 1012 return true; 1013 1014 /* if we want mirror memory skip non-mirror memory regions */ 1015 if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m)) 1016 return true; 1017 1018 /* skip nomap memory unless we were asked for it explicitly */ 1019 if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m)) 1020 return true; 1021 1022 /* skip driver-managed memory unless we were asked for it explicitly */ 1023 if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m)) 1024 return true; 1025 1026 return false; 1027 } 1028 1029 /** 1030 * __next_mem_range - next function for for_each_free_mem_range() etc. 1031 * @idx: pointer to u64 loop variable 1032 * @nid: node selector, %NUMA_NO_NODE for all nodes 1033 * @flags: pick from blocks based on memory attributes 1034 * @type_a: pointer to memblock_type from where the range is taken 1035 * @type_b: pointer to memblock_type which excludes memory from being taken 1036 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1037 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1038 * @out_nid: ptr to int for nid of the range, can be %NULL 1039 * 1040 * Find the first area from *@idx which matches @nid, fill the out 1041 * parameters, and update *@idx for the next iteration. The lower 32bit of 1042 * *@idx contains index into type_a and the upper 32bit indexes the 1043 * areas before each region in type_b. For example, if type_b regions 1044 * look like the following, 1045 * 1046 * 0:[0-16), 1:[32-48), 2:[128-130) 1047 * 1048 * The upper 32bit indexes the following regions. 1049 * 1050 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX) 1051 * 1052 * As both region arrays are sorted, the function advances the two indices 1053 * in lockstep and returns each intersection. 1054 */ 1055 void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags, 1056 struct memblock_type *type_a, 1057 struct memblock_type *type_b, phys_addr_t *out_start, 1058 phys_addr_t *out_end, int *out_nid) 1059 { 1060 int idx_a = *idx & 0xffffffff; 1061 int idx_b = *idx >> 32; 1062 1063 if (WARN_ONCE(nid == MAX_NUMNODES, 1064 "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 1065 nid = NUMA_NO_NODE; 1066 1067 for (; idx_a < type_a->cnt; idx_a++) { 1068 struct memblock_region *m = &type_a->regions[idx_a]; 1069 1070 phys_addr_t m_start = m->base; 1071 phys_addr_t m_end = m->base + m->size; 1072 int m_nid = memblock_get_region_node(m); 1073 1074 if (should_skip_region(type_a, m, nid, flags)) 1075 continue; 1076 1077 if (!type_b) { 1078 if (out_start) 1079 *out_start = m_start; 1080 if (out_end) 1081 *out_end = m_end; 1082 if (out_nid) 1083 *out_nid = m_nid; 1084 idx_a++; 1085 *idx = (u32)idx_a | (u64)idx_b << 32; 1086 return; 1087 } 1088 1089 /* scan areas before each reservation */ 1090 for (; idx_b < type_b->cnt + 1; idx_b++) { 1091 struct memblock_region *r; 1092 phys_addr_t r_start; 1093 phys_addr_t r_end; 1094 1095 r = &type_b->regions[idx_b]; 1096 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1097 r_end = idx_b < type_b->cnt ? 1098 r->base : PHYS_ADDR_MAX; 1099 1100 /* 1101 * if idx_b advanced past idx_a, 1102 * break out to advance idx_a 1103 */ 1104 if (r_start >= m_end) 1105 break; 1106 /* if the two regions intersect, we're done */ 1107 if (m_start < r_end) { 1108 if (out_start) 1109 *out_start = 1110 max(m_start, r_start); 1111 if (out_end) 1112 *out_end = min(m_end, r_end); 1113 if (out_nid) 1114 *out_nid = m_nid; 1115 /* 1116 * The region which ends first is 1117 * advanced for the next iteration. 1118 */ 1119 if (m_end <= r_end) 1120 idx_a++; 1121 else 1122 idx_b++; 1123 *idx = (u32)idx_a | (u64)idx_b << 32; 1124 return; 1125 } 1126 } 1127 } 1128 1129 /* signal end of iteration */ 1130 *idx = ULLONG_MAX; 1131 } 1132 1133 /** 1134 * __next_mem_range_rev - generic next function for for_each_*_range_rev() 1135 * 1136 * @idx: pointer to u64 loop variable 1137 * @nid: node selector, %NUMA_NO_NODE for all nodes 1138 * @flags: pick from blocks based on memory attributes 1139 * @type_a: pointer to memblock_type from where the range is taken 1140 * @type_b: pointer to memblock_type which excludes memory from being taken 1141 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL 1142 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL 1143 * @out_nid: ptr to int for nid of the range, can be %NULL 1144 * 1145 * Finds the next range from type_a which is not marked as unsuitable 1146 * in type_b. 1147 * 1148 * Reverse of __next_mem_range(). 1149 */ 1150 void __init_memblock __next_mem_range_rev(u64 *idx, int nid, 1151 enum memblock_flags flags, 1152 struct memblock_type *type_a, 1153 struct memblock_type *type_b, 1154 phys_addr_t *out_start, 1155 phys_addr_t *out_end, int *out_nid) 1156 { 1157 int idx_a = *idx & 0xffffffff; 1158 int idx_b = *idx >> 32; 1159 1160 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 1161 nid = NUMA_NO_NODE; 1162 1163 if (*idx == (u64)ULLONG_MAX) { 1164 idx_a = type_a->cnt - 1; 1165 if (type_b != NULL) 1166 idx_b = type_b->cnt; 1167 else 1168 idx_b = 0; 1169 } 1170 1171 for (; idx_a >= 0; idx_a--) { 1172 struct memblock_region *m = &type_a->regions[idx_a]; 1173 1174 phys_addr_t m_start = m->base; 1175 phys_addr_t m_end = m->base + m->size; 1176 int m_nid = memblock_get_region_node(m); 1177 1178 if (should_skip_region(type_a, m, nid, flags)) 1179 continue; 1180 1181 if (!type_b) { 1182 if (out_start) 1183 *out_start = m_start; 1184 if (out_end) 1185 *out_end = m_end; 1186 if (out_nid) 1187 *out_nid = m_nid; 1188 idx_a--; 1189 *idx = (u32)idx_a | (u64)idx_b << 32; 1190 return; 1191 } 1192 1193 /* scan areas before each reservation */ 1194 for (; idx_b >= 0; idx_b--) { 1195 struct memblock_region *r; 1196 phys_addr_t r_start; 1197 phys_addr_t r_end; 1198 1199 r = &type_b->regions[idx_b]; 1200 r_start = idx_b ? r[-1].base + r[-1].size : 0; 1201 r_end = idx_b < type_b->cnt ? 1202 r->base : PHYS_ADDR_MAX; 1203 /* 1204 * if idx_b advanced past idx_a, 1205 * break out to advance idx_a 1206 */ 1207 1208 if (r_end <= m_start) 1209 break; 1210 /* if the two regions intersect, we're done */ 1211 if (m_end > r_start) { 1212 if (out_start) 1213 *out_start = max(m_start, r_start); 1214 if (out_end) 1215 *out_end = min(m_end, r_end); 1216 if (out_nid) 1217 *out_nid = m_nid; 1218 if (m_start >= r_start) 1219 idx_a--; 1220 else 1221 idx_b--; 1222 *idx = (u32)idx_a | (u64)idx_b << 32; 1223 return; 1224 } 1225 } 1226 } 1227 /* signal end of iteration */ 1228 *idx = ULLONG_MAX; 1229 } 1230 1231 /* 1232 * Common iterator interface used to define for_each_mem_pfn_range(). 1233 */ 1234 void __init_memblock __next_mem_pfn_range(int *idx, int nid, 1235 unsigned long *out_start_pfn, 1236 unsigned long *out_end_pfn, int *out_nid) 1237 { 1238 struct memblock_type *type = &memblock.memory; 1239 struct memblock_region *r; 1240 int r_nid; 1241 1242 while (++*idx < type->cnt) { 1243 r = &type->regions[*idx]; 1244 r_nid = memblock_get_region_node(r); 1245 1246 if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size)) 1247 continue; 1248 if (nid == MAX_NUMNODES || nid == r_nid) 1249 break; 1250 } 1251 if (*idx >= type->cnt) { 1252 *idx = -1; 1253 return; 1254 } 1255 1256 if (out_start_pfn) 1257 *out_start_pfn = PFN_UP(r->base); 1258 if (out_end_pfn) 1259 *out_end_pfn = PFN_DOWN(r->base + r->size); 1260 if (out_nid) 1261 *out_nid = r_nid; 1262 } 1263 1264 /** 1265 * memblock_set_node - set node ID on memblock regions 1266 * @base: base of area to set node ID for 1267 * @size: size of area to set node ID for 1268 * @type: memblock type to set node ID for 1269 * @nid: node ID to set 1270 * 1271 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid. 1272 * Regions which cross the area boundaries are split as necessary. 1273 * 1274 * Return: 1275 * 0 on success, -errno on failure. 1276 */ 1277 int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size, 1278 struct memblock_type *type, int nid) 1279 { 1280 #ifdef CONFIG_NUMA 1281 int start_rgn, end_rgn; 1282 int i, ret; 1283 1284 ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn); 1285 if (ret) 1286 return ret; 1287 1288 for (i = start_rgn; i < end_rgn; i++) 1289 memblock_set_region_node(&type->regions[i], nid); 1290 1291 memblock_merge_regions(type, start_rgn, end_rgn); 1292 #endif 1293 return 0; 1294 } 1295 1296 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT 1297 /** 1298 * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone() 1299 * 1300 * @idx: pointer to u64 loop variable 1301 * @zone: zone in which all of the memory blocks reside 1302 * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL 1303 * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL 1304 * 1305 * This function is meant to be a zone/pfn specific wrapper for the 1306 * for_each_mem_range type iterators. Specifically they are used in the 1307 * deferred memory init routines and as such we were duplicating much of 1308 * this logic throughout the code. So instead of having it in multiple 1309 * locations it seemed like it would make more sense to centralize this to 1310 * one new iterator that does everything they need. 1311 */ 1312 void __init_memblock 1313 __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone, 1314 unsigned long *out_spfn, unsigned long *out_epfn) 1315 { 1316 int zone_nid = zone_to_nid(zone); 1317 phys_addr_t spa, epa; 1318 1319 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE, 1320 &memblock.memory, &memblock.reserved, 1321 &spa, &epa, NULL); 1322 1323 while (*idx != U64_MAX) { 1324 unsigned long epfn = PFN_DOWN(epa); 1325 unsigned long spfn = PFN_UP(spa); 1326 1327 /* 1328 * Verify the end is at least past the start of the zone and 1329 * that we have at least one PFN to initialize. 1330 */ 1331 if (zone->zone_start_pfn < epfn && spfn < epfn) { 1332 /* if we went too far just stop searching */ 1333 if (zone_end_pfn(zone) <= spfn) { 1334 *idx = U64_MAX; 1335 break; 1336 } 1337 1338 if (out_spfn) 1339 *out_spfn = max(zone->zone_start_pfn, spfn); 1340 if (out_epfn) 1341 *out_epfn = min(zone_end_pfn(zone), epfn); 1342 1343 return; 1344 } 1345 1346 __next_mem_range(idx, zone_nid, MEMBLOCK_NONE, 1347 &memblock.memory, &memblock.reserved, 1348 &spa, &epa, NULL); 1349 } 1350 1351 /* signal end of iteration */ 1352 if (out_spfn) 1353 *out_spfn = ULONG_MAX; 1354 if (out_epfn) 1355 *out_epfn = 0; 1356 } 1357 1358 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ 1359 1360 /** 1361 * memblock_alloc_range_nid - allocate boot memory block 1362 * @size: size of memory block to be allocated in bytes 1363 * @align: alignment of the region and block's size 1364 * @start: the lower bound of the memory region to allocate (phys address) 1365 * @end: the upper bound of the memory region to allocate (phys address) 1366 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1367 * @exact_nid: control the allocation fall back to other nodes 1368 * 1369 * The allocation is performed from memory region limited by 1370 * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE. 1371 * 1372 * If the specified node can not hold the requested memory and @exact_nid 1373 * is false, the allocation falls back to any node in the system. 1374 * 1375 * For systems with memory mirroring, the allocation is attempted first 1376 * from the regions with mirroring enabled and then retried from any 1377 * memory region. 1378 * 1379 * In addition, function using kmemleak_alloc_phys for allocated boot 1380 * memory block, it is never reported as leaks. 1381 * 1382 * Return: 1383 * Physical address of allocated memory block on success, %0 on failure. 1384 */ 1385 phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size, 1386 phys_addr_t align, phys_addr_t start, 1387 phys_addr_t end, int nid, 1388 bool exact_nid) 1389 { 1390 enum memblock_flags flags = choose_memblock_flags(); 1391 phys_addr_t found; 1392 1393 if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n")) 1394 nid = NUMA_NO_NODE; 1395 1396 if (!align) { 1397 /* Can't use WARNs this early in boot on powerpc */ 1398 dump_stack(); 1399 align = SMP_CACHE_BYTES; 1400 } 1401 1402 again: 1403 found = memblock_find_in_range_node(size, align, start, end, nid, 1404 flags); 1405 if (found && !memblock_reserve(found, size)) 1406 goto done; 1407 1408 if (nid != NUMA_NO_NODE && !exact_nid) { 1409 found = memblock_find_in_range_node(size, align, start, 1410 end, NUMA_NO_NODE, 1411 flags); 1412 if (found && !memblock_reserve(found, size)) 1413 goto done; 1414 } 1415 1416 if (flags & MEMBLOCK_MIRROR) { 1417 flags &= ~MEMBLOCK_MIRROR; 1418 pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n", 1419 &size); 1420 goto again; 1421 } 1422 1423 return 0; 1424 1425 done: 1426 /* 1427 * Skip kmemleak for those places like kasan_init() and 1428 * early_pgtable_alloc() due to high volume. 1429 */ 1430 if (end != MEMBLOCK_ALLOC_NOLEAKTRACE) 1431 /* 1432 * Memblock allocated blocks are never reported as 1433 * leaks. This is because many of these blocks are 1434 * only referred via the physical address which is 1435 * not looked up by kmemleak. 1436 */ 1437 kmemleak_alloc_phys(found, size, 0); 1438 1439 /* 1440 * Some Virtual Machine platforms, such as Intel TDX or AMD SEV-SNP, 1441 * require memory to be accepted before it can be used by the 1442 * guest. 1443 * 1444 * Accept the memory of the allocated buffer. 1445 */ 1446 accept_memory(found, found + size); 1447 1448 return found; 1449 } 1450 1451 /** 1452 * memblock_phys_alloc_range - allocate a memory block inside specified range 1453 * @size: size of memory block to be allocated in bytes 1454 * @align: alignment of the region and block's size 1455 * @start: the lower bound of the memory region to allocate (physical address) 1456 * @end: the upper bound of the memory region to allocate (physical address) 1457 * 1458 * Allocate @size bytes in the between @start and @end. 1459 * 1460 * Return: physical address of the allocated memory block on success, 1461 * %0 on failure. 1462 */ 1463 phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size, 1464 phys_addr_t align, 1465 phys_addr_t start, 1466 phys_addr_t end) 1467 { 1468 memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n", 1469 __func__, (u64)size, (u64)align, &start, &end, 1470 (void *)_RET_IP_); 1471 return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE, 1472 false); 1473 } 1474 1475 /** 1476 * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node 1477 * @size: size of memory block to be allocated in bytes 1478 * @align: alignment of the region and block's size 1479 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1480 * 1481 * Allocates memory block from the specified NUMA node. If the node 1482 * has no available memory, attempts to allocated from any node in the 1483 * system. 1484 * 1485 * Return: physical address of the allocated memory block on success, 1486 * %0 on failure. 1487 */ 1488 phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid) 1489 { 1490 return memblock_alloc_range_nid(size, align, 0, 1491 MEMBLOCK_ALLOC_ACCESSIBLE, nid, false); 1492 } 1493 1494 /** 1495 * memblock_alloc_internal - allocate boot memory block 1496 * @size: size of memory block to be allocated in bytes 1497 * @align: alignment of the region and block's size 1498 * @min_addr: the lower bound of the memory region to allocate (phys address) 1499 * @max_addr: the upper bound of the memory region to allocate (phys address) 1500 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1501 * @exact_nid: control the allocation fall back to other nodes 1502 * 1503 * Allocates memory block using memblock_alloc_range_nid() and 1504 * converts the returned physical address to virtual. 1505 * 1506 * The @min_addr limit is dropped if it can not be satisfied and the allocation 1507 * will fall back to memory below @min_addr. Other constraints, such 1508 * as node and mirrored memory will be handled again in 1509 * memblock_alloc_range_nid(). 1510 * 1511 * Return: 1512 * Virtual address of allocated memory block on success, NULL on failure. 1513 */ 1514 static void * __init memblock_alloc_internal( 1515 phys_addr_t size, phys_addr_t align, 1516 phys_addr_t min_addr, phys_addr_t max_addr, 1517 int nid, bool exact_nid) 1518 { 1519 phys_addr_t alloc; 1520 1521 /* 1522 * Detect any accidental use of these APIs after slab is ready, as at 1523 * this moment memblock may be deinitialized already and its 1524 * internal data may be destroyed (after execution of memblock_free_all) 1525 */ 1526 if (WARN_ON_ONCE(slab_is_available())) 1527 return kzalloc_node(size, GFP_NOWAIT, nid); 1528 1529 if (max_addr > memblock.current_limit) 1530 max_addr = memblock.current_limit; 1531 1532 alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid, 1533 exact_nid); 1534 1535 /* retry allocation without lower limit */ 1536 if (!alloc && min_addr) 1537 alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid, 1538 exact_nid); 1539 1540 if (!alloc) 1541 return NULL; 1542 1543 return phys_to_virt(alloc); 1544 } 1545 1546 /** 1547 * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node 1548 * without zeroing memory 1549 * @size: size of memory block to be allocated in bytes 1550 * @align: alignment of the region and block's size 1551 * @min_addr: the lower bound of the memory region from where the allocation 1552 * is preferred (phys address) 1553 * @max_addr: the upper bound of the memory region from where the allocation 1554 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1555 * allocate only from memory limited by memblock.current_limit value 1556 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1557 * 1558 * Public function, provides additional debug information (including caller 1559 * info), if enabled. Does not zero allocated memory. 1560 * 1561 * Return: 1562 * Virtual address of allocated memory block on success, NULL on failure. 1563 */ 1564 void * __init memblock_alloc_exact_nid_raw( 1565 phys_addr_t size, phys_addr_t align, 1566 phys_addr_t min_addr, phys_addr_t max_addr, 1567 int nid) 1568 { 1569 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1570 __func__, (u64)size, (u64)align, nid, &min_addr, 1571 &max_addr, (void *)_RET_IP_); 1572 1573 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1574 true); 1575 } 1576 1577 /** 1578 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing 1579 * memory and without panicking 1580 * @size: size of memory block to be allocated in bytes 1581 * @align: alignment of the region and block's size 1582 * @min_addr: the lower bound of the memory region from where the allocation 1583 * is preferred (phys address) 1584 * @max_addr: the upper bound of the memory region from where the allocation 1585 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1586 * allocate only from memory limited by memblock.current_limit value 1587 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1588 * 1589 * Public function, provides additional debug information (including caller 1590 * info), if enabled. Does not zero allocated memory, does not panic if request 1591 * cannot be satisfied. 1592 * 1593 * Return: 1594 * Virtual address of allocated memory block on success, NULL on failure. 1595 */ 1596 void * __init memblock_alloc_try_nid_raw( 1597 phys_addr_t size, phys_addr_t align, 1598 phys_addr_t min_addr, phys_addr_t max_addr, 1599 int nid) 1600 { 1601 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1602 __func__, (u64)size, (u64)align, nid, &min_addr, 1603 &max_addr, (void *)_RET_IP_); 1604 1605 return memblock_alloc_internal(size, align, min_addr, max_addr, nid, 1606 false); 1607 } 1608 1609 /** 1610 * memblock_alloc_try_nid - allocate boot memory block 1611 * @size: size of memory block to be allocated in bytes 1612 * @align: alignment of the region and block's size 1613 * @min_addr: the lower bound of the memory region from where the allocation 1614 * is preferred (phys address) 1615 * @max_addr: the upper bound of the memory region from where the allocation 1616 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to 1617 * allocate only from memory limited by memblock.current_limit value 1618 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node 1619 * 1620 * Public function, provides additional debug information (including caller 1621 * info), if enabled. This function zeroes the allocated memory. 1622 * 1623 * Return: 1624 * Virtual address of allocated memory block on success, NULL on failure. 1625 */ 1626 void * __init memblock_alloc_try_nid( 1627 phys_addr_t size, phys_addr_t align, 1628 phys_addr_t min_addr, phys_addr_t max_addr, 1629 int nid) 1630 { 1631 void *ptr; 1632 1633 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n", 1634 __func__, (u64)size, (u64)align, nid, &min_addr, 1635 &max_addr, (void *)_RET_IP_); 1636 ptr = memblock_alloc_internal(size, align, 1637 min_addr, max_addr, nid, false); 1638 if (ptr) 1639 memset(ptr, 0, size); 1640 1641 return ptr; 1642 } 1643 1644 /** 1645 * memblock_free_late - free pages directly to buddy allocator 1646 * @base: phys starting address of the boot memory block 1647 * @size: size of the boot memory block in bytes 1648 * 1649 * This is only useful when the memblock allocator has already been torn 1650 * down, but we are still initializing the system. Pages are released directly 1651 * to the buddy allocator. 1652 */ 1653 void __init memblock_free_late(phys_addr_t base, phys_addr_t size) 1654 { 1655 phys_addr_t cursor, end; 1656 1657 end = base + size - 1; 1658 memblock_dbg("%s: [%pa-%pa] %pS\n", 1659 __func__, &base, &end, (void *)_RET_IP_); 1660 kmemleak_free_part_phys(base, size); 1661 cursor = PFN_UP(base); 1662 end = PFN_DOWN(base + size); 1663 1664 for (; cursor < end; cursor++) { 1665 memblock_free_pages(pfn_to_page(cursor), cursor, 0); 1666 totalram_pages_inc(); 1667 } 1668 } 1669 1670 /* 1671 * Remaining API functions 1672 */ 1673 1674 phys_addr_t __init_memblock memblock_phys_mem_size(void) 1675 { 1676 return memblock.memory.total_size; 1677 } 1678 1679 phys_addr_t __init_memblock memblock_reserved_size(void) 1680 { 1681 return memblock.reserved.total_size; 1682 } 1683 1684 /* lowest address */ 1685 phys_addr_t __init_memblock memblock_start_of_DRAM(void) 1686 { 1687 return memblock.memory.regions[0].base; 1688 } 1689 1690 phys_addr_t __init_memblock memblock_end_of_DRAM(void) 1691 { 1692 int idx = memblock.memory.cnt - 1; 1693 1694 return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size); 1695 } 1696 1697 static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit) 1698 { 1699 phys_addr_t max_addr = PHYS_ADDR_MAX; 1700 struct memblock_region *r; 1701 1702 /* 1703 * translate the memory @limit size into the max address within one of 1704 * the memory memblock regions, if the @limit exceeds the total size 1705 * of those regions, max_addr will keep original value PHYS_ADDR_MAX 1706 */ 1707 for_each_mem_region(r) { 1708 if (limit <= r->size) { 1709 max_addr = r->base + limit; 1710 break; 1711 } 1712 limit -= r->size; 1713 } 1714 1715 return max_addr; 1716 } 1717 1718 void __init memblock_enforce_memory_limit(phys_addr_t limit) 1719 { 1720 phys_addr_t max_addr; 1721 1722 if (!limit) 1723 return; 1724 1725 max_addr = __find_max_addr(limit); 1726 1727 /* @limit exceeds the total size of the memory, do nothing */ 1728 if (max_addr == PHYS_ADDR_MAX) 1729 return; 1730 1731 /* truncate both memory and reserved regions */ 1732 memblock_remove_range(&memblock.memory, max_addr, 1733 PHYS_ADDR_MAX); 1734 memblock_remove_range(&memblock.reserved, max_addr, 1735 PHYS_ADDR_MAX); 1736 } 1737 1738 void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size) 1739 { 1740 int start_rgn, end_rgn; 1741 int i, ret; 1742 1743 if (!size) 1744 return; 1745 1746 if (!memblock_memory->total_size) { 1747 pr_warn("%s: No memory registered yet\n", __func__); 1748 return; 1749 } 1750 1751 ret = memblock_isolate_range(&memblock.memory, base, size, 1752 &start_rgn, &end_rgn); 1753 if (ret) 1754 return; 1755 1756 /* remove all the MAP regions */ 1757 for (i = memblock.memory.cnt - 1; i >= end_rgn; i--) 1758 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1759 memblock_remove_region(&memblock.memory, i); 1760 1761 for (i = start_rgn - 1; i >= 0; i--) 1762 if (!memblock_is_nomap(&memblock.memory.regions[i])) 1763 memblock_remove_region(&memblock.memory, i); 1764 1765 /* truncate the reserved regions */ 1766 memblock_remove_range(&memblock.reserved, 0, base); 1767 memblock_remove_range(&memblock.reserved, 1768 base + size, PHYS_ADDR_MAX); 1769 } 1770 1771 void __init memblock_mem_limit_remove_map(phys_addr_t limit) 1772 { 1773 phys_addr_t max_addr; 1774 1775 if (!limit) 1776 return; 1777 1778 max_addr = __find_max_addr(limit); 1779 1780 /* @limit exceeds the total size of the memory, do nothing */ 1781 if (max_addr == PHYS_ADDR_MAX) 1782 return; 1783 1784 memblock_cap_memory_range(0, max_addr); 1785 } 1786 1787 static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr) 1788 { 1789 unsigned int left = 0, right = type->cnt; 1790 1791 do { 1792 unsigned int mid = (right + left) / 2; 1793 1794 if (addr < type->regions[mid].base) 1795 right = mid; 1796 else if (addr >= (type->regions[mid].base + 1797 type->regions[mid].size)) 1798 left = mid + 1; 1799 else 1800 return mid; 1801 } while (left < right); 1802 return -1; 1803 } 1804 1805 bool __init_memblock memblock_is_reserved(phys_addr_t addr) 1806 { 1807 return memblock_search(&memblock.reserved, addr) != -1; 1808 } 1809 1810 bool __init_memblock memblock_is_memory(phys_addr_t addr) 1811 { 1812 return memblock_search(&memblock.memory, addr) != -1; 1813 } 1814 1815 bool __init_memblock memblock_is_map_memory(phys_addr_t addr) 1816 { 1817 int i = memblock_search(&memblock.memory, addr); 1818 1819 if (i == -1) 1820 return false; 1821 return !memblock_is_nomap(&memblock.memory.regions[i]); 1822 } 1823 1824 int __init_memblock memblock_search_pfn_nid(unsigned long pfn, 1825 unsigned long *start_pfn, unsigned long *end_pfn) 1826 { 1827 struct memblock_type *type = &memblock.memory; 1828 int mid = memblock_search(type, PFN_PHYS(pfn)); 1829 1830 if (mid == -1) 1831 return -1; 1832 1833 *start_pfn = PFN_DOWN(type->regions[mid].base); 1834 *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size); 1835 1836 return memblock_get_region_node(&type->regions[mid]); 1837 } 1838 1839 /** 1840 * memblock_is_region_memory - check if a region is a subset of memory 1841 * @base: base of region to check 1842 * @size: size of region to check 1843 * 1844 * Check if the region [@base, @base + @size) is a subset of a memory block. 1845 * 1846 * Return: 1847 * 0 if false, non-zero if true 1848 */ 1849 bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size) 1850 { 1851 int idx = memblock_search(&memblock.memory, base); 1852 phys_addr_t end = base + memblock_cap_size(base, &size); 1853 1854 if (idx == -1) 1855 return false; 1856 return (memblock.memory.regions[idx].base + 1857 memblock.memory.regions[idx].size) >= end; 1858 } 1859 1860 /** 1861 * memblock_is_region_reserved - check if a region intersects reserved memory 1862 * @base: base of region to check 1863 * @size: size of region to check 1864 * 1865 * Check if the region [@base, @base + @size) intersects a reserved 1866 * memory block. 1867 * 1868 * Return: 1869 * True if they intersect, false if not. 1870 */ 1871 bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size) 1872 { 1873 return memblock_overlaps_region(&memblock.reserved, base, size); 1874 } 1875 1876 void __init_memblock memblock_trim_memory(phys_addr_t align) 1877 { 1878 phys_addr_t start, end, orig_start, orig_end; 1879 struct memblock_region *r; 1880 1881 for_each_mem_region(r) { 1882 orig_start = r->base; 1883 orig_end = r->base + r->size; 1884 start = round_up(orig_start, align); 1885 end = round_down(orig_end, align); 1886 1887 if (start == orig_start && end == orig_end) 1888 continue; 1889 1890 if (start < end) { 1891 r->base = start; 1892 r->size = end - start; 1893 } else { 1894 memblock_remove_region(&memblock.memory, 1895 r - memblock.memory.regions); 1896 r--; 1897 } 1898 } 1899 } 1900 1901 void __init_memblock memblock_set_current_limit(phys_addr_t limit) 1902 { 1903 memblock.current_limit = limit; 1904 } 1905 1906 phys_addr_t __init_memblock memblock_get_current_limit(void) 1907 { 1908 return memblock.current_limit; 1909 } 1910 1911 static void __init_memblock memblock_dump(struct memblock_type *type) 1912 { 1913 phys_addr_t base, end, size; 1914 enum memblock_flags flags; 1915 int idx; 1916 struct memblock_region *rgn; 1917 1918 pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt); 1919 1920 for_each_memblock_type(idx, type, rgn) { 1921 char nid_buf[32] = ""; 1922 1923 base = rgn->base; 1924 size = rgn->size; 1925 end = base + size - 1; 1926 flags = rgn->flags; 1927 #ifdef CONFIG_NUMA 1928 if (memblock_get_region_node(rgn) != MAX_NUMNODES) 1929 snprintf(nid_buf, sizeof(nid_buf), " on node %d", 1930 memblock_get_region_node(rgn)); 1931 #endif 1932 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n", 1933 type->name, idx, &base, &end, &size, nid_buf, flags); 1934 } 1935 } 1936 1937 static void __init_memblock __memblock_dump_all(void) 1938 { 1939 pr_info("MEMBLOCK configuration:\n"); 1940 pr_info(" memory size = %pa reserved size = %pa\n", 1941 &memblock.memory.total_size, 1942 &memblock.reserved.total_size); 1943 1944 memblock_dump(&memblock.memory); 1945 memblock_dump(&memblock.reserved); 1946 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 1947 memblock_dump(&physmem); 1948 #endif 1949 } 1950 1951 void __init_memblock memblock_dump_all(void) 1952 { 1953 if (memblock_debug) 1954 __memblock_dump_all(); 1955 } 1956 1957 void __init memblock_allow_resize(void) 1958 { 1959 memblock_can_resize = 1; 1960 } 1961 1962 static int __init early_memblock(char *p) 1963 { 1964 if (p && strstr(p, "debug")) 1965 memblock_debug = 1; 1966 return 0; 1967 } 1968 early_param("memblock", early_memblock); 1969 1970 static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn) 1971 { 1972 struct page *start_pg, *end_pg; 1973 phys_addr_t pg, pgend; 1974 1975 /* 1976 * Convert start_pfn/end_pfn to a struct page pointer. 1977 */ 1978 start_pg = pfn_to_page(start_pfn - 1) + 1; 1979 end_pg = pfn_to_page(end_pfn - 1) + 1; 1980 1981 /* 1982 * Convert to physical addresses, and round start upwards and end 1983 * downwards. 1984 */ 1985 pg = PAGE_ALIGN(__pa(start_pg)); 1986 pgend = __pa(end_pg) & PAGE_MASK; 1987 1988 /* 1989 * If there are free pages between these, free the section of the 1990 * memmap array. 1991 */ 1992 if (pg < pgend) 1993 memblock_phys_free(pg, pgend - pg); 1994 } 1995 1996 /* 1997 * The mem_map array can get very big. Free the unused area of the memory map. 1998 */ 1999 static void __init free_unused_memmap(void) 2000 { 2001 unsigned long start, end, prev_end = 0; 2002 int i; 2003 2004 if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) || 2005 IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) 2006 return; 2007 2008 /* 2009 * This relies on each bank being in address order. 2010 * The banks are sorted previously in bootmem_init(). 2011 */ 2012 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) { 2013 #ifdef CONFIG_SPARSEMEM 2014 /* 2015 * Take care not to free memmap entries that don't exist 2016 * due to SPARSEMEM sections which aren't present. 2017 */ 2018 start = min(start, ALIGN(prev_end, PAGES_PER_SECTION)); 2019 #endif 2020 /* 2021 * Align down here since many operations in VM subsystem 2022 * presume that there are no holes in the memory map inside 2023 * a pageblock 2024 */ 2025 start = pageblock_start_pfn(start); 2026 2027 /* 2028 * If we had a previous bank, and there is a space 2029 * between the current bank and the previous, free it. 2030 */ 2031 if (prev_end && prev_end < start) 2032 free_memmap(prev_end, start); 2033 2034 /* 2035 * Align up here since many operations in VM subsystem 2036 * presume that there are no holes in the memory map inside 2037 * a pageblock 2038 */ 2039 prev_end = pageblock_align(end); 2040 } 2041 2042 #ifdef CONFIG_SPARSEMEM 2043 if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) { 2044 prev_end = pageblock_align(end); 2045 free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); 2046 } 2047 #endif 2048 } 2049 2050 static void __init __free_pages_memory(unsigned long start, unsigned long end) 2051 { 2052 int order; 2053 2054 while (start < end) { 2055 /* 2056 * Free the pages in the largest chunks alignment allows. 2057 * 2058 * __ffs() behaviour is undefined for 0. start == 0 is 2059 * MAX_ORDER-aligned, set order to MAX_ORDER for the case. 2060 */ 2061 if (start) 2062 order = min_t(int, MAX_ORDER, __ffs(start)); 2063 else 2064 order = MAX_ORDER; 2065 2066 while (start + (1UL << order) > end) 2067 order--; 2068 2069 memblock_free_pages(pfn_to_page(start), start, order); 2070 2071 start += (1UL << order); 2072 } 2073 } 2074 2075 static unsigned long __init __free_memory_core(phys_addr_t start, 2076 phys_addr_t end) 2077 { 2078 unsigned long start_pfn = PFN_UP(start); 2079 unsigned long end_pfn = min_t(unsigned long, 2080 PFN_DOWN(end), max_low_pfn); 2081 2082 if (start_pfn >= end_pfn) 2083 return 0; 2084 2085 __free_pages_memory(start_pfn, end_pfn); 2086 2087 return end_pfn - start_pfn; 2088 } 2089 2090 static void __init memmap_init_reserved_pages(void) 2091 { 2092 struct memblock_region *region; 2093 phys_addr_t start, end; 2094 u64 i; 2095 2096 /* initialize struct pages for the reserved regions */ 2097 for_each_reserved_mem_range(i, &start, &end) 2098 reserve_bootmem_region(start, end); 2099 2100 /* and also treat struct pages for the NOMAP regions as PageReserved */ 2101 for_each_mem_region(region) { 2102 if (memblock_is_nomap(region)) { 2103 start = region->base; 2104 end = start + region->size; 2105 reserve_bootmem_region(start, end); 2106 } 2107 } 2108 } 2109 2110 static unsigned long __init free_low_memory_core_early(void) 2111 { 2112 unsigned long count = 0; 2113 phys_addr_t start, end; 2114 u64 i; 2115 2116 memblock_clear_hotplug(0, -1); 2117 2118 memmap_init_reserved_pages(); 2119 2120 /* 2121 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id 2122 * because in some case like Node0 doesn't have RAM installed 2123 * low ram will be on Node1 2124 */ 2125 for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, 2126 NULL) 2127 count += __free_memory_core(start, end); 2128 2129 return count; 2130 } 2131 2132 static int reset_managed_pages_done __initdata; 2133 2134 void reset_node_managed_pages(pg_data_t *pgdat) 2135 { 2136 struct zone *z; 2137 2138 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 2139 atomic_long_set(&z->managed_pages, 0); 2140 } 2141 2142 void __init reset_all_zones_managed_pages(void) 2143 { 2144 struct pglist_data *pgdat; 2145 2146 if (reset_managed_pages_done) 2147 return; 2148 2149 for_each_online_pgdat(pgdat) 2150 reset_node_managed_pages(pgdat); 2151 2152 reset_managed_pages_done = 1; 2153 } 2154 2155 /** 2156 * memblock_free_all - release free pages to the buddy allocator 2157 */ 2158 void __init memblock_free_all(void) 2159 { 2160 unsigned long pages; 2161 2162 free_unused_memmap(); 2163 reset_all_zones_managed_pages(); 2164 2165 pages = free_low_memory_core_early(); 2166 totalram_pages_add(pages); 2167 } 2168 2169 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK) 2170 2171 static int memblock_debug_show(struct seq_file *m, void *private) 2172 { 2173 struct memblock_type *type = m->private; 2174 struct memblock_region *reg; 2175 int i; 2176 phys_addr_t end; 2177 2178 for (i = 0; i < type->cnt; i++) { 2179 reg = &type->regions[i]; 2180 end = reg->base + reg->size - 1; 2181 2182 seq_printf(m, "%4d: ", i); 2183 seq_printf(m, "%pa..%pa\n", ®->base, &end); 2184 } 2185 return 0; 2186 } 2187 DEFINE_SHOW_ATTRIBUTE(memblock_debug); 2188 2189 static int __init memblock_init_debugfs(void) 2190 { 2191 struct dentry *root = debugfs_create_dir("memblock", NULL); 2192 2193 debugfs_create_file("memory", 0444, root, 2194 &memblock.memory, &memblock_debug_fops); 2195 debugfs_create_file("reserved", 0444, root, 2196 &memblock.reserved, &memblock_debug_fops); 2197 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP 2198 debugfs_create_file("physmem", 0444, root, &physmem, 2199 &memblock_debug_fops); 2200 #endif 2201 2202 return 0; 2203 } 2204 __initcall(memblock_init_debugfs); 2205 2206 #endif /* CONFIG_DEBUG_FS */ 2207