1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019, Intel Corporation. 4 * 5 * Heterogeneous Memory Attributes Table (HMAT) representation 6 * 7 * This program parses and reports the platform's HMAT tables, and registers 8 * the applicable attributes with the node's interfaces. 9 */ 10 11 #define pr_fmt(fmt) "acpi/hmat: " fmt 12 13 #include <linux/acpi.h> 14 #include <linux/bitops.h> 15 #include <linux/device.h> 16 #include <linux/init.h> 17 #include <linux/list.h> 18 #include <linux/mm.h> 19 #include <linux/platform_device.h> 20 #include <linux/list_sort.h> 21 #include <linux/memregion.h> 22 #include <linux/memory.h> 23 #include <linux/mutex.h> 24 #include <linux/node.h> 25 #include <linux/sysfs.h> 26 #include <linux/dax.h> 27 28 static u8 hmat_revision; 29 static int hmat_disable __initdata; 30 31 void __init disable_hmat(void) 32 { 33 hmat_disable = 1; 34 } 35 36 static LIST_HEAD(targets); 37 static LIST_HEAD(initiators); 38 static LIST_HEAD(localities); 39 40 static DEFINE_MUTEX(target_lock); 41 42 /* 43 * The defined enum order is used to prioritize attributes to break ties when 44 * selecting the best performing node. 45 */ 46 enum locality_types { 47 WRITE_LATENCY, 48 READ_LATENCY, 49 WRITE_BANDWIDTH, 50 READ_BANDWIDTH, 51 }; 52 53 static struct memory_locality *localities_types[4]; 54 55 struct target_cache { 56 struct list_head node; 57 struct node_cache_attrs cache_attrs; 58 }; 59 60 struct memory_target { 61 struct list_head node; 62 unsigned int memory_pxm; 63 unsigned int processor_pxm; 64 struct resource memregions; 65 struct node_hmem_attrs hmem_attrs[2]; 66 struct list_head caches; 67 struct node_cache_attrs cache_attrs; 68 bool registered; 69 }; 70 71 struct memory_initiator { 72 struct list_head node; 73 unsigned int processor_pxm; 74 bool has_cpu; 75 }; 76 77 struct memory_locality { 78 struct list_head node; 79 struct acpi_hmat_locality *hmat_loc; 80 }; 81 82 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm) 83 { 84 struct memory_initiator *initiator; 85 86 list_for_each_entry(initiator, &initiators, node) 87 if (initiator->processor_pxm == cpu_pxm) 88 return initiator; 89 return NULL; 90 } 91 92 static struct memory_target *find_mem_target(unsigned int mem_pxm) 93 { 94 struct memory_target *target; 95 96 list_for_each_entry(target, &targets, node) 97 if (target->memory_pxm == mem_pxm) 98 return target; 99 return NULL; 100 } 101 102 static __init void alloc_memory_initiator(unsigned int cpu_pxm) 103 { 104 struct memory_initiator *initiator; 105 106 if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE) 107 return; 108 109 initiator = find_mem_initiator(cpu_pxm); 110 if (initiator) 111 return; 112 113 initiator = kzalloc(sizeof(*initiator), GFP_KERNEL); 114 if (!initiator) 115 return; 116 117 initiator->processor_pxm = cpu_pxm; 118 initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU); 119 list_add_tail(&initiator->node, &initiators); 120 } 121 122 static __init void alloc_memory_target(unsigned int mem_pxm, 123 resource_size_t start, resource_size_t len) 124 { 125 struct memory_target *target; 126 127 target = find_mem_target(mem_pxm); 128 if (!target) { 129 target = kzalloc(sizeof(*target), GFP_KERNEL); 130 if (!target) 131 return; 132 target->memory_pxm = mem_pxm; 133 target->processor_pxm = PXM_INVAL; 134 target->memregions = (struct resource) { 135 .name = "ACPI mem", 136 .start = 0, 137 .end = -1, 138 .flags = IORESOURCE_MEM, 139 }; 140 list_add_tail(&target->node, &targets); 141 INIT_LIST_HEAD(&target->caches); 142 } 143 144 /* 145 * There are potentially multiple ranges per PXM, so record each 146 * in the per-target memregions resource tree. 147 */ 148 if (!__request_region(&target->memregions, start, len, "memory target", 149 IORESOURCE_MEM)) 150 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n", 151 start, start + len, mem_pxm); 152 } 153 154 static __init const char *hmat_data_type(u8 type) 155 { 156 switch (type) { 157 case ACPI_HMAT_ACCESS_LATENCY: 158 return "Access Latency"; 159 case ACPI_HMAT_READ_LATENCY: 160 return "Read Latency"; 161 case ACPI_HMAT_WRITE_LATENCY: 162 return "Write Latency"; 163 case ACPI_HMAT_ACCESS_BANDWIDTH: 164 return "Access Bandwidth"; 165 case ACPI_HMAT_READ_BANDWIDTH: 166 return "Read Bandwidth"; 167 case ACPI_HMAT_WRITE_BANDWIDTH: 168 return "Write Bandwidth"; 169 default: 170 return "Reserved"; 171 } 172 } 173 174 static __init const char *hmat_data_type_suffix(u8 type) 175 { 176 switch (type) { 177 case ACPI_HMAT_ACCESS_LATENCY: 178 case ACPI_HMAT_READ_LATENCY: 179 case ACPI_HMAT_WRITE_LATENCY: 180 return " nsec"; 181 case ACPI_HMAT_ACCESS_BANDWIDTH: 182 case ACPI_HMAT_READ_BANDWIDTH: 183 case ACPI_HMAT_WRITE_BANDWIDTH: 184 return " MB/s"; 185 default: 186 return ""; 187 } 188 } 189 190 static u32 hmat_normalize(u16 entry, u64 base, u8 type) 191 { 192 u32 value; 193 194 /* 195 * Check for invalid and overflow values 196 */ 197 if (entry == 0xffff || !entry) 198 return 0; 199 else if (base > (UINT_MAX / (entry))) 200 return 0; 201 202 /* 203 * Divide by the base unit for version 1, convert latency from 204 * picosenonds to nanoseconds if revision 2. 205 */ 206 value = entry * base; 207 if (hmat_revision == 1) { 208 if (value < 10) 209 return 0; 210 value = DIV_ROUND_UP(value, 10); 211 } else if (hmat_revision == 2) { 212 switch (type) { 213 case ACPI_HMAT_ACCESS_LATENCY: 214 case ACPI_HMAT_READ_LATENCY: 215 case ACPI_HMAT_WRITE_LATENCY: 216 value = DIV_ROUND_UP(value, 1000); 217 break; 218 default: 219 break; 220 } 221 } 222 return value; 223 } 224 225 static void hmat_update_target_access(struct memory_target *target, 226 u8 type, u32 value, int access) 227 { 228 switch (type) { 229 case ACPI_HMAT_ACCESS_LATENCY: 230 target->hmem_attrs[access].read_latency = value; 231 target->hmem_attrs[access].write_latency = value; 232 break; 233 case ACPI_HMAT_READ_LATENCY: 234 target->hmem_attrs[access].read_latency = value; 235 break; 236 case ACPI_HMAT_WRITE_LATENCY: 237 target->hmem_attrs[access].write_latency = value; 238 break; 239 case ACPI_HMAT_ACCESS_BANDWIDTH: 240 target->hmem_attrs[access].read_bandwidth = value; 241 target->hmem_attrs[access].write_bandwidth = value; 242 break; 243 case ACPI_HMAT_READ_BANDWIDTH: 244 target->hmem_attrs[access].read_bandwidth = value; 245 break; 246 case ACPI_HMAT_WRITE_BANDWIDTH: 247 target->hmem_attrs[access].write_bandwidth = value; 248 break; 249 default: 250 break; 251 } 252 } 253 254 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) 255 { 256 struct memory_locality *loc; 257 258 loc = kzalloc(sizeof(*loc), GFP_KERNEL); 259 if (!loc) { 260 pr_notice_once("Failed to allocate HMAT locality\n"); 261 return; 262 } 263 264 loc->hmat_loc = hmat_loc; 265 list_add_tail(&loc->node, &localities); 266 267 switch (hmat_loc->data_type) { 268 case ACPI_HMAT_ACCESS_LATENCY: 269 localities_types[READ_LATENCY] = loc; 270 localities_types[WRITE_LATENCY] = loc; 271 break; 272 case ACPI_HMAT_READ_LATENCY: 273 localities_types[READ_LATENCY] = loc; 274 break; 275 case ACPI_HMAT_WRITE_LATENCY: 276 localities_types[WRITE_LATENCY] = loc; 277 break; 278 case ACPI_HMAT_ACCESS_BANDWIDTH: 279 localities_types[READ_BANDWIDTH] = loc; 280 localities_types[WRITE_BANDWIDTH] = loc; 281 break; 282 case ACPI_HMAT_READ_BANDWIDTH: 283 localities_types[READ_BANDWIDTH] = loc; 284 break; 285 case ACPI_HMAT_WRITE_BANDWIDTH: 286 localities_types[WRITE_BANDWIDTH] = loc; 287 break; 288 default: 289 break; 290 } 291 } 292 293 static __init int hmat_parse_locality(union acpi_subtable_headers *header, 294 const unsigned long end) 295 { 296 struct acpi_hmat_locality *hmat_loc = (void *)header; 297 struct memory_target *target; 298 unsigned int init, targ, total_size, ipds, tpds; 299 u32 *inits, *targs, value; 300 u16 *entries; 301 u8 type, mem_hier; 302 303 if (hmat_loc->header.length < sizeof(*hmat_loc)) { 304 pr_notice("Unexpected locality header length: %u\n", 305 hmat_loc->header.length); 306 return -EINVAL; 307 } 308 309 type = hmat_loc->data_type; 310 mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY; 311 ipds = hmat_loc->number_of_initiator_Pds; 312 tpds = hmat_loc->number_of_target_Pds; 313 total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds + 314 sizeof(*inits) * ipds + sizeof(*targs) * tpds; 315 if (hmat_loc->header.length < total_size) { 316 pr_notice("Unexpected locality header length:%u, minimum required:%u\n", 317 hmat_loc->header.length, total_size); 318 return -EINVAL; 319 } 320 321 pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n", 322 hmat_loc->flags, hmat_data_type(type), ipds, tpds, 323 hmat_loc->entry_base_unit); 324 325 inits = (u32 *)(hmat_loc + 1); 326 targs = inits + ipds; 327 entries = (u16 *)(targs + tpds); 328 for (init = 0; init < ipds; init++) { 329 alloc_memory_initiator(inits[init]); 330 for (targ = 0; targ < tpds; targ++) { 331 value = hmat_normalize(entries[init * tpds + targ], 332 hmat_loc->entry_base_unit, 333 type); 334 pr_info(" Initiator-Target[%u-%u]:%u%s\n", 335 inits[init], targs[targ], value, 336 hmat_data_type_suffix(type)); 337 338 if (mem_hier == ACPI_HMAT_MEMORY) { 339 target = find_mem_target(targs[targ]); 340 if (target && target->processor_pxm == inits[init]) { 341 hmat_update_target_access(target, type, value, 0); 342 /* If the node has a CPU, update access 1 */ 343 if (node_state(pxm_to_node(inits[init]), N_CPU)) 344 hmat_update_target_access(target, type, value, 1); 345 } 346 } 347 } 348 } 349 350 if (mem_hier == ACPI_HMAT_MEMORY) 351 hmat_add_locality(hmat_loc); 352 353 return 0; 354 } 355 356 static __init int hmat_parse_cache(union acpi_subtable_headers *header, 357 const unsigned long end) 358 { 359 struct acpi_hmat_cache *cache = (void *)header; 360 struct memory_target *target; 361 struct target_cache *tcache; 362 u32 attrs; 363 364 if (cache->header.length < sizeof(*cache)) { 365 pr_notice("Unexpected cache header length: %u\n", 366 cache->header.length); 367 return -EINVAL; 368 } 369 370 attrs = cache->cache_attributes; 371 pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n", 372 cache->memory_PD, cache->cache_size, attrs, 373 cache->number_of_SMBIOShandles); 374 375 target = find_mem_target(cache->memory_PD); 376 if (!target) 377 return 0; 378 379 tcache = kzalloc(sizeof(*tcache), GFP_KERNEL); 380 if (!tcache) { 381 pr_notice_once("Failed to allocate HMAT cache info\n"); 382 return 0; 383 } 384 385 tcache->cache_attrs.size = cache->cache_size; 386 tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4; 387 tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16; 388 389 switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) { 390 case ACPI_HMAT_CA_DIRECT_MAPPED: 391 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP; 392 break; 393 case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING: 394 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED; 395 break; 396 case ACPI_HMAT_CA_NONE: 397 default: 398 tcache->cache_attrs.indexing = NODE_CACHE_OTHER; 399 break; 400 } 401 402 switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) { 403 case ACPI_HMAT_CP_WB: 404 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK; 405 break; 406 case ACPI_HMAT_CP_WT: 407 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH; 408 break; 409 case ACPI_HMAT_CP_NONE: 410 default: 411 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER; 412 break; 413 } 414 list_add_tail(&tcache->node, &target->caches); 415 416 return 0; 417 } 418 419 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header, 420 const unsigned long end) 421 { 422 struct acpi_hmat_proximity_domain *p = (void *)header; 423 struct memory_target *target = NULL; 424 425 if (p->header.length != sizeof(*p)) { 426 pr_notice("Unexpected address range header length: %u\n", 427 p->header.length); 428 return -EINVAL; 429 } 430 431 if (hmat_revision == 1) 432 pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n", 433 p->reserved3, p->reserved4, p->flags, p->processor_PD, 434 p->memory_PD); 435 else 436 pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n", 437 p->flags, p->processor_PD, p->memory_PD); 438 439 if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) || 440 hmat_revision > 1) { 441 target = find_mem_target(p->memory_PD); 442 if (!target) { 443 pr_debug("Memory Domain missing from SRAT\n"); 444 return -EINVAL; 445 } 446 } 447 if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) { 448 int p_node = pxm_to_node(p->processor_PD); 449 450 if (p_node == NUMA_NO_NODE) { 451 pr_debug("Invalid Processor Domain\n"); 452 return -EINVAL; 453 } 454 target->processor_pxm = p->processor_PD; 455 } 456 457 return 0; 458 } 459 460 static int __init hmat_parse_subtable(union acpi_subtable_headers *header, 461 const unsigned long end) 462 { 463 struct acpi_hmat_structure *hdr = (void *)header; 464 465 if (!hdr) 466 return -EINVAL; 467 468 switch (hdr->type) { 469 case ACPI_HMAT_TYPE_PROXIMITY: 470 return hmat_parse_proximity_domain(header, end); 471 case ACPI_HMAT_TYPE_LOCALITY: 472 return hmat_parse_locality(header, end); 473 case ACPI_HMAT_TYPE_CACHE: 474 return hmat_parse_cache(header, end); 475 default: 476 return -EINVAL; 477 } 478 } 479 480 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header, 481 const unsigned long end) 482 { 483 struct acpi_srat_mem_affinity *ma = (void *)header; 484 485 if (!ma) 486 return -EINVAL; 487 if (!(ma->flags & ACPI_SRAT_MEM_ENABLED)) 488 return 0; 489 alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length); 490 return 0; 491 } 492 493 static u32 hmat_initiator_perf(struct memory_target *target, 494 struct memory_initiator *initiator, 495 struct acpi_hmat_locality *hmat_loc) 496 { 497 unsigned int ipds, tpds, i, idx = 0, tdx = 0; 498 u32 *inits, *targs; 499 u16 *entries; 500 501 ipds = hmat_loc->number_of_initiator_Pds; 502 tpds = hmat_loc->number_of_target_Pds; 503 inits = (u32 *)(hmat_loc + 1); 504 targs = inits + ipds; 505 entries = (u16 *)(targs + tpds); 506 507 for (i = 0; i < ipds; i++) { 508 if (inits[i] == initiator->processor_pxm) { 509 idx = i; 510 break; 511 } 512 } 513 514 if (i == ipds) 515 return 0; 516 517 for (i = 0; i < tpds; i++) { 518 if (targs[i] == target->memory_pxm) { 519 tdx = i; 520 break; 521 } 522 } 523 if (i == tpds) 524 return 0; 525 526 return hmat_normalize(entries[idx * tpds + tdx], 527 hmat_loc->entry_base_unit, 528 hmat_loc->data_type); 529 } 530 531 static bool hmat_update_best(u8 type, u32 value, u32 *best) 532 { 533 bool updated = false; 534 535 if (!value) 536 return false; 537 538 switch (type) { 539 case ACPI_HMAT_ACCESS_LATENCY: 540 case ACPI_HMAT_READ_LATENCY: 541 case ACPI_HMAT_WRITE_LATENCY: 542 if (!*best || *best > value) { 543 *best = value; 544 updated = true; 545 } 546 break; 547 case ACPI_HMAT_ACCESS_BANDWIDTH: 548 case ACPI_HMAT_READ_BANDWIDTH: 549 case ACPI_HMAT_WRITE_BANDWIDTH: 550 if (!*best || *best < value) { 551 *best = value; 552 updated = true; 553 } 554 break; 555 } 556 557 return updated; 558 } 559 560 static int initiator_cmp(void *priv, const struct list_head *a, 561 const struct list_head *b) 562 { 563 struct memory_initiator *ia; 564 struct memory_initiator *ib; 565 566 ia = list_entry(a, struct memory_initiator, node); 567 ib = list_entry(b, struct memory_initiator, node); 568 569 return ia->processor_pxm - ib->processor_pxm; 570 } 571 572 static int initiators_to_nodemask(unsigned long *p_nodes) 573 { 574 struct memory_initiator *initiator; 575 576 if (list_empty(&initiators)) 577 return -ENXIO; 578 579 list_for_each_entry(initiator, &initiators, node) 580 set_bit(initiator->processor_pxm, p_nodes); 581 582 return 0; 583 } 584 585 static void hmat_register_target_initiators(struct memory_target *target) 586 { 587 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 588 struct memory_initiator *initiator; 589 unsigned int mem_nid, cpu_nid; 590 struct memory_locality *loc = NULL; 591 u32 best = 0; 592 bool access0done = false; 593 int i; 594 595 mem_nid = pxm_to_node(target->memory_pxm); 596 /* 597 * If the Address Range Structure provides a local processor pxm, link 598 * only that one. Otherwise, find the best performance attributes and 599 * register all initiators that match. 600 */ 601 if (target->processor_pxm != PXM_INVAL) { 602 cpu_nid = pxm_to_node(target->processor_pxm); 603 register_memory_node_under_compute_node(mem_nid, cpu_nid, 0); 604 access0done = true; 605 if (node_state(cpu_nid, N_CPU)) { 606 register_memory_node_under_compute_node(mem_nid, cpu_nid, 1); 607 return; 608 } 609 } 610 611 if (list_empty(&localities)) 612 return; 613 614 /* 615 * We need the initiator list sorted so we can use bitmap_clear for 616 * previously set initiators when we find a better memory accessor. 617 * We'll also use the sorting to prime the candidate nodes with known 618 * initiators. 619 */ 620 bitmap_zero(p_nodes, MAX_NUMNODES); 621 list_sort(NULL, &initiators, initiator_cmp); 622 if (initiators_to_nodemask(p_nodes) < 0) 623 return; 624 625 if (!access0done) { 626 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { 627 loc = localities_types[i]; 628 if (!loc) 629 continue; 630 631 best = 0; 632 list_for_each_entry(initiator, &initiators, node) { 633 u32 value; 634 635 if (!test_bit(initiator->processor_pxm, p_nodes)) 636 continue; 637 638 value = hmat_initiator_perf(target, initiator, 639 loc->hmat_loc); 640 if (hmat_update_best(loc->hmat_loc->data_type, value, &best)) 641 bitmap_clear(p_nodes, 0, initiator->processor_pxm); 642 if (value != best) 643 clear_bit(initiator->processor_pxm, p_nodes); 644 } 645 if (best) 646 hmat_update_target_access(target, loc->hmat_loc->data_type, 647 best, 0); 648 } 649 650 for_each_set_bit(i, p_nodes, MAX_NUMNODES) { 651 cpu_nid = pxm_to_node(i); 652 register_memory_node_under_compute_node(mem_nid, cpu_nid, 0); 653 } 654 } 655 656 /* Access 1 ignores Generic Initiators */ 657 bitmap_zero(p_nodes, MAX_NUMNODES); 658 if (initiators_to_nodemask(p_nodes) < 0) 659 return; 660 661 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { 662 loc = localities_types[i]; 663 if (!loc) 664 continue; 665 666 best = 0; 667 list_for_each_entry(initiator, &initiators, node) { 668 u32 value; 669 670 if (!initiator->has_cpu) { 671 clear_bit(initiator->processor_pxm, p_nodes); 672 continue; 673 } 674 if (!test_bit(initiator->processor_pxm, p_nodes)) 675 continue; 676 677 value = hmat_initiator_perf(target, initiator, loc->hmat_loc); 678 if (hmat_update_best(loc->hmat_loc->data_type, value, &best)) 679 bitmap_clear(p_nodes, 0, initiator->processor_pxm); 680 if (value != best) 681 clear_bit(initiator->processor_pxm, p_nodes); 682 } 683 if (best) 684 hmat_update_target_access(target, loc->hmat_loc->data_type, best, 1); 685 } 686 for_each_set_bit(i, p_nodes, MAX_NUMNODES) { 687 cpu_nid = pxm_to_node(i); 688 register_memory_node_under_compute_node(mem_nid, cpu_nid, 1); 689 } 690 } 691 692 static void hmat_register_target_cache(struct memory_target *target) 693 { 694 unsigned mem_nid = pxm_to_node(target->memory_pxm); 695 struct target_cache *tcache; 696 697 list_for_each_entry(tcache, &target->caches, node) 698 node_add_cache(mem_nid, &tcache->cache_attrs); 699 } 700 701 static void hmat_register_target_perf(struct memory_target *target, int access) 702 { 703 unsigned mem_nid = pxm_to_node(target->memory_pxm); 704 node_set_perf_attrs(mem_nid, &target->hmem_attrs[access], access); 705 } 706 707 static void hmat_register_target_devices(struct memory_target *target) 708 { 709 struct resource *res; 710 711 /* 712 * Do not bother creating devices if no driver is available to 713 * consume them. 714 */ 715 if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM)) 716 return; 717 718 for (res = target->memregions.child; res; res = res->sibling) { 719 int target_nid = pxm_to_node(target->memory_pxm); 720 721 hmem_register_device(target_nid, res); 722 } 723 } 724 725 static void hmat_register_target(struct memory_target *target) 726 { 727 int nid = pxm_to_node(target->memory_pxm); 728 729 /* 730 * Devices may belong to either an offline or online 731 * node, so unconditionally add them. 732 */ 733 hmat_register_target_devices(target); 734 735 /* 736 * Skip offline nodes. This can happen when memory 737 * marked EFI_MEMORY_SP, "specific purpose", is applied 738 * to all the memory in a proximity domain leading to 739 * the node being marked offline / unplugged, or if 740 * memory-only "hotplug" node is offline. 741 */ 742 if (nid == NUMA_NO_NODE || !node_online(nid)) 743 return; 744 745 mutex_lock(&target_lock); 746 if (!target->registered) { 747 hmat_register_target_initiators(target); 748 hmat_register_target_cache(target); 749 hmat_register_target_perf(target, 0); 750 hmat_register_target_perf(target, 1); 751 target->registered = true; 752 } 753 mutex_unlock(&target_lock); 754 } 755 756 static void hmat_register_targets(void) 757 { 758 struct memory_target *target; 759 760 list_for_each_entry(target, &targets, node) 761 hmat_register_target(target); 762 } 763 764 static int hmat_callback(struct notifier_block *self, 765 unsigned long action, void *arg) 766 { 767 struct memory_target *target; 768 struct memory_notify *mnb = arg; 769 int pxm, nid = mnb->status_change_nid; 770 771 if (nid == NUMA_NO_NODE || action != MEM_ONLINE) 772 return NOTIFY_OK; 773 774 pxm = node_to_pxm(nid); 775 target = find_mem_target(pxm); 776 if (!target) 777 return NOTIFY_OK; 778 779 hmat_register_target(target); 780 return NOTIFY_OK; 781 } 782 783 static __init void hmat_free_structures(void) 784 { 785 struct memory_target *target, *tnext; 786 struct memory_locality *loc, *lnext; 787 struct memory_initiator *initiator, *inext; 788 struct target_cache *tcache, *cnext; 789 790 list_for_each_entry_safe(target, tnext, &targets, node) { 791 struct resource *res, *res_next; 792 793 list_for_each_entry_safe(tcache, cnext, &target->caches, node) { 794 list_del(&tcache->node); 795 kfree(tcache); 796 } 797 798 list_del(&target->node); 799 res = target->memregions.child; 800 while (res) { 801 res_next = res->sibling; 802 __release_region(&target->memregions, res->start, 803 resource_size(res)); 804 res = res_next; 805 } 806 kfree(target); 807 } 808 809 list_for_each_entry_safe(initiator, inext, &initiators, node) { 810 list_del(&initiator->node); 811 kfree(initiator); 812 } 813 814 list_for_each_entry_safe(loc, lnext, &localities, node) { 815 list_del(&loc->node); 816 kfree(loc); 817 } 818 } 819 820 static __init int hmat_init(void) 821 { 822 struct acpi_table_header *tbl; 823 enum acpi_hmat_type i; 824 acpi_status status; 825 826 if (srat_disabled() || hmat_disable) 827 return 0; 828 829 status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl); 830 if (ACPI_FAILURE(status)) 831 return 0; 832 833 if (acpi_table_parse_entries(ACPI_SIG_SRAT, 834 sizeof(struct acpi_table_srat), 835 ACPI_SRAT_TYPE_MEMORY_AFFINITY, 836 srat_parse_mem_affinity, 0) < 0) 837 goto out_put; 838 acpi_put_table(tbl); 839 840 status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl); 841 if (ACPI_FAILURE(status)) 842 goto out_put; 843 844 hmat_revision = tbl->revision; 845 switch (hmat_revision) { 846 case 1: 847 case 2: 848 break; 849 default: 850 pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision); 851 goto out_put; 852 } 853 854 for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) { 855 if (acpi_table_parse_entries(ACPI_SIG_HMAT, 856 sizeof(struct acpi_table_hmat), i, 857 hmat_parse_subtable, 0) < 0) { 858 pr_notice("Ignoring: Invalid table"); 859 goto out_put; 860 } 861 } 862 hmat_register_targets(); 863 864 /* Keep the table and structures if the notifier may use them */ 865 if (!hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI)) 866 return 0; 867 out_put: 868 hmat_free_structures(); 869 acpi_put_table(tbl); 870 return 0; 871 } 872 device_initcall(hmat_init); 873