1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/module.h> 3 4 #include <asm/cpu_device_id.h> 5 #include <asm/intel-family.h> 6 #include "uncore.h" 7 8 static struct intel_uncore_type *empty_uncore[] = { NULL, }; 9 struct intel_uncore_type **uncore_msr_uncores = empty_uncore; 10 struct intel_uncore_type **uncore_pci_uncores = empty_uncore; 11 struct intel_uncore_type **uncore_mmio_uncores = empty_uncore; 12 13 static bool pcidrv_registered; 14 struct pci_driver *uncore_pci_driver; 15 /* pci bus to socket mapping */ 16 DEFINE_RAW_SPINLOCK(pci2phy_map_lock); 17 struct list_head pci2phy_map_head = LIST_HEAD_INIT(pci2phy_map_head); 18 struct pci_extra_dev *uncore_extra_pci_dev; 19 static int max_dies; 20 21 /* mask of cpus that collect uncore events */ 22 static cpumask_t uncore_cpu_mask; 23 24 /* constraint for the fixed counter */ 25 static struct event_constraint uncore_constraint_fixed = 26 EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL); 27 struct event_constraint uncore_constraint_empty = 28 EVENT_CONSTRAINT(0, 0, 0); 29 30 MODULE_LICENSE("GPL"); 31 32 int uncore_pcibus_to_physid(struct pci_bus *bus) 33 { 34 struct pci2phy_map *map; 35 int phys_id = -1; 36 37 raw_spin_lock(&pci2phy_map_lock); 38 list_for_each_entry(map, &pci2phy_map_head, list) { 39 if (map->segment == pci_domain_nr(bus)) { 40 phys_id = map->pbus_to_physid[bus->number]; 41 break; 42 } 43 } 44 raw_spin_unlock(&pci2phy_map_lock); 45 46 return phys_id; 47 } 48 49 static void uncore_free_pcibus_map(void) 50 { 51 struct pci2phy_map *map, *tmp; 52 53 list_for_each_entry_safe(map, tmp, &pci2phy_map_head, list) { 54 list_del(&map->list); 55 kfree(map); 56 } 57 } 58 59 struct pci2phy_map *__find_pci2phy_map(int segment) 60 { 61 struct pci2phy_map *map, *alloc = NULL; 62 int i; 63 64 lockdep_assert_held(&pci2phy_map_lock); 65 66 lookup: 67 list_for_each_entry(map, &pci2phy_map_head, list) { 68 if (map->segment == segment) 69 goto end; 70 } 71 72 if (!alloc) { 73 raw_spin_unlock(&pci2phy_map_lock); 74 alloc = kmalloc(sizeof(struct pci2phy_map), GFP_KERNEL); 75 raw_spin_lock(&pci2phy_map_lock); 76 77 if (!alloc) 78 return NULL; 79 80 goto lookup; 81 } 82 83 map = alloc; 84 alloc = NULL; 85 map->segment = segment; 86 for (i = 0; i < 256; i++) 87 map->pbus_to_physid[i] = -1; 88 list_add_tail(&map->list, &pci2phy_map_head); 89 90 end: 91 kfree(alloc); 92 return map; 93 } 94 95 ssize_t uncore_event_show(struct kobject *kobj, 96 struct kobj_attribute *attr, char *buf) 97 { 98 struct uncore_event_desc *event = 99 container_of(attr, struct uncore_event_desc, attr); 100 return sprintf(buf, "%s", event->config); 101 } 102 103 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu) 104 { 105 unsigned int dieid = topology_logical_die_id(cpu); 106 107 /* 108 * The unsigned check also catches the '-1' return value for non 109 * existent mappings in the topology map. 110 */ 111 return dieid < max_dies ? pmu->boxes[dieid] : NULL; 112 } 113 114 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event) 115 { 116 u64 count; 117 118 rdmsrl(event->hw.event_base, count); 119 120 return count; 121 } 122 123 void uncore_mmio_exit_box(struct intel_uncore_box *box) 124 { 125 if (box->io_addr) 126 iounmap(box->io_addr); 127 } 128 129 u64 uncore_mmio_read_counter(struct intel_uncore_box *box, 130 struct perf_event *event) 131 { 132 if (!box->io_addr) 133 return 0; 134 135 return readq(box->io_addr + event->hw.event_base); 136 } 137 138 /* 139 * generic get constraint function for shared match/mask registers. 140 */ 141 struct event_constraint * 142 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event) 143 { 144 struct intel_uncore_extra_reg *er; 145 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 146 struct hw_perf_event_extra *reg2 = &event->hw.branch_reg; 147 unsigned long flags; 148 bool ok = false; 149 150 /* 151 * reg->alloc can be set due to existing state, so for fake box we 152 * need to ignore this, otherwise we might fail to allocate proper 153 * fake state for this extra reg constraint. 154 */ 155 if (reg1->idx == EXTRA_REG_NONE || 156 (!uncore_box_is_fake(box) && reg1->alloc)) 157 return NULL; 158 159 er = &box->shared_regs[reg1->idx]; 160 raw_spin_lock_irqsave(&er->lock, flags); 161 if (!atomic_read(&er->ref) || 162 (er->config1 == reg1->config && er->config2 == reg2->config)) { 163 atomic_inc(&er->ref); 164 er->config1 = reg1->config; 165 er->config2 = reg2->config; 166 ok = true; 167 } 168 raw_spin_unlock_irqrestore(&er->lock, flags); 169 170 if (ok) { 171 if (!uncore_box_is_fake(box)) 172 reg1->alloc = 1; 173 return NULL; 174 } 175 176 return &uncore_constraint_empty; 177 } 178 179 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event) 180 { 181 struct intel_uncore_extra_reg *er; 182 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg; 183 184 /* 185 * Only put constraint if extra reg was actually allocated. Also 186 * takes care of event which do not use an extra shared reg. 187 * 188 * Also, if this is a fake box we shouldn't touch any event state 189 * (reg->alloc) and we don't care about leaving inconsistent box 190 * state either since it will be thrown out. 191 */ 192 if (uncore_box_is_fake(box) || !reg1->alloc) 193 return; 194 195 er = &box->shared_regs[reg1->idx]; 196 atomic_dec(&er->ref); 197 reg1->alloc = 0; 198 } 199 200 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx) 201 { 202 struct intel_uncore_extra_reg *er; 203 unsigned long flags; 204 u64 config; 205 206 er = &box->shared_regs[idx]; 207 208 raw_spin_lock_irqsave(&er->lock, flags); 209 config = er->config; 210 raw_spin_unlock_irqrestore(&er->lock, flags); 211 212 return config; 213 } 214 215 static void uncore_assign_hw_event(struct intel_uncore_box *box, 216 struct perf_event *event, int idx) 217 { 218 struct hw_perf_event *hwc = &event->hw; 219 220 hwc->idx = idx; 221 hwc->last_tag = ++box->tags[idx]; 222 223 if (uncore_pmc_fixed(hwc->idx)) { 224 hwc->event_base = uncore_fixed_ctr(box); 225 hwc->config_base = uncore_fixed_ctl(box); 226 return; 227 } 228 229 hwc->config_base = uncore_event_ctl(box, hwc->idx); 230 hwc->event_base = uncore_perf_ctr(box, hwc->idx); 231 } 232 233 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event) 234 { 235 u64 prev_count, new_count, delta; 236 int shift; 237 238 if (uncore_pmc_freerunning(event->hw.idx)) 239 shift = 64 - uncore_freerunning_bits(box, event); 240 else if (uncore_pmc_fixed(event->hw.idx)) 241 shift = 64 - uncore_fixed_ctr_bits(box); 242 else 243 shift = 64 - uncore_perf_ctr_bits(box); 244 245 /* the hrtimer might modify the previous event value */ 246 again: 247 prev_count = local64_read(&event->hw.prev_count); 248 new_count = uncore_read_counter(box, event); 249 if (local64_xchg(&event->hw.prev_count, new_count) != prev_count) 250 goto again; 251 252 delta = (new_count << shift) - (prev_count << shift); 253 delta >>= shift; 254 255 local64_add(delta, &event->count); 256 } 257 258 /* 259 * The overflow interrupt is unavailable for SandyBridge-EP, is broken 260 * for SandyBridge. So we use hrtimer to periodically poll the counter 261 * to avoid overflow. 262 */ 263 static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer) 264 { 265 struct intel_uncore_box *box; 266 struct perf_event *event; 267 unsigned long flags; 268 int bit; 269 270 box = container_of(hrtimer, struct intel_uncore_box, hrtimer); 271 if (!box->n_active || box->cpu != smp_processor_id()) 272 return HRTIMER_NORESTART; 273 /* 274 * disable local interrupt to prevent uncore_pmu_event_start/stop 275 * to interrupt the update process 276 */ 277 local_irq_save(flags); 278 279 /* 280 * handle boxes with an active event list as opposed to active 281 * counters 282 */ 283 list_for_each_entry(event, &box->active_list, active_entry) { 284 uncore_perf_event_update(box, event); 285 } 286 287 for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX) 288 uncore_perf_event_update(box, box->events[bit]); 289 290 local_irq_restore(flags); 291 292 hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration)); 293 return HRTIMER_RESTART; 294 } 295 296 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box) 297 { 298 hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration), 299 HRTIMER_MODE_REL_PINNED); 300 } 301 302 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box) 303 { 304 hrtimer_cancel(&box->hrtimer); 305 } 306 307 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box) 308 { 309 hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 310 box->hrtimer.function = uncore_pmu_hrtimer; 311 } 312 313 static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, 314 int node) 315 { 316 int i, size, numshared = type->num_shared_regs ; 317 struct intel_uncore_box *box; 318 319 size = sizeof(*box) + numshared * sizeof(struct intel_uncore_extra_reg); 320 321 box = kzalloc_node(size, GFP_KERNEL, node); 322 if (!box) 323 return NULL; 324 325 for (i = 0; i < numshared; i++) 326 raw_spin_lock_init(&box->shared_regs[i].lock); 327 328 uncore_pmu_init_hrtimer(box); 329 box->cpu = -1; 330 box->pci_phys_id = -1; 331 box->dieid = -1; 332 333 /* set default hrtimer timeout */ 334 box->hrtimer_duration = UNCORE_PMU_HRTIMER_INTERVAL; 335 336 INIT_LIST_HEAD(&box->active_list); 337 338 return box; 339 } 340 341 /* 342 * Using uncore_pmu_event_init pmu event_init callback 343 * as a detection point for uncore events. 344 */ 345 static int uncore_pmu_event_init(struct perf_event *event); 346 347 static bool is_box_event(struct intel_uncore_box *box, struct perf_event *event) 348 { 349 return &box->pmu->pmu == event->pmu; 350 } 351 352 static int 353 uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader, 354 bool dogrp) 355 { 356 struct perf_event *event; 357 int n, max_count; 358 359 max_count = box->pmu->type->num_counters; 360 if (box->pmu->type->fixed_ctl) 361 max_count++; 362 363 if (box->n_events >= max_count) 364 return -EINVAL; 365 366 n = box->n_events; 367 368 if (is_box_event(box, leader)) { 369 box->event_list[n] = leader; 370 n++; 371 } 372 373 if (!dogrp) 374 return n; 375 376 for_each_sibling_event(event, leader) { 377 if (!is_box_event(box, event) || 378 event->state <= PERF_EVENT_STATE_OFF) 379 continue; 380 381 if (n >= max_count) 382 return -EINVAL; 383 384 box->event_list[n] = event; 385 n++; 386 } 387 return n; 388 } 389 390 static struct event_constraint * 391 uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event) 392 { 393 struct intel_uncore_type *type = box->pmu->type; 394 struct event_constraint *c; 395 396 if (type->ops->get_constraint) { 397 c = type->ops->get_constraint(box, event); 398 if (c) 399 return c; 400 } 401 402 if (event->attr.config == UNCORE_FIXED_EVENT) 403 return &uncore_constraint_fixed; 404 405 if (type->constraints) { 406 for_each_event_constraint(c, type->constraints) { 407 if ((event->hw.config & c->cmask) == c->code) 408 return c; 409 } 410 } 411 412 return &type->unconstrainted; 413 } 414 415 static void uncore_put_event_constraint(struct intel_uncore_box *box, 416 struct perf_event *event) 417 { 418 if (box->pmu->type->ops->put_constraint) 419 box->pmu->type->ops->put_constraint(box, event); 420 } 421 422 static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n) 423 { 424 unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)]; 425 struct event_constraint *c; 426 int i, wmin, wmax, ret = 0; 427 struct hw_perf_event *hwc; 428 429 bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX); 430 431 for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) { 432 c = uncore_get_event_constraint(box, box->event_list[i]); 433 box->event_constraint[i] = c; 434 wmin = min(wmin, c->weight); 435 wmax = max(wmax, c->weight); 436 } 437 438 /* fastpath, try to reuse previous register */ 439 for (i = 0; i < n; i++) { 440 hwc = &box->event_list[i]->hw; 441 c = box->event_constraint[i]; 442 443 /* never assigned */ 444 if (hwc->idx == -1) 445 break; 446 447 /* constraint still honored */ 448 if (!test_bit(hwc->idx, c->idxmsk)) 449 break; 450 451 /* not already used */ 452 if (test_bit(hwc->idx, used_mask)) 453 break; 454 455 __set_bit(hwc->idx, used_mask); 456 if (assign) 457 assign[i] = hwc->idx; 458 } 459 /* slow path */ 460 if (i != n) 461 ret = perf_assign_events(box->event_constraint, n, 462 wmin, wmax, n, assign); 463 464 if (!assign || ret) { 465 for (i = 0; i < n; i++) 466 uncore_put_event_constraint(box, box->event_list[i]); 467 } 468 return ret ? -EINVAL : 0; 469 } 470 471 void uncore_pmu_event_start(struct perf_event *event, int flags) 472 { 473 struct intel_uncore_box *box = uncore_event_to_box(event); 474 int idx = event->hw.idx; 475 476 if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX)) 477 return; 478 479 /* 480 * Free running counter is read-only and always active. 481 * Use the current counter value as start point. 482 * There is no overflow interrupt for free running counter. 483 * Use hrtimer to periodically poll the counter to avoid overflow. 484 */ 485 if (uncore_pmc_freerunning(event->hw.idx)) { 486 list_add_tail(&event->active_entry, &box->active_list); 487 local64_set(&event->hw.prev_count, 488 uncore_read_counter(box, event)); 489 if (box->n_active++ == 0) 490 uncore_pmu_start_hrtimer(box); 491 return; 492 } 493 494 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 495 return; 496 497 event->hw.state = 0; 498 box->events[idx] = event; 499 box->n_active++; 500 __set_bit(idx, box->active_mask); 501 502 local64_set(&event->hw.prev_count, uncore_read_counter(box, event)); 503 uncore_enable_event(box, event); 504 505 if (box->n_active == 1) 506 uncore_pmu_start_hrtimer(box); 507 } 508 509 void uncore_pmu_event_stop(struct perf_event *event, int flags) 510 { 511 struct intel_uncore_box *box = uncore_event_to_box(event); 512 struct hw_perf_event *hwc = &event->hw; 513 514 /* Cannot disable free running counter which is read-only */ 515 if (uncore_pmc_freerunning(hwc->idx)) { 516 list_del(&event->active_entry); 517 if (--box->n_active == 0) 518 uncore_pmu_cancel_hrtimer(box); 519 uncore_perf_event_update(box, event); 520 return; 521 } 522 523 if (__test_and_clear_bit(hwc->idx, box->active_mask)) { 524 uncore_disable_event(box, event); 525 box->n_active--; 526 box->events[hwc->idx] = NULL; 527 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 528 hwc->state |= PERF_HES_STOPPED; 529 530 if (box->n_active == 0) 531 uncore_pmu_cancel_hrtimer(box); 532 } 533 534 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 535 /* 536 * Drain the remaining delta count out of a event 537 * that we are disabling: 538 */ 539 uncore_perf_event_update(box, event); 540 hwc->state |= PERF_HES_UPTODATE; 541 } 542 } 543 544 int uncore_pmu_event_add(struct perf_event *event, int flags) 545 { 546 struct intel_uncore_box *box = uncore_event_to_box(event); 547 struct hw_perf_event *hwc = &event->hw; 548 int assign[UNCORE_PMC_IDX_MAX]; 549 int i, n, ret; 550 551 if (!box) 552 return -ENODEV; 553 554 /* 555 * The free funning counter is assigned in event_init(). 556 * The free running counter event and free running counter 557 * are 1:1 mapped. It doesn't need to be tracked in event_list. 558 */ 559 if (uncore_pmc_freerunning(hwc->idx)) { 560 if (flags & PERF_EF_START) 561 uncore_pmu_event_start(event, 0); 562 return 0; 563 } 564 565 ret = n = uncore_collect_events(box, event, false); 566 if (ret < 0) 567 return ret; 568 569 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 570 if (!(flags & PERF_EF_START)) 571 hwc->state |= PERF_HES_ARCH; 572 573 ret = uncore_assign_events(box, assign, n); 574 if (ret) 575 return ret; 576 577 /* save events moving to new counters */ 578 for (i = 0; i < box->n_events; i++) { 579 event = box->event_list[i]; 580 hwc = &event->hw; 581 582 if (hwc->idx == assign[i] && 583 hwc->last_tag == box->tags[assign[i]]) 584 continue; 585 /* 586 * Ensure we don't accidentally enable a stopped 587 * counter simply because we rescheduled. 588 */ 589 if (hwc->state & PERF_HES_STOPPED) 590 hwc->state |= PERF_HES_ARCH; 591 592 uncore_pmu_event_stop(event, PERF_EF_UPDATE); 593 } 594 595 /* reprogram moved events into new counters */ 596 for (i = 0; i < n; i++) { 597 event = box->event_list[i]; 598 hwc = &event->hw; 599 600 if (hwc->idx != assign[i] || 601 hwc->last_tag != box->tags[assign[i]]) 602 uncore_assign_hw_event(box, event, assign[i]); 603 else if (i < box->n_events) 604 continue; 605 606 if (hwc->state & PERF_HES_ARCH) 607 continue; 608 609 uncore_pmu_event_start(event, 0); 610 } 611 box->n_events = n; 612 613 return 0; 614 } 615 616 void uncore_pmu_event_del(struct perf_event *event, int flags) 617 { 618 struct intel_uncore_box *box = uncore_event_to_box(event); 619 int i; 620 621 uncore_pmu_event_stop(event, PERF_EF_UPDATE); 622 623 /* 624 * The event for free running counter is not tracked by event_list. 625 * It doesn't need to force event->hw.idx = -1 to reassign the counter. 626 * Because the event and the free running counter are 1:1 mapped. 627 */ 628 if (uncore_pmc_freerunning(event->hw.idx)) 629 return; 630 631 for (i = 0; i < box->n_events; i++) { 632 if (event == box->event_list[i]) { 633 uncore_put_event_constraint(box, event); 634 635 for (++i; i < box->n_events; i++) 636 box->event_list[i - 1] = box->event_list[i]; 637 638 --box->n_events; 639 break; 640 } 641 } 642 643 event->hw.idx = -1; 644 event->hw.last_tag = ~0ULL; 645 } 646 647 void uncore_pmu_event_read(struct perf_event *event) 648 { 649 struct intel_uncore_box *box = uncore_event_to_box(event); 650 uncore_perf_event_update(box, event); 651 } 652 653 /* 654 * validation ensures the group can be loaded onto the 655 * PMU if it was the only group available. 656 */ 657 static int uncore_validate_group(struct intel_uncore_pmu *pmu, 658 struct perf_event *event) 659 { 660 struct perf_event *leader = event->group_leader; 661 struct intel_uncore_box *fake_box; 662 int ret = -EINVAL, n; 663 664 /* The free running counter is always active. */ 665 if (uncore_pmc_freerunning(event->hw.idx)) 666 return 0; 667 668 fake_box = uncore_alloc_box(pmu->type, NUMA_NO_NODE); 669 if (!fake_box) 670 return -ENOMEM; 671 672 fake_box->pmu = pmu; 673 /* 674 * the event is not yet connected with its 675 * siblings therefore we must first collect 676 * existing siblings, then add the new event 677 * before we can simulate the scheduling 678 */ 679 n = uncore_collect_events(fake_box, leader, true); 680 if (n < 0) 681 goto out; 682 683 fake_box->n_events = n; 684 n = uncore_collect_events(fake_box, event, false); 685 if (n < 0) 686 goto out; 687 688 fake_box->n_events = n; 689 690 ret = uncore_assign_events(fake_box, NULL, n); 691 out: 692 kfree(fake_box); 693 return ret; 694 } 695 696 static int uncore_pmu_event_init(struct perf_event *event) 697 { 698 struct intel_uncore_pmu *pmu; 699 struct intel_uncore_box *box; 700 struct hw_perf_event *hwc = &event->hw; 701 int ret; 702 703 if (event->attr.type != event->pmu->type) 704 return -ENOENT; 705 706 pmu = uncore_event_to_pmu(event); 707 /* no device found for this pmu */ 708 if (pmu->func_id < 0) 709 return -ENOENT; 710 711 /* Sampling not supported yet */ 712 if (hwc->sample_period) 713 return -EINVAL; 714 715 /* 716 * Place all uncore events for a particular physical package 717 * onto a single cpu 718 */ 719 if (event->cpu < 0) 720 return -EINVAL; 721 box = uncore_pmu_to_box(pmu, event->cpu); 722 if (!box || box->cpu < 0) 723 return -EINVAL; 724 event->cpu = box->cpu; 725 event->pmu_private = box; 726 727 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; 728 729 event->hw.idx = -1; 730 event->hw.last_tag = ~0ULL; 731 event->hw.extra_reg.idx = EXTRA_REG_NONE; 732 event->hw.branch_reg.idx = EXTRA_REG_NONE; 733 734 if (event->attr.config == UNCORE_FIXED_EVENT) { 735 /* no fixed counter */ 736 if (!pmu->type->fixed_ctl) 737 return -EINVAL; 738 /* 739 * if there is only one fixed counter, only the first pmu 740 * can access the fixed counter 741 */ 742 if (pmu->type->single_fixed && pmu->pmu_idx > 0) 743 return -EINVAL; 744 745 /* fixed counters have event field hardcoded to zero */ 746 hwc->config = 0ULL; 747 } else if (is_freerunning_event(event)) { 748 hwc->config = event->attr.config; 749 if (!check_valid_freerunning_event(box, event)) 750 return -EINVAL; 751 event->hw.idx = UNCORE_PMC_IDX_FREERUNNING; 752 /* 753 * The free running counter event and free running counter 754 * are always 1:1 mapped. 755 * The free running counter is always active. 756 * Assign the free running counter here. 757 */ 758 event->hw.event_base = uncore_freerunning_counter(box, event); 759 } else { 760 hwc->config = event->attr.config & 761 (pmu->type->event_mask | ((u64)pmu->type->event_mask_ext << 32)); 762 if (pmu->type->ops->hw_config) { 763 ret = pmu->type->ops->hw_config(box, event); 764 if (ret) 765 return ret; 766 } 767 } 768 769 if (event->group_leader != event) 770 ret = uncore_validate_group(pmu, event); 771 else 772 ret = 0; 773 774 return ret; 775 } 776 777 static void uncore_pmu_enable(struct pmu *pmu) 778 { 779 struct intel_uncore_pmu *uncore_pmu; 780 struct intel_uncore_box *box; 781 782 uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu); 783 if (!uncore_pmu) 784 return; 785 786 box = uncore_pmu_to_box(uncore_pmu, smp_processor_id()); 787 if (!box) 788 return; 789 790 if (uncore_pmu->type->ops->enable_box) 791 uncore_pmu->type->ops->enable_box(box); 792 } 793 794 static void uncore_pmu_disable(struct pmu *pmu) 795 { 796 struct intel_uncore_pmu *uncore_pmu; 797 struct intel_uncore_box *box; 798 799 uncore_pmu = container_of(pmu, struct intel_uncore_pmu, pmu); 800 if (!uncore_pmu) 801 return; 802 803 box = uncore_pmu_to_box(uncore_pmu, smp_processor_id()); 804 if (!box) 805 return; 806 807 if (uncore_pmu->type->ops->disable_box) 808 uncore_pmu->type->ops->disable_box(box); 809 } 810 811 static ssize_t uncore_get_attr_cpumask(struct device *dev, 812 struct device_attribute *attr, char *buf) 813 { 814 return cpumap_print_to_pagebuf(true, buf, &uncore_cpu_mask); 815 } 816 817 static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL); 818 819 static struct attribute *uncore_pmu_attrs[] = { 820 &dev_attr_cpumask.attr, 821 NULL, 822 }; 823 824 static const struct attribute_group uncore_pmu_attr_group = { 825 .attrs = uncore_pmu_attrs, 826 }; 827 828 static int uncore_pmu_register(struct intel_uncore_pmu *pmu) 829 { 830 int ret; 831 832 if (!pmu->type->pmu) { 833 pmu->pmu = (struct pmu) { 834 .attr_groups = pmu->type->attr_groups, 835 .task_ctx_nr = perf_invalid_context, 836 .pmu_enable = uncore_pmu_enable, 837 .pmu_disable = uncore_pmu_disable, 838 .event_init = uncore_pmu_event_init, 839 .add = uncore_pmu_event_add, 840 .del = uncore_pmu_event_del, 841 .start = uncore_pmu_event_start, 842 .stop = uncore_pmu_event_stop, 843 .read = uncore_pmu_event_read, 844 .module = THIS_MODULE, 845 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 846 }; 847 } else { 848 pmu->pmu = *pmu->type->pmu; 849 pmu->pmu.attr_groups = pmu->type->attr_groups; 850 } 851 852 if (pmu->type->num_boxes == 1) { 853 if (strlen(pmu->type->name) > 0) 854 sprintf(pmu->name, "uncore_%s", pmu->type->name); 855 else 856 sprintf(pmu->name, "uncore"); 857 } else { 858 sprintf(pmu->name, "uncore_%s_%d", pmu->type->name, 859 pmu->pmu_idx); 860 } 861 862 ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); 863 if (!ret) 864 pmu->registered = true; 865 return ret; 866 } 867 868 static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) 869 { 870 if (!pmu->registered) 871 return; 872 perf_pmu_unregister(&pmu->pmu); 873 pmu->registered = false; 874 } 875 876 static void uncore_free_boxes(struct intel_uncore_pmu *pmu) 877 { 878 int die; 879 880 for (die = 0; die < max_dies; die++) 881 kfree(pmu->boxes[die]); 882 kfree(pmu->boxes); 883 } 884 885 static void uncore_type_exit(struct intel_uncore_type *type) 886 { 887 struct intel_uncore_pmu *pmu = type->pmus; 888 int i; 889 890 if (pmu) { 891 for (i = 0; i < type->num_boxes; i++, pmu++) { 892 uncore_pmu_unregister(pmu); 893 uncore_free_boxes(pmu); 894 } 895 kfree(type->pmus); 896 type->pmus = NULL; 897 } 898 kfree(type->events_group); 899 type->events_group = NULL; 900 } 901 902 static void uncore_types_exit(struct intel_uncore_type **types) 903 { 904 for (; *types; types++) 905 uncore_type_exit(*types); 906 } 907 908 static int __init uncore_type_init(struct intel_uncore_type *type, bool setid) 909 { 910 struct intel_uncore_pmu *pmus; 911 size_t size; 912 int i, j; 913 914 pmus = kcalloc(type->num_boxes, sizeof(*pmus), GFP_KERNEL); 915 if (!pmus) 916 return -ENOMEM; 917 918 size = max_dies * sizeof(struct intel_uncore_box *); 919 920 for (i = 0; i < type->num_boxes; i++) { 921 pmus[i].func_id = setid ? i : -1; 922 pmus[i].pmu_idx = i; 923 pmus[i].type = type; 924 pmus[i].boxes = kzalloc(size, GFP_KERNEL); 925 if (!pmus[i].boxes) 926 goto err; 927 } 928 929 type->pmus = pmus; 930 type->unconstrainted = (struct event_constraint) 931 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1, 932 0, type->num_counters, 0, 0); 933 934 if (type->event_descs) { 935 struct { 936 struct attribute_group group; 937 struct attribute *attrs[]; 938 } *attr_group; 939 for (i = 0; type->event_descs[i].attr.attr.name; i++); 940 941 attr_group = kzalloc(struct_size(attr_group, attrs, i + 1), 942 GFP_KERNEL); 943 if (!attr_group) 944 goto err; 945 946 attr_group->group.name = "events"; 947 attr_group->group.attrs = attr_group->attrs; 948 949 for (j = 0; j < i; j++) 950 attr_group->attrs[j] = &type->event_descs[j].attr.attr; 951 952 type->events_group = &attr_group->group; 953 } 954 955 type->pmu_group = &uncore_pmu_attr_group; 956 957 return 0; 958 959 err: 960 for (i = 0; i < type->num_boxes; i++) 961 kfree(pmus[i].boxes); 962 kfree(pmus); 963 964 return -ENOMEM; 965 } 966 967 static int __init 968 uncore_types_init(struct intel_uncore_type **types, bool setid) 969 { 970 int ret; 971 972 for (; *types; types++) { 973 ret = uncore_type_init(*types, setid); 974 if (ret) 975 return ret; 976 } 977 return 0; 978 } 979 980 /* 981 * add a pci uncore device 982 */ 983 static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 984 { 985 struct intel_uncore_type *type; 986 struct intel_uncore_pmu *pmu = NULL; 987 struct intel_uncore_box *box; 988 int phys_id, die, ret; 989 990 phys_id = uncore_pcibus_to_physid(pdev->bus); 991 if (phys_id < 0) 992 return -ENODEV; 993 994 die = (topology_max_die_per_package() > 1) ? phys_id : 995 topology_phys_to_logical_pkg(phys_id); 996 if (die < 0) 997 return -EINVAL; 998 999 if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) { 1000 int idx = UNCORE_PCI_DEV_IDX(id->driver_data); 1001 1002 uncore_extra_pci_dev[die].dev[idx] = pdev; 1003 pci_set_drvdata(pdev, NULL); 1004 return 0; 1005 } 1006 1007 type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)]; 1008 1009 /* 1010 * Some platforms, e.g. Knights Landing, use a common PCI device ID 1011 * for multiple instances of an uncore PMU device type. We should check 1012 * PCI slot and func to indicate the uncore box. 1013 */ 1014 if (id->driver_data & ~0xffff) { 1015 struct pci_driver *pci_drv = pdev->driver; 1016 const struct pci_device_id *ids = pci_drv->id_table; 1017 unsigned int devfn; 1018 1019 while (ids && ids->vendor) { 1020 if ((ids->vendor == pdev->vendor) && 1021 (ids->device == pdev->device)) { 1022 devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data), 1023 UNCORE_PCI_DEV_FUNC(ids->driver_data)); 1024 if (devfn == pdev->devfn) { 1025 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)]; 1026 break; 1027 } 1028 } 1029 ids++; 1030 } 1031 if (pmu == NULL) 1032 return -ENODEV; 1033 } else { 1034 /* 1035 * for performance monitoring unit with multiple boxes, 1036 * each box has a different function id. 1037 */ 1038 pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)]; 1039 } 1040 1041 if (WARN_ON_ONCE(pmu->boxes[die] != NULL)) 1042 return -EINVAL; 1043 1044 box = uncore_alloc_box(type, NUMA_NO_NODE); 1045 if (!box) 1046 return -ENOMEM; 1047 1048 if (pmu->func_id < 0) 1049 pmu->func_id = pdev->devfn; 1050 else 1051 WARN_ON_ONCE(pmu->func_id != pdev->devfn); 1052 1053 atomic_inc(&box->refcnt); 1054 box->pci_phys_id = phys_id; 1055 box->dieid = die; 1056 box->pci_dev = pdev; 1057 box->pmu = pmu; 1058 uncore_box_init(box); 1059 pci_set_drvdata(pdev, box); 1060 1061 pmu->boxes[die] = box; 1062 if (atomic_inc_return(&pmu->activeboxes) > 1) 1063 return 0; 1064 1065 /* First active box registers the pmu */ 1066 ret = uncore_pmu_register(pmu); 1067 if (ret) { 1068 pci_set_drvdata(pdev, NULL); 1069 pmu->boxes[die] = NULL; 1070 uncore_box_exit(box); 1071 kfree(box); 1072 } 1073 return ret; 1074 } 1075 1076 static void uncore_pci_remove(struct pci_dev *pdev) 1077 { 1078 struct intel_uncore_box *box; 1079 struct intel_uncore_pmu *pmu; 1080 int i, phys_id, die; 1081 1082 phys_id = uncore_pcibus_to_physid(pdev->bus); 1083 1084 box = pci_get_drvdata(pdev); 1085 if (!box) { 1086 die = (topology_max_die_per_package() > 1) ? phys_id : 1087 topology_phys_to_logical_pkg(phys_id); 1088 for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) { 1089 if (uncore_extra_pci_dev[die].dev[i] == pdev) { 1090 uncore_extra_pci_dev[die].dev[i] = NULL; 1091 break; 1092 } 1093 } 1094 WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); 1095 return; 1096 } 1097 1098 pmu = box->pmu; 1099 if (WARN_ON_ONCE(phys_id != box->pci_phys_id)) 1100 return; 1101 1102 pci_set_drvdata(pdev, NULL); 1103 pmu->boxes[box->dieid] = NULL; 1104 if (atomic_dec_return(&pmu->activeboxes) == 0) 1105 uncore_pmu_unregister(pmu); 1106 uncore_box_exit(box); 1107 kfree(box); 1108 } 1109 1110 static int __init uncore_pci_init(void) 1111 { 1112 size_t size; 1113 int ret; 1114 1115 size = max_dies * sizeof(struct pci_extra_dev); 1116 uncore_extra_pci_dev = kzalloc(size, GFP_KERNEL); 1117 if (!uncore_extra_pci_dev) { 1118 ret = -ENOMEM; 1119 goto err; 1120 } 1121 1122 ret = uncore_types_init(uncore_pci_uncores, false); 1123 if (ret) 1124 goto errtype; 1125 1126 uncore_pci_driver->probe = uncore_pci_probe; 1127 uncore_pci_driver->remove = uncore_pci_remove; 1128 1129 ret = pci_register_driver(uncore_pci_driver); 1130 if (ret) 1131 goto errtype; 1132 1133 pcidrv_registered = true; 1134 return 0; 1135 1136 errtype: 1137 uncore_types_exit(uncore_pci_uncores); 1138 kfree(uncore_extra_pci_dev); 1139 uncore_extra_pci_dev = NULL; 1140 uncore_free_pcibus_map(); 1141 err: 1142 uncore_pci_uncores = empty_uncore; 1143 return ret; 1144 } 1145 1146 static void uncore_pci_exit(void) 1147 { 1148 if (pcidrv_registered) { 1149 pcidrv_registered = false; 1150 pci_unregister_driver(uncore_pci_driver); 1151 uncore_types_exit(uncore_pci_uncores); 1152 kfree(uncore_extra_pci_dev); 1153 uncore_free_pcibus_map(); 1154 } 1155 } 1156 1157 static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, 1158 int new_cpu) 1159 { 1160 struct intel_uncore_pmu *pmu = type->pmus; 1161 struct intel_uncore_box *box; 1162 int i, die; 1163 1164 die = topology_logical_die_id(old_cpu < 0 ? new_cpu : old_cpu); 1165 for (i = 0; i < type->num_boxes; i++, pmu++) { 1166 box = pmu->boxes[die]; 1167 if (!box) 1168 continue; 1169 1170 if (old_cpu < 0) { 1171 WARN_ON_ONCE(box->cpu != -1); 1172 box->cpu = new_cpu; 1173 continue; 1174 } 1175 1176 WARN_ON_ONCE(box->cpu != old_cpu); 1177 box->cpu = -1; 1178 if (new_cpu < 0) 1179 continue; 1180 1181 uncore_pmu_cancel_hrtimer(box); 1182 perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); 1183 box->cpu = new_cpu; 1184 } 1185 } 1186 1187 static void uncore_change_context(struct intel_uncore_type **uncores, 1188 int old_cpu, int new_cpu) 1189 { 1190 for (; *uncores; uncores++) 1191 uncore_change_type_ctx(*uncores, old_cpu, new_cpu); 1192 } 1193 1194 static void uncore_box_unref(struct intel_uncore_type **types, int id) 1195 { 1196 struct intel_uncore_type *type; 1197 struct intel_uncore_pmu *pmu; 1198 struct intel_uncore_box *box; 1199 int i; 1200 1201 for (; *types; types++) { 1202 type = *types; 1203 pmu = type->pmus; 1204 for (i = 0; i < type->num_boxes; i++, pmu++) { 1205 box = pmu->boxes[id]; 1206 if (box && atomic_dec_return(&box->refcnt) == 0) 1207 uncore_box_exit(box); 1208 } 1209 } 1210 } 1211 1212 static int uncore_event_cpu_offline(unsigned int cpu) 1213 { 1214 int die, target; 1215 1216 /* Check if exiting cpu is used for collecting uncore events */ 1217 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) 1218 goto unref; 1219 /* Find a new cpu to collect uncore events */ 1220 target = cpumask_any_but(topology_die_cpumask(cpu), cpu); 1221 1222 /* Migrate uncore events to the new target */ 1223 if (target < nr_cpu_ids) 1224 cpumask_set_cpu(target, &uncore_cpu_mask); 1225 else 1226 target = -1; 1227 1228 uncore_change_context(uncore_msr_uncores, cpu, target); 1229 uncore_change_context(uncore_mmio_uncores, cpu, target); 1230 uncore_change_context(uncore_pci_uncores, cpu, target); 1231 1232 unref: 1233 /* Clear the references */ 1234 die = topology_logical_die_id(cpu); 1235 uncore_box_unref(uncore_msr_uncores, die); 1236 uncore_box_unref(uncore_mmio_uncores, die); 1237 return 0; 1238 } 1239 1240 static int allocate_boxes(struct intel_uncore_type **types, 1241 unsigned int die, unsigned int cpu) 1242 { 1243 struct intel_uncore_box *box, *tmp; 1244 struct intel_uncore_type *type; 1245 struct intel_uncore_pmu *pmu; 1246 LIST_HEAD(allocated); 1247 int i; 1248 1249 /* Try to allocate all required boxes */ 1250 for (; *types; types++) { 1251 type = *types; 1252 pmu = type->pmus; 1253 for (i = 0; i < type->num_boxes; i++, pmu++) { 1254 if (pmu->boxes[die]) 1255 continue; 1256 box = uncore_alloc_box(type, cpu_to_node(cpu)); 1257 if (!box) 1258 goto cleanup; 1259 box->pmu = pmu; 1260 box->dieid = die; 1261 list_add(&box->active_list, &allocated); 1262 } 1263 } 1264 /* Install them in the pmus */ 1265 list_for_each_entry_safe(box, tmp, &allocated, active_list) { 1266 list_del_init(&box->active_list); 1267 box->pmu->boxes[die] = box; 1268 } 1269 return 0; 1270 1271 cleanup: 1272 list_for_each_entry_safe(box, tmp, &allocated, active_list) { 1273 list_del_init(&box->active_list); 1274 kfree(box); 1275 } 1276 return -ENOMEM; 1277 } 1278 1279 static int uncore_box_ref(struct intel_uncore_type **types, 1280 int id, unsigned int cpu) 1281 { 1282 struct intel_uncore_type *type; 1283 struct intel_uncore_pmu *pmu; 1284 struct intel_uncore_box *box; 1285 int i, ret; 1286 1287 ret = allocate_boxes(types, id, cpu); 1288 if (ret) 1289 return ret; 1290 1291 for (; *types; types++) { 1292 type = *types; 1293 pmu = type->pmus; 1294 for (i = 0; i < type->num_boxes; i++, pmu++) { 1295 box = pmu->boxes[id]; 1296 if (box && atomic_inc_return(&box->refcnt) == 1) 1297 uncore_box_init(box); 1298 } 1299 } 1300 return 0; 1301 } 1302 1303 static int uncore_event_cpu_online(unsigned int cpu) 1304 { 1305 int die, target, msr_ret, mmio_ret; 1306 1307 die = topology_logical_die_id(cpu); 1308 msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); 1309 mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); 1310 if (msr_ret && mmio_ret) 1311 return -ENOMEM; 1312 1313 /* 1314 * Check if there is an online cpu in the package 1315 * which collects uncore events already. 1316 */ 1317 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); 1318 if (target < nr_cpu_ids) 1319 return 0; 1320 1321 cpumask_set_cpu(cpu, &uncore_cpu_mask); 1322 1323 if (!msr_ret) 1324 uncore_change_context(uncore_msr_uncores, -1, cpu); 1325 if (!mmio_ret) 1326 uncore_change_context(uncore_mmio_uncores, -1, cpu); 1327 uncore_change_context(uncore_pci_uncores, -1, cpu); 1328 return 0; 1329 } 1330 1331 static int __init type_pmu_register(struct intel_uncore_type *type) 1332 { 1333 int i, ret; 1334 1335 for (i = 0; i < type->num_boxes; i++) { 1336 ret = uncore_pmu_register(&type->pmus[i]); 1337 if (ret) 1338 return ret; 1339 } 1340 return 0; 1341 } 1342 1343 static int __init uncore_msr_pmus_register(void) 1344 { 1345 struct intel_uncore_type **types = uncore_msr_uncores; 1346 int ret; 1347 1348 for (; *types; types++) { 1349 ret = type_pmu_register(*types); 1350 if (ret) 1351 return ret; 1352 } 1353 return 0; 1354 } 1355 1356 static int __init uncore_cpu_init(void) 1357 { 1358 int ret; 1359 1360 ret = uncore_types_init(uncore_msr_uncores, true); 1361 if (ret) 1362 goto err; 1363 1364 ret = uncore_msr_pmus_register(); 1365 if (ret) 1366 goto err; 1367 return 0; 1368 err: 1369 uncore_types_exit(uncore_msr_uncores); 1370 uncore_msr_uncores = empty_uncore; 1371 return ret; 1372 } 1373 1374 static int __init uncore_mmio_init(void) 1375 { 1376 struct intel_uncore_type **types = uncore_mmio_uncores; 1377 int ret; 1378 1379 ret = uncore_types_init(types, true); 1380 if (ret) 1381 goto err; 1382 1383 for (; *types; types++) { 1384 ret = type_pmu_register(*types); 1385 if (ret) 1386 goto err; 1387 } 1388 return 0; 1389 err: 1390 uncore_types_exit(uncore_mmio_uncores); 1391 uncore_mmio_uncores = empty_uncore; 1392 return ret; 1393 } 1394 1395 1396 #define X86_UNCORE_MODEL_MATCH(model, init) \ 1397 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&init } 1398 1399 struct intel_uncore_init_fun { 1400 void (*cpu_init)(void); 1401 int (*pci_init)(void); 1402 void (*mmio_init)(void); 1403 }; 1404 1405 static const struct intel_uncore_init_fun nhm_uncore_init __initconst = { 1406 .cpu_init = nhm_uncore_cpu_init, 1407 }; 1408 1409 static const struct intel_uncore_init_fun snb_uncore_init __initconst = { 1410 .cpu_init = snb_uncore_cpu_init, 1411 .pci_init = snb_uncore_pci_init, 1412 }; 1413 1414 static const struct intel_uncore_init_fun ivb_uncore_init __initconst = { 1415 .cpu_init = snb_uncore_cpu_init, 1416 .pci_init = ivb_uncore_pci_init, 1417 }; 1418 1419 static const struct intel_uncore_init_fun hsw_uncore_init __initconst = { 1420 .cpu_init = snb_uncore_cpu_init, 1421 .pci_init = hsw_uncore_pci_init, 1422 }; 1423 1424 static const struct intel_uncore_init_fun bdw_uncore_init __initconst = { 1425 .cpu_init = snb_uncore_cpu_init, 1426 .pci_init = bdw_uncore_pci_init, 1427 }; 1428 1429 static const struct intel_uncore_init_fun snbep_uncore_init __initconst = { 1430 .cpu_init = snbep_uncore_cpu_init, 1431 .pci_init = snbep_uncore_pci_init, 1432 }; 1433 1434 static const struct intel_uncore_init_fun nhmex_uncore_init __initconst = { 1435 .cpu_init = nhmex_uncore_cpu_init, 1436 }; 1437 1438 static const struct intel_uncore_init_fun ivbep_uncore_init __initconst = { 1439 .cpu_init = ivbep_uncore_cpu_init, 1440 .pci_init = ivbep_uncore_pci_init, 1441 }; 1442 1443 static const struct intel_uncore_init_fun hswep_uncore_init __initconst = { 1444 .cpu_init = hswep_uncore_cpu_init, 1445 .pci_init = hswep_uncore_pci_init, 1446 }; 1447 1448 static const struct intel_uncore_init_fun bdx_uncore_init __initconst = { 1449 .cpu_init = bdx_uncore_cpu_init, 1450 .pci_init = bdx_uncore_pci_init, 1451 }; 1452 1453 static const struct intel_uncore_init_fun knl_uncore_init __initconst = { 1454 .cpu_init = knl_uncore_cpu_init, 1455 .pci_init = knl_uncore_pci_init, 1456 }; 1457 1458 static const struct intel_uncore_init_fun skl_uncore_init __initconst = { 1459 .cpu_init = skl_uncore_cpu_init, 1460 .pci_init = skl_uncore_pci_init, 1461 }; 1462 1463 static const struct intel_uncore_init_fun skx_uncore_init __initconst = { 1464 .cpu_init = skx_uncore_cpu_init, 1465 .pci_init = skx_uncore_pci_init, 1466 }; 1467 1468 static const struct intel_uncore_init_fun icl_uncore_init __initconst = { 1469 .cpu_init = icl_uncore_cpu_init, 1470 .pci_init = skl_uncore_pci_init, 1471 }; 1472 1473 static const struct intel_uncore_init_fun snr_uncore_init __initconst = { 1474 .cpu_init = snr_uncore_cpu_init, 1475 .pci_init = snr_uncore_pci_init, 1476 .mmio_init = snr_uncore_mmio_init, 1477 }; 1478 1479 static const struct x86_cpu_id intel_uncore_match[] __initconst = { 1480 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EP, nhm_uncore_init), 1481 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM, nhm_uncore_init), 1482 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE, nhm_uncore_init), 1483 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EP, nhm_uncore_init), 1484 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE, snb_uncore_init), 1485 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE, ivb_uncore_init), 1486 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL, hsw_uncore_init), 1487 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_L, hsw_uncore_init), 1488 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_G, hsw_uncore_init), 1489 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL, bdw_uncore_init), 1490 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_G, bdw_uncore_init), 1491 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE_X, snbep_uncore_init), 1492 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EX, nhmex_uncore_init), 1493 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EX, nhmex_uncore_init), 1494 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE_X, ivbep_uncore_init), 1495 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_X, hswep_uncore_init), 1496 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_X, bdx_uncore_init), 1497 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_D, bdx_uncore_init), 1498 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNL, knl_uncore_init), 1499 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNM, knl_uncore_init), 1500 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE, skl_uncore_init), 1501 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_L, skl_uncore_init), 1502 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_X, skx_uncore_init), 1503 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_KABYLAKE_L, skl_uncore_init), 1504 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_KABYLAKE, skl_uncore_init), 1505 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE_L, icl_uncore_init), 1506 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE_NNPI, icl_uncore_init), 1507 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ICELAKE, icl_uncore_init), 1508 X86_UNCORE_MODEL_MATCH(INTEL_FAM6_ATOM_TREMONT_D, snr_uncore_init), 1509 {}, 1510 }; 1511 1512 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match); 1513 1514 static int __init intel_uncore_init(void) 1515 { 1516 const struct x86_cpu_id *id; 1517 struct intel_uncore_init_fun *uncore_init; 1518 int pret = 0, cret = 0, mret = 0, ret; 1519 1520 id = x86_match_cpu(intel_uncore_match); 1521 if (!id) 1522 return -ENODEV; 1523 1524 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) 1525 return -ENODEV; 1526 1527 max_dies = topology_max_packages() * topology_max_die_per_package(); 1528 1529 uncore_init = (struct intel_uncore_init_fun *)id->driver_data; 1530 if (uncore_init->pci_init) { 1531 pret = uncore_init->pci_init(); 1532 if (!pret) 1533 pret = uncore_pci_init(); 1534 } 1535 1536 if (uncore_init->cpu_init) { 1537 uncore_init->cpu_init(); 1538 cret = uncore_cpu_init(); 1539 } 1540 1541 if (uncore_init->mmio_init) { 1542 uncore_init->mmio_init(); 1543 mret = uncore_mmio_init(); 1544 } 1545 1546 if (cret && pret && mret) 1547 return -ENODEV; 1548 1549 /* Install hotplug callbacks to setup the targets for each package */ 1550 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE, 1551 "perf/x86/intel/uncore:online", 1552 uncore_event_cpu_online, 1553 uncore_event_cpu_offline); 1554 if (ret) 1555 goto err; 1556 return 0; 1557 1558 err: 1559 uncore_types_exit(uncore_msr_uncores); 1560 uncore_types_exit(uncore_mmio_uncores); 1561 uncore_pci_exit(); 1562 return ret; 1563 } 1564 module_init(intel_uncore_init); 1565 1566 static void __exit intel_uncore_exit(void) 1567 { 1568 cpuhp_remove_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE); 1569 uncore_types_exit(uncore_msr_uncores); 1570 uncore_types_exit(uncore_mmio_uncores); 1571 uncore_pci_exit(); 1572 } 1573 module_exit(intel_uncore_exit); 1574