1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * apic_introp.c: 30 * Has code for Advanced DDI interrupt framework support. 31 */ 32 33 #include <sys/cpuvar.h> 34 #include <sys/psm.h> 35 #include "apic.h" 36 #include <sys/sunddi.h> 37 #include <sys/ddi_impldefs.h> 38 #include <sys/mach_intr.h> 39 #include <sys/sysmacros.h> 40 #include <sys/trap.h> 41 #include <sys/pci.h> 42 #include <sys/pci_intr_lib.h> 43 44 extern struct av_head autovect[]; 45 46 /* 47 * Local Function Prototypes 48 */ 49 int apic_pci_msi_enable_vector(dev_info_t *, int, int, 50 int, int, int); 51 apic_irq_t *apic_find_irq(dev_info_t *, struct intrspec *, int); 52 static int apic_get_pending(apic_irq_t *, int); 53 static void apic_clear_mask(apic_irq_t *); 54 static void apic_set_mask(apic_irq_t *); 55 static uchar_t apic_find_multi_vectors(int, int); 56 int apic_navail_vector(dev_info_t *, int); 57 int apic_alloc_vectors(dev_info_t *, int, int, int, int); 58 void apic_free_vectors(dev_info_t *, int, int, int, int); 59 int apic_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 60 psm_intr_op_t, int *); 61 62 extern int intr_clear(void); 63 extern void intr_restore(uint_t); 64 extern uchar_t apic_bind_intr(dev_info_t *, int, uchar_t, uchar_t); 65 extern int apic_allocate_irq(int); 66 extern int apic_introp_xlate(dev_info_t *, struct intrspec *, int); 67 extern int apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu, int safe); 68 extern boolean_t apic_cpu_in_range(int cpu); 69 70 /* 71 * MSI support flag: 72 * reflects whether MSI is supported at APIC level 73 * it can also be patched through /etc/system 74 * 75 * 0 = default value - don't know and need to call apic_check_msi_support() 76 * to find out then set it accordingly 77 * 1 = supported 78 * -1 = not supported 79 */ 80 int apic_support_msi = 0; 81 82 /* Multiple vector support for MSI */ 83 int apic_multi_msi_enable = 1; 84 int apic_multi_msi_max = 2; 85 86 extern uchar_t apic_ipltopri[MAXIPL+1]; 87 extern uchar_t apic_vector_to_irq[APIC_MAX_VECTOR+1]; 88 extern int apic_max_device_irq; 89 extern int apic_min_device_irq; 90 extern apic_irq_t *apic_irq_table[APIC_MAX_VECTOR+1]; 91 extern volatile uint32_t *apicadr; /* virtual addr of local APIC */ 92 extern volatile int32_t *apicioadr[MAX_IO_APIC]; 93 extern lock_t apic_ioapic_lock; 94 extern kmutex_t airq_mutex; 95 extern apic_cpus_info_t *apic_cpus; 96 extern int apic_first_avail_irq; 97 98 99 /* 100 * apic_pci_msi_enable_vector: 101 * Set the address/data fields in the MSI/X capability structure 102 * XXX: MSI-X support 103 */ 104 /* ARGSUSED */ 105 int 106 apic_pci_msi_enable_vector(dev_info_t *dip, int type, int inum, int vector, 107 int count, int target_apic_id) 108 { 109 uint64_t msi_addr, msi_data; 110 ushort_t msi_ctrl; 111 int cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip); 112 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(dip); 113 114 DDI_INTR_IMPLDBG((CE_CONT, "apic_pci_msi_enable_vector: dip=0x%p\n" 115 "\tdriver = %s, inum=0x%x vector=0x%x apicid=0x%x\n", (void *)dip, 116 ddi_driver_name(dip), inum, vector, target_apic_id)); 117 118 if (handle == NULL) 119 return (PSM_FAILURE); 120 121 /* MSI Address */ 122 msi_addr = (MSI_ADDR_HDR | (target_apic_id << MSI_ADDR_DEST_SHIFT)); 123 msi_addr |= ((MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) | 124 (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT)); 125 126 /* MSI Data: MSI is edge triggered according to spec */ 127 msi_data = ((MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) | vector); 128 129 DDI_INTR_IMPLDBG((CE_CONT, "apic_pci_msi_enable_vector: addr=0x%lx " 130 "data=0x%lx\n", (long)msi_addr, (long)msi_data)); 131 132 if (type == DDI_INTR_TYPE_MSI) { 133 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL); 134 135 /* Set the bits to inform how many MSIs are enabled */ 136 msi_ctrl |= ((highbit(count) -1) << PCI_MSI_MME_SHIFT); 137 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl); 138 139 pci_config_put32(handle, 140 cap_ptr + PCI_MSI_ADDR_OFFSET, msi_addr); 141 142 if (msi_ctrl & PCI_MSI_64BIT_MASK) { 143 pci_config_put32(handle, 144 cap_ptr + PCI_MSI_ADDR_OFFSET + 4, msi_addr >> 32); 145 pci_config_put16(handle, 146 cap_ptr + PCI_MSI_64BIT_DATA, msi_data); 147 } else { 148 pci_config_put16(handle, 149 cap_ptr + PCI_MSI_32BIT_DATA, msi_data); 150 } 151 152 } else if (type == DDI_INTR_TYPE_MSIX) { 153 uintptr_t off; 154 ddi_intr_msix_t *msix_p = i_ddi_get_msix(dip); 155 156 /* Offset into the "inum"th entry in the MSI-X table */ 157 off = (uintptr_t)msix_p->msix_tbl_addr + 158 (inum * PCI_MSIX_VECTOR_SIZE); 159 160 ddi_put32(msix_p->msix_tbl_hdl, 161 (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), msi_data); 162 ddi_put64(msix_p->msix_tbl_hdl, 163 (uint64_t *)(off + PCI_MSIX_LOWER_ADDR_OFFSET), msi_addr); 164 } 165 166 return (PSM_SUCCESS); 167 } 168 169 170 /* 171 * This function returns the no. of vectors available for the pri. 172 * dip is not used at this moment. If we really don't need that, 173 * it will be removed. 174 */ 175 /*ARGSUSED*/ 176 int 177 apic_navail_vector(dev_info_t *dip, int pri) 178 { 179 int lowest, highest, i, navail, count; 180 181 DDI_INTR_IMPLDBG((CE_CONT, "apic_navail_vector: dip: %p, pri: %x\n", 182 (void *)dip, pri)); 183 184 highest = apic_ipltopri[pri] + APIC_VECTOR_MASK; 185 lowest = apic_ipltopri[pri - 1] + APIC_VECTOR_PER_IPL; 186 navail = count = 0; 187 188 /* It has to be contiguous */ 189 for (i = lowest; i < highest; i++) { 190 count = 0; 191 while ((apic_vector_to_irq[i] == APIC_RESV_IRQ) && 192 (i < highest)) { 193 if ((i == T_FASTTRAP) || (i == APIC_SPUR_INTR)) 194 break; 195 count++; 196 i++; 197 } 198 if (count > navail) 199 navail = count; 200 } 201 return (navail); 202 } 203 204 static uchar_t 205 apic_find_multi_vectors(int pri, int count) 206 { 207 int lowest, highest, i, navail, start; 208 209 DDI_INTR_IMPLDBG((CE_CONT, "apic_find_mult: pri: %x, count: %x\n", 210 pri, count)); 211 212 highest = apic_ipltopri[pri] + APIC_VECTOR_MASK; 213 lowest = apic_ipltopri[pri - 1] + APIC_VECTOR_PER_IPL; 214 navail = 0; 215 216 /* It has to be contiguous */ 217 for (i = lowest; i < highest; i++) { 218 navail = 0; 219 start = i; 220 while ((apic_vector_to_irq[i] == APIC_RESV_IRQ) && 221 (i < highest)) { 222 if ((i == T_FASTTRAP) || (i == APIC_SPUR_INTR)) 223 break; 224 navail++; 225 if (navail >= count) 226 return (start); 227 i++; 228 } 229 } 230 return (0); 231 } 232 233 234 /* 235 * It finds the apic_irq_t associates with the dip, ispec and type. 236 */ 237 apic_irq_t * 238 apic_find_irq(dev_info_t *dip, struct intrspec *ispec, int type) 239 { 240 apic_irq_t *irqp; 241 int i; 242 243 DDI_INTR_IMPLDBG((CE_CONT, "apic_find_irq: dip=0x%p vec=0x%x " 244 "ipl=0x%x type=0x%x\n", (void *)dip, ispec->intrspec_vec, 245 ispec->intrspec_pri, type)); 246 247 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) { 248 if ((irqp = apic_irq_table[i]) == NULL) 249 continue; 250 if ((irqp->airq_dip == dip) && 251 (irqp->airq_origirq == ispec->intrspec_vec) && 252 (irqp->airq_ipl == ispec->intrspec_pri)) { 253 if (DDI_INTR_IS_MSI_OR_MSIX(type)) { 254 if (APIC_IS_MSI_OR_MSIX_INDEX(irqp-> 255 airq_mps_intr_index)) 256 return (irqp); 257 } else 258 return (irqp); 259 } 260 } 261 DDI_INTR_IMPLDBG((CE_CONT, "apic_find_irq: return NULL\n")); 262 return (NULL); 263 } 264 265 266 /* 267 * This function will return the pending bit of the irqp. 268 * It either comes from the IRR register of the APIC or the RDT 269 * entry of the I/O APIC. 270 * For the IRR to work, it needs to be to its binding CPU 271 */ 272 static int 273 apic_get_pending(apic_irq_t *irqp, int type) 274 { 275 int bit, index, irr, pending; 276 int intin_no; 277 volatile int32_t *ioapic; 278 279 DDI_INTR_IMPLDBG((CE_CONT, "apic_get_pending: irqp: %p, cpuid: %x " 280 "type: %x\n", (void *)irqp, irqp->airq_cpu & ~IRQ_USER_BOUND, 281 type)); 282 283 /* need to get on the bound cpu */ 284 mutex_enter(&cpu_lock); 285 affinity_set(irqp->airq_cpu & ~IRQ_USER_BOUND); 286 287 index = irqp->airq_vector / 32; 288 bit = irqp->airq_vector % 32; 289 irr = apicadr[APIC_IRR_REG + index]; 290 291 affinity_clear(); 292 mutex_exit(&cpu_lock); 293 294 pending = (irr & (1 << bit)) ? 1 : 0; 295 if (!pending && (type == DDI_INTR_TYPE_FIXED)) { 296 /* check I/O APIC for fixed interrupt */ 297 intin_no = irqp->airq_intin_no; 298 ioapic = apicioadr[irqp->airq_ioapicindex]; 299 pending = (READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no) & 300 AV_PENDING) ? 1 : 0; 301 } 302 return (pending); 303 } 304 305 306 /* 307 * This function will clear the mask for the interrupt on the I/O APIC 308 */ 309 static void 310 apic_clear_mask(apic_irq_t *irqp) 311 { 312 int intin_no; 313 int iflag; 314 int32_t rdt_entry; 315 volatile int32_t *ioapic; 316 317 DDI_INTR_IMPLDBG((CE_CONT, "apic_clear_mask: irqp: %p\n", 318 (void *)irqp)); 319 320 intin_no = irqp->airq_intin_no; 321 ioapic = apicioadr[irqp->airq_ioapicindex]; 322 323 iflag = intr_clear(); 324 lock_set(&apic_ioapic_lock); 325 326 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no); 327 328 /* clear mask */ 329 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no, 330 ((~AV_MASK) & rdt_entry)); 331 332 lock_clear(&apic_ioapic_lock); 333 intr_restore(iflag); 334 } 335 336 337 /* 338 * This function will mask the interrupt on the I/O APIC 339 */ 340 static void 341 apic_set_mask(apic_irq_t *irqp) 342 { 343 int intin_no; 344 volatile int32_t *ioapic; 345 int iflag; 346 int32_t rdt_entry; 347 348 DDI_INTR_IMPLDBG((CE_CONT, "apic_set_mask: irqp: %p\n", (void *)irqp)); 349 350 intin_no = irqp->airq_intin_no; 351 ioapic = apicioadr[irqp->airq_ioapicindex]; 352 353 iflag = intr_clear(); 354 355 lock_set(&apic_ioapic_lock); 356 357 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no); 358 359 /* mask it */ 360 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no, 361 (AV_MASK | rdt_entry)); 362 363 lock_clear(&apic_ioapic_lock); 364 intr_restore(iflag); 365 } 366 367 368 /* 369 * This function allocate "count" vector(s) for the given "dip/pri/type" 370 */ 371 int 372 apic_alloc_vectors(dev_info_t *dip, int inum, int count, int pri, int type) 373 { 374 int rcount, i; 375 uchar_t start, irqno, cpu; 376 short idx; 377 major_t major; 378 apic_irq_t *irqptr; 379 380 /* for MSI/X only */ 381 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) 382 return (0); 383 384 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_vectors: dip=0x%p type=%d " 385 "inum=0x%x pri=0x%x count=0x%x\n", 386 (void *)dip, type, inum, pri, count)); 387 388 if (count > 1) { 389 if (apic_multi_msi_enable == 0) 390 count = 1; 391 else if (count > apic_multi_msi_max) 392 count = apic_multi_msi_max; 393 } 394 395 if ((rcount = apic_navail_vector(dip, pri)) > count) 396 rcount = count; 397 398 mutex_enter(&airq_mutex); 399 400 for (start = 0; rcount > 0; rcount--) { 401 if ((start = apic_find_multi_vectors(pri, rcount)) != 0) 402 break; 403 } 404 405 if (start == 0) { 406 /* no vector available */ 407 mutex_exit(&airq_mutex); 408 return (0); 409 } 410 411 idx = (short)((type == DDI_INTR_TYPE_MSI) ? MSI_INDEX : MSIX_INDEX); 412 major = (dip != NULL) ? ddi_name_to_major(ddi_get_name(dip)) : 0; 413 for (i = 0; i < rcount; i++) { 414 if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == 415 (uchar_t)-1) { 416 mutex_exit(&airq_mutex); 417 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_vectors: " 418 "apic_allocate_irq failed\n")); 419 return (i); 420 } 421 apic_max_device_irq = max(irqno, apic_max_device_irq); 422 apic_min_device_irq = min(irqno, apic_min_device_irq); 423 irqptr = apic_irq_table[irqno]; 424 #ifdef DEBUG 425 if (apic_vector_to_irq[start + i] != APIC_RESV_IRQ) 426 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_vectors: " 427 "apic_vector_to_irq is not APIC_RESV_IRQ\n")); 428 #endif 429 apic_vector_to_irq[start + i] = (uchar_t)irqno; 430 431 irqptr->airq_vector = (uchar_t)(start + i); 432 irqptr->airq_ioapicindex = (uchar_t)inum; /* start */ 433 irqptr->airq_intin_no = (uchar_t)rcount; 434 irqptr->airq_ipl = pri; 435 irqptr->airq_vector = start + i; 436 irqptr->airq_origirq = (uchar_t)(inum + i); 437 irqptr->airq_share_id = 0; 438 irqptr->airq_mps_intr_index = idx; 439 irqptr->airq_dip = dip; 440 irqptr->airq_major = major; 441 if (i == 0) /* they all bound to the same cpu */ 442 cpu = irqptr->airq_cpu = apic_bind_intr(dip, irqno, 443 0xff, 0xff); 444 else 445 irqptr->airq_cpu = cpu; 446 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_vectors: irq=0x%x " 447 "dip=0x%p vector=0x%x origirq=0x%x pri=0x%x\n", irqno, 448 (void *)irqptr->airq_dip, irqptr->airq_vector, 449 irqptr->airq_origirq, pri)); 450 } 451 mutex_exit(&airq_mutex); 452 return (rcount); 453 } 454 455 456 void 457 apic_free_vectors(dev_info_t *dip, int inum, int count, int pri, int type) 458 { 459 int i; 460 apic_irq_t *irqptr; 461 struct intrspec ispec; 462 463 DDI_INTR_IMPLDBG((CE_CONT, "apic_free_vectors: dip: %p inum: %x " 464 "count: %x pri: %x type: %x\n", 465 (void *)dip, inum, count, pri, type)); 466 467 /* for MSI/X only */ 468 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) 469 return; 470 471 for (i = 0; i < count; i++) { 472 DDI_INTR_IMPLDBG((CE_CONT, "apic_free_vectors: inum=0x%x " 473 "pri=0x%x count=0x%x\n", inum, pri, count)); 474 ispec.intrspec_vec = inum + i; 475 ispec.intrspec_pri = pri; 476 if ((irqptr = apic_find_irq(dip, &ispec, type)) == NULL) { 477 DDI_INTR_IMPLDBG((CE_CONT, "apic_free_vectors: " 478 "dip=0x%p inum=0x%x pri=0x%x apic_find_irq() " 479 "failed\n", (void *)dip, inum, pri)); 480 continue; 481 } 482 irqptr->airq_mps_intr_index = FREE_INDEX; 483 apic_vector_to_irq[irqptr->airq_vector] = APIC_RESV_IRQ; 484 } 485 } 486 487 488 /* 489 * check whether the system supports MSI 490 * 491 * If PCI-E capability is found, then this must be a PCI-E system. 492 * Since MSI is required for PCI-E system, it returns PSM_SUCCESS 493 * to indicate this system supports MSI. 494 */ 495 int 496 apic_check_msi_support() 497 { 498 dev_info_t *cdip; 499 char dev_type[16]; 500 int dev_len; 501 502 DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support:\n")); 503 504 /* 505 * check whether the first level children of root_node have 506 * PCI-E capability 507 */ 508 for (cdip = ddi_get_child(ddi_root_node()); cdip != NULL; 509 cdip = ddi_get_next_sibling(cdip)) { 510 511 DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: cdip: 0x%p," 512 " driver: %s, binding: %s, nodename: %s\n", (void *)cdip, 513 ddi_driver_name(cdip), ddi_binding_name(cdip), 514 ddi_node_name(cdip))); 515 dev_len = sizeof (dev_type); 516 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS, 517 "device_type", (caddr_t)dev_type, &dev_len) 518 != DDI_PROP_SUCCESS) 519 continue; 520 if (strcmp(dev_type, "pciex") == 0) 521 return (PSM_SUCCESS); 522 } 523 524 /* MSI is not supported on this system */ 525 DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: no 'pciex' " 526 "device_type found\n")); 527 return (PSM_FAILURE); 528 } 529 530 int 531 apic_get_vector_intr_info(int vecirq, apic_get_intr_t *intr_params_p) 532 { 533 struct autovec *av_dev; 534 uchar_t irqno; 535 int i; 536 apic_irq_t *irq_p; 537 538 /* Sanity check the vector/irq argument. */ 539 ASSERT((vecirq >= 0) || (vecirq <= APIC_MAX_VECTOR)); 540 541 mutex_enter(&airq_mutex); 542 543 /* 544 * Convert the vecirq arg to an irq using vector_to_irq table 545 * if the arg is a vector. Pass thru if already an irq. 546 */ 547 if ((intr_params_p->avgi_req_flags & PSMGI_INTRBY_FLAGS) == 548 PSMGI_INTRBY_VEC) 549 irqno = apic_vector_to_irq[vecirq]; 550 else 551 irqno = vecirq; 552 553 irq_p = apic_irq_table[irqno]; 554 555 if ((irq_p == NULL) || 556 (irq_p->airq_temp_cpu == IRQ_UNBOUND) || 557 (irq_p->airq_temp_cpu == IRQ_UNINIT)) { 558 mutex_exit(&airq_mutex); 559 return (PSM_FAILURE); 560 } 561 562 if (intr_params_p->avgi_req_flags & PSMGI_REQ_CPUID) { 563 564 /* Get the (temp) cpu from apic_irq table, indexed by irq. */ 565 intr_params_p->avgi_cpu_id = irq_p->airq_temp_cpu; 566 567 /* Return user bound info for intrd. */ 568 if (intr_params_p->avgi_cpu_id & IRQ_USER_BOUND) { 569 intr_params_p->avgi_cpu_id &= ~IRQ_USER_BOUND; 570 intr_params_p->avgi_cpu_id |= PSMGI_CPU_USER_BOUND; 571 } 572 } 573 574 if (intr_params_p->avgi_req_flags & PSMGI_REQ_VECTOR) { 575 intr_params_p->avgi_vector = irq_p->airq_vector; 576 } 577 578 if (intr_params_p->avgi_req_flags & 579 (PSMGI_REQ_NUM_DEVS | PSMGI_REQ_GET_DEVS)) { 580 /* Get number of devices from apic_irq table shared field. */ 581 intr_params_p->avgi_num_devs = irq_p->airq_share; 582 } 583 584 if (intr_params_p->avgi_req_flags & PSMGI_REQ_GET_DEVS) { 585 586 intr_params_p->avgi_req_flags |= PSMGI_REQ_NUM_DEVS; 587 588 /* Some devices have NULL dip. Don't count these. */ 589 if (intr_params_p->avgi_num_devs > 0) { 590 for (i = 0, av_dev = autovect[irqno].avh_link; 591 av_dev; av_dev = av_dev->av_link) 592 if (av_dev->av_vector && av_dev->av_dip) 593 i++; 594 intr_params_p->avgi_num_devs = 595 MIN(intr_params_p->avgi_num_devs, i); 596 } 597 598 /* There are no viable dips to return. */ 599 if (intr_params_p->avgi_num_devs == 0) 600 intr_params_p->avgi_dip_list = NULL; 601 602 else { /* Return list of dips */ 603 604 /* Allocate space in array for that number of devs. */ 605 intr_params_p->avgi_dip_list = kmem_zalloc( 606 intr_params_p->avgi_num_devs * 607 sizeof (dev_info_t *), 608 KM_SLEEP); 609 610 /* 611 * Loop through the device list of the autovec table 612 * filling in the dip array. 613 * 614 * Note that the autovect table may have some special 615 * entries which contain NULL dips. These will be 616 * ignored. 617 */ 618 for (i = 0, av_dev = autovect[irqno].avh_link; 619 av_dev; av_dev = av_dev->av_link) 620 if (av_dev->av_vector && av_dev->av_dip) 621 intr_params_p->avgi_dip_list[i++] = 622 av_dev->av_dip; 623 } 624 } 625 626 mutex_exit(&airq_mutex); 627 628 return (PSM_SUCCESS); 629 } 630 631 /* 632 * apic_pci_msi_unconfigure: 633 * 634 * This and next two interfaces are copied from pci_intr_lib.c 635 * Do ensure that these two files stay in sync. 636 * These needed to be copied over here to avoid a deadlock situation on 637 * certain mp systems that use MSI interrupts. 638 * 639 * IMPORTANT regards next three interfaces: 640 * i) are called only for MSI/X interrupts. 641 * ii) called with interrupts disabled, and must not block 642 */ 643 int 644 apic_pci_msi_unconfigure(dev_info_t *rdip, int type, int inum) 645 { 646 ushort_t msi_ctrl; 647 int cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip); 648 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(rdip); 649 650 if (handle == NULL) 651 return (PSM_FAILURE); 652 653 if (type == DDI_INTR_TYPE_MSI) { 654 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL); 655 msi_ctrl &= (~PCI_MSI_MME_MASK); 656 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl); 657 pci_config_put32(handle, cap_ptr + PCI_MSI_ADDR_OFFSET, 0); 658 659 if (msi_ctrl & PCI_MSI_64BIT_MASK) { 660 pci_config_put16(handle, 661 cap_ptr + PCI_MSI_64BIT_DATA, 0); 662 pci_config_put32(handle, 663 cap_ptr + PCI_MSI_ADDR_OFFSET + 4, 0); 664 } else { 665 pci_config_put16(handle, 666 cap_ptr + PCI_MSI_32BIT_DATA, 0); 667 } 668 669 } else if (type == DDI_INTR_TYPE_MSIX) { 670 uintptr_t off; 671 ddi_intr_msix_t *msix_p = i_ddi_get_msix(rdip); 672 673 /* Offset into the "inum"th entry in the MSI-X table */ 674 off = (uintptr_t)msix_p->msix_tbl_addr + 675 (inum * PCI_MSIX_VECTOR_SIZE); 676 677 /* Reset the "data" and "addr" bits */ 678 ddi_put32(msix_p->msix_tbl_hdl, 679 (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), 0); 680 ddi_put64(msix_p->msix_tbl_hdl, (uint64_t *)off, 0); 681 } 682 683 return (PSM_SUCCESS); 684 } 685 686 687 /* 688 * apic_pci_msi_enable_mode: 689 */ 690 int 691 apic_pci_msi_enable_mode(dev_info_t *rdip, int type, int inum) 692 { 693 ushort_t msi_ctrl; 694 int cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip); 695 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(rdip); 696 697 if (handle == NULL) 698 return (PSM_FAILURE); 699 700 if (type == DDI_INTR_TYPE_MSI) { 701 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL); 702 if ((msi_ctrl & PCI_MSI_ENABLE_BIT)) 703 return (PSM_SUCCESS); 704 705 msi_ctrl |= PCI_MSI_ENABLE_BIT; 706 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl); 707 708 } else if (type == DDI_INTR_TYPE_MSIX) { 709 uintptr_t off; 710 ddi_intr_msix_t *msix_p; 711 712 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL); 713 714 if (msi_ctrl & PCI_MSIX_ENABLE_BIT) 715 return (PSM_SUCCESS); 716 717 msi_ctrl |= PCI_MSIX_ENABLE_BIT; 718 pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL, msi_ctrl); 719 720 msix_p = i_ddi_get_msix(rdip); 721 722 /* Offset into "inum"th entry in the MSI-X table & clear mask */ 723 off = (uintptr_t)msix_p->msix_tbl_addr + (inum * 724 PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET; 725 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, 0); 726 } 727 728 return (PSM_SUCCESS); 729 } 730 731 /* 732 * apic_pci_msi_disable_mode: 733 */ 734 int 735 apic_pci_msi_disable_mode(dev_info_t *rdip, int type, int inum) 736 { 737 ushort_t msi_ctrl; 738 int cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip); 739 ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(rdip); 740 741 if (handle == NULL) 742 return (PSM_FAILURE); 743 744 if (type == DDI_INTR_TYPE_MSI) { 745 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL); 746 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT)) 747 return (PSM_SUCCESS); 748 749 msi_ctrl &= ~PCI_MSI_ENABLE_BIT; /* MSI disable */ 750 pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl); 751 752 } else if (type == DDI_INTR_TYPE_MSIX) { 753 uintptr_t off; 754 ddi_intr_msix_t *msix_p; 755 756 msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL); 757 758 if (!(msi_ctrl & PCI_MSIX_ENABLE_BIT)) 759 return (PSM_SUCCESS); 760 761 msix_p = i_ddi_get_msix(rdip); 762 763 /* Offset into "inum"th entry in the MSI-X table & mask it */ 764 off = (uintptr_t)msix_p->msix_tbl_addr + (inum * 765 PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET; 766 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, 0x1); 767 } 768 769 return (PSM_SUCCESS); 770 } 771 772 /* 773 * This function provides external interface to the nexus for all 774 * functionalities related to the new DDI interrupt framework. 775 * 776 * Input: 777 * dip - pointer to the dev_info structure of the requested device 778 * hdlp - pointer to the internal interrupt handle structure for the 779 * requested interrupt 780 * intr_op - opcode for this call 781 * result - pointer to the integer that will hold the result to be 782 * passed back if return value is PSM_SUCCESS 783 * 784 * Output: 785 * return value is either PSM_SUCCESS or PSM_FAILURE 786 */ 787 int 788 apic_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 789 psm_intr_op_t intr_op, int *result) 790 { 791 int cap, ret; 792 int count_vec; 793 int cpu; 794 int old_priority; 795 int new_priority; 796 apic_irq_t *irqp; 797 struct intrspec *ispec, intr_spec; 798 799 DDI_INTR_IMPLDBG((CE_CONT, "apic_intr_ops: dip: %p hdlp: %p " 800 "intr_op: %x\n", (void *)dip, (void *)hdlp, intr_op)); 801 802 ispec = &intr_spec; 803 ispec->intrspec_pri = hdlp->ih_pri; 804 ispec->intrspec_vec = hdlp->ih_inum; 805 ispec->intrspec_func = hdlp->ih_cb_func; 806 807 switch (intr_op) { 808 case PSM_INTR_OP_CHECK_MSI: 809 /* 810 * Check MSI/X is supported or not at APIC level and 811 * masked off the MSI/X bits in hdlp->ih_type if not 812 * supported before return. If MSI/X is supported, 813 * leave the ih_type unchanged and return. 814 * 815 * hdlp->ih_type passed in from the nexus has all the 816 * interrupt types supported by the device. 817 */ 818 if (apic_support_msi == 0) { 819 /* 820 * if apic_support_msi is not set, call 821 * apic_check_msi_support() to check whether msi 822 * is supported first 823 */ 824 if (apic_check_msi_support() == PSM_SUCCESS) 825 apic_support_msi = 1; 826 else 827 apic_support_msi = -1; 828 } 829 if (apic_support_msi == 1) 830 *result = hdlp->ih_type; 831 else 832 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 833 DDI_INTR_TYPE_MSIX); 834 break; 835 case PSM_INTR_OP_ALLOC_VECTORS: 836 *result = apic_alloc_vectors(dip, hdlp->ih_inum, 837 hdlp->ih_scratch1, hdlp->ih_pri, hdlp->ih_type); 838 break; 839 case PSM_INTR_OP_FREE_VECTORS: 840 apic_free_vectors(dip, hdlp->ih_inum, hdlp->ih_scratch1, 841 hdlp->ih_pri, hdlp->ih_type); 842 break; 843 case PSM_INTR_OP_NAVAIL_VECTORS: 844 *result = apic_navail_vector(dip, hdlp->ih_pri); 845 break; 846 case PSM_INTR_OP_XLATE_VECTOR: 847 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 848 *result = apic_introp_xlate(dip, ispec, hdlp->ih_type); 849 break; 850 case PSM_INTR_OP_GET_PENDING: 851 if ((irqp = apic_find_irq(dip, ispec, hdlp->ih_type)) == NULL) 852 return (PSM_FAILURE); 853 *result = apic_get_pending(irqp, hdlp->ih_type); 854 break; 855 case PSM_INTR_OP_CLEAR_MASK: 856 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED) 857 return (PSM_FAILURE); 858 irqp = apic_find_irq(dip, ispec, hdlp->ih_type); 859 if (irqp == NULL) 860 return (PSM_FAILURE); 861 apic_clear_mask(irqp); 862 break; 863 case PSM_INTR_OP_SET_MASK: 864 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED) 865 return (PSM_FAILURE); 866 if ((irqp = apic_find_irq(dip, ispec, hdlp->ih_type)) == NULL) 867 return (PSM_FAILURE); 868 apic_set_mask(irqp); 869 break; 870 case PSM_INTR_OP_GET_CAP: 871 cap = DDI_INTR_FLAG_PENDING; 872 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 873 cap |= DDI_INTR_FLAG_MASKABLE; 874 *result = cap; 875 break; 876 case PSM_INTR_OP_GET_SHARED: 877 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED) 878 return (PSM_FAILURE); 879 if ((irqp = apic_find_irq(dip, ispec, hdlp->ih_type)) == NULL) 880 return (PSM_FAILURE); 881 *result = irqp->airq_share ? 1: 0; 882 break; 883 case PSM_INTR_OP_SET_PRI: 884 old_priority = hdlp->ih_pri; /* save old value */ 885 new_priority = *(int *)result; /* try the new value */ 886 887 /* First, check if "hdlp->ih_scratch1" vectors exist? */ 888 if (apic_navail_vector(dip, new_priority) < hdlp->ih_scratch1) 889 return (PSM_FAILURE); 890 891 /* Now allocate the vectors */ 892 count_vec = apic_alloc_vectors(dip, hdlp->ih_inum, 893 hdlp->ih_scratch1, new_priority, hdlp->ih_type); 894 895 /* Did we get fewer vectors? */ 896 if (count_vec != hdlp->ih_scratch1) { 897 apic_free_vectors(dip, hdlp->ih_inum, count_vec, 898 new_priority, hdlp->ih_type); 899 return (PSM_FAILURE); 900 } 901 902 /* Finally, free the previously allocated vectors */ 903 apic_free_vectors(dip, hdlp->ih_inum, count_vec, 904 old_priority, hdlp->ih_type); 905 hdlp->ih_pri = new_priority; /* set the new value */ 906 break; 907 case PSM_INTR_OP_SET_CPU: 908 /* 909 * The interrupt handle given here has been allocated 910 * specifically for this command, and ih_private carries 911 * a CPU value. 912 */ 913 cpu = (int)(intptr_t)hdlp->ih_private; 914 915 if (!apic_cpu_in_range(cpu)) { 916 *result = EINVAL; 917 return (PSM_FAILURE); 918 } 919 920 mutex_enter(&airq_mutex); 921 922 /* Convert the vector to the irq using vector_to_irq table. */ 923 irqp = apic_irq_table[apic_vector_to_irq[hdlp->ih_vector]]; 924 if (irqp == NULL) { 925 mutex_exit(&airq_mutex); 926 *result = ENXIO; 927 return (PSM_FAILURE); 928 } 929 ret = apic_rebind_all(irqp, cpu, 1); 930 mutex_exit(&airq_mutex); 931 if (ret) { 932 *result = EIO; 933 return (PSM_FAILURE); 934 } 935 *result = 0; 936 break; 937 case PSM_INTR_OP_GET_INTR: 938 /* 939 * The interrupt handle given here has been allocated 940 * specifically for this command, and ih_private carries 941 * a pointer to a apic_get_intr_t. 942 */ 943 if (apic_get_vector_intr_info( 944 hdlp->ih_vector, hdlp->ih_private) != PSM_SUCCESS) 945 return (PSM_FAILURE); 946 break; 947 case PSM_INTR_OP_SET_CAP: 948 default: 949 return (PSM_FAILURE); 950 } 951 return (PSM_SUCCESS); 952 } 953