17ff178cdSJimmy Vetayases /* 27ff178cdSJimmy Vetayases * CDDL HEADER START 37ff178cdSJimmy Vetayases * 47ff178cdSJimmy Vetayases * The contents of this file are subject to the terms of the 57ff178cdSJimmy Vetayases * Common Development and Distribution License (the "License"). 67ff178cdSJimmy Vetayases * You may not use this file except in compliance with the License. 77ff178cdSJimmy Vetayases * 87ff178cdSJimmy Vetayases * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97ff178cdSJimmy Vetayases * or http://www.opensolaris.org/os/licensing. 107ff178cdSJimmy Vetayases * See the License for the specific language governing permissions 117ff178cdSJimmy Vetayases * and limitations under the License. 127ff178cdSJimmy Vetayases * 137ff178cdSJimmy Vetayases * When distributing Covered Code, include this CDDL HEADER in each 147ff178cdSJimmy Vetayases * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157ff178cdSJimmy Vetayases * If applicable, add the following below this CDDL HEADER, with the 167ff178cdSJimmy Vetayases * fields enclosed by brackets "[]" replaced with your own identifying 177ff178cdSJimmy Vetayases * information: Portions Copyright [yyyy] [name of copyright owner] 187ff178cdSJimmy Vetayases * 197ff178cdSJimmy Vetayases * CDDL HEADER END 207ff178cdSJimmy Vetayases */ 217ff178cdSJimmy Vetayases 227ff178cdSJimmy Vetayases /* 237ff178cdSJimmy Vetayases * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 247ff178cdSJimmy Vetayases */ 257ff178cdSJimmy Vetayases /* 267ff178cdSJimmy Vetayases * Copyright (c) 2010, Intel Corporation. 277ff178cdSJimmy Vetayases * All rights reserved. 287ff178cdSJimmy Vetayases */ 297ff178cdSJimmy Vetayases 307ff178cdSJimmy Vetayases #include <sys/processor.h> 317ff178cdSJimmy Vetayases #include <sys/time.h> 327ff178cdSJimmy Vetayases #include <sys/psm.h> 337ff178cdSJimmy Vetayases #include <sys/smp_impldefs.h> 347ff178cdSJimmy Vetayases #include <sys/cram.h> 357ff178cdSJimmy Vetayases #include <sys/acpi/acpi.h> 367ff178cdSJimmy Vetayases #include <sys/acpica.h> 377ff178cdSJimmy Vetayases #include <sys/psm_common.h> 387ff178cdSJimmy Vetayases #include <sys/pit.h> 397ff178cdSJimmy Vetayases #include <sys/ddi.h> 407ff178cdSJimmy Vetayases #include <sys/sunddi.h> 417ff178cdSJimmy Vetayases #include <sys/ddi_impldefs.h> 427ff178cdSJimmy Vetayases #include <sys/pci.h> 437ff178cdSJimmy Vetayases #include <sys/promif.h> 447ff178cdSJimmy Vetayases #include <sys/x86_archext.h> 457ff178cdSJimmy Vetayases #include <sys/cpc_impl.h> 467ff178cdSJimmy Vetayases #include <sys/uadmin.h> 477ff178cdSJimmy Vetayases #include <sys/panic.h> 487ff178cdSJimmy Vetayases #include <sys/debug.h> 497ff178cdSJimmy Vetayases #include <sys/archsystm.h> 507ff178cdSJimmy Vetayases #include <sys/trap.h> 517ff178cdSJimmy Vetayases #include <sys/machsystm.h> 527ff178cdSJimmy Vetayases #include <sys/sysmacros.h> 537ff178cdSJimmy Vetayases #include <sys/cpuvar.h> 547ff178cdSJimmy Vetayases #include <sys/rm_platter.h> 557ff178cdSJimmy Vetayases #include <sys/privregs.h> 567ff178cdSJimmy Vetayases #include <sys/note.h> 577ff178cdSJimmy Vetayases #include <sys/pci_intr_lib.h> 587ff178cdSJimmy Vetayases #include <sys/spl.h> 597ff178cdSJimmy Vetayases #include <sys/clock.h> 607ff178cdSJimmy Vetayases #include <sys/dditypes.h> 617ff178cdSJimmy Vetayases #include <sys/sunddi.h> 627ff178cdSJimmy Vetayases #include <sys/x_call.h> 637ff178cdSJimmy Vetayases #include <sys/reboot.h> 647ff178cdSJimmy Vetayases #include <sys/apix.h> 657ff178cdSJimmy Vetayases 667ff178cdSJimmy Vetayases static int apix_get_avail_vector_oncpu(uint32_t, int, int); 677ff178cdSJimmy Vetayases static apix_vector_t *apix_init_vector(processorid_t, uchar_t); 687ff178cdSJimmy Vetayases static void apix_cleanup_vector(apix_vector_t *); 697ff178cdSJimmy Vetayases static void apix_insert_av(apix_vector_t *, void *, avfunc, caddr_t, caddr_t, 707ff178cdSJimmy Vetayases uint64_t *, int, dev_info_t *); 717ff178cdSJimmy Vetayases static void apix_remove_av(apix_vector_t *, struct autovec *); 727ff178cdSJimmy Vetayases static void apix_clear_dev_map(dev_info_t *, int, int); 737ff178cdSJimmy Vetayases static boolean_t apix_is_cpu_enabled(processorid_t); 747ff178cdSJimmy Vetayases static void apix_wait_till_seen(processorid_t, int); 757ff178cdSJimmy Vetayases 767ff178cdSJimmy Vetayases #define GET_INTR_INUM(ihdlp) \ 777ff178cdSJimmy Vetayases (((ihdlp) != NULL) ? ((ddi_intr_handle_impl_t *)(ihdlp))->ih_inum : 0) 787ff178cdSJimmy Vetayases 797ff178cdSJimmy Vetayases apix_rebind_info_t apix_rebindinfo = {0, 0, 0, NULL, 0, NULL}; 807ff178cdSJimmy Vetayases 817ff178cdSJimmy Vetayases /* 827ff178cdSJimmy Vetayases * Allocate IPI 837ff178cdSJimmy Vetayases * 847ff178cdSJimmy Vetayases * Return vector number or 0 on error 857ff178cdSJimmy Vetayases */ 867ff178cdSJimmy Vetayases uchar_t 877ff178cdSJimmy Vetayases apix_alloc_ipi(int ipl) 887ff178cdSJimmy Vetayases { 897ff178cdSJimmy Vetayases apix_vector_t *vecp; 907ff178cdSJimmy Vetayases uchar_t vector; 917ff178cdSJimmy Vetayases int cpun; 927ff178cdSJimmy Vetayases int nproc; 937ff178cdSJimmy Vetayases 947ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(0); 957ff178cdSJimmy Vetayases 967ff178cdSJimmy Vetayases vector = apix_get_avail_vector_oncpu(0, APIX_IPI_MIN, APIX_IPI_MAX); 977ff178cdSJimmy Vetayases if (vector == 0) { 987ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(0); 997ff178cdSJimmy Vetayases cmn_err(CE_WARN, "apix: no available IPI\n"); 1007ff178cdSJimmy Vetayases apic_error |= APIC_ERR_GET_IPIVECT_FAIL; 1017ff178cdSJimmy Vetayases return (0); 1027ff178cdSJimmy Vetayases } 1037ff178cdSJimmy Vetayases 1047ff178cdSJimmy Vetayases nproc = max(apic_nproc, apic_max_nproc); 1057ff178cdSJimmy Vetayases for (cpun = 0; cpun < nproc; cpun++) { 1067ff178cdSJimmy Vetayases vecp = xv_vector(cpun, vector); 1077ff178cdSJimmy Vetayases if (vecp == NULL) { 1087ff178cdSJimmy Vetayases vecp = kmem_zalloc(sizeof (apix_vector_t), KM_NOSLEEP); 1097ff178cdSJimmy Vetayases if (vecp == NULL) { 1107ff178cdSJimmy Vetayases cmn_err(CE_WARN, "apix: No memory for ipi"); 1117ff178cdSJimmy Vetayases goto fail; 1127ff178cdSJimmy Vetayases } 1137ff178cdSJimmy Vetayases xv_vector(cpun, vector) = vecp; 1147ff178cdSJimmy Vetayases } 1157ff178cdSJimmy Vetayases vecp->v_state = APIX_STATE_ALLOCED; 1167ff178cdSJimmy Vetayases vecp->v_type = APIX_TYPE_IPI; 1177ff178cdSJimmy Vetayases vecp->v_cpuid = vecp->v_bound_cpuid = cpun; 1187ff178cdSJimmy Vetayases vecp->v_vector = vector; 1197ff178cdSJimmy Vetayases vecp->v_pri = ipl; 1207ff178cdSJimmy Vetayases } 1217ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(0); 1227ff178cdSJimmy Vetayases return (vector); 1237ff178cdSJimmy Vetayases 1247ff178cdSJimmy Vetayases fail: 1257ff178cdSJimmy Vetayases while (--cpun >= 0) 1267ff178cdSJimmy Vetayases apix_cleanup_vector(xv_vector(cpun, vector)); 1277ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(0); 1287ff178cdSJimmy Vetayases return (0); 1297ff178cdSJimmy Vetayases } 1307ff178cdSJimmy Vetayases 1317ff178cdSJimmy Vetayases /* 1327ff178cdSJimmy Vetayases * Add IPI service routine 1337ff178cdSJimmy Vetayases */ 1347ff178cdSJimmy Vetayases static int 1357ff178cdSJimmy Vetayases apix_add_ipi(int ipl, avfunc xxintr, char *name, int vector, 1367ff178cdSJimmy Vetayases caddr_t arg1, caddr_t arg2) 1377ff178cdSJimmy Vetayases { 1387ff178cdSJimmy Vetayases int cpun; 1397ff178cdSJimmy Vetayases apix_vector_t *vecp; 1407ff178cdSJimmy Vetayases int nproc; 1417ff178cdSJimmy Vetayases 1427ff178cdSJimmy Vetayases ASSERT(vector >= APIX_IPI_MIN && vector <= APIX_IPI_MAX); 1437ff178cdSJimmy Vetayases 1447ff178cdSJimmy Vetayases nproc = max(apic_nproc, apic_max_nproc); 1457ff178cdSJimmy Vetayases for (cpun = 0; cpun < nproc; cpun++) { 1467ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(cpun); 1477ff178cdSJimmy Vetayases vecp = xv_vector(cpun, vector); 1487ff178cdSJimmy Vetayases apix_insert_av(vecp, NULL, xxintr, arg1, arg2, NULL, ipl, NULL); 1497ff178cdSJimmy Vetayases vecp->v_state = APIX_STATE_ENABLED; 1507ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpun); 1517ff178cdSJimmy Vetayases } 1527ff178cdSJimmy Vetayases 1537ff178cdSJimmy Vetayases APIC_VERBOSE(IPI, (CE_CONT, "apix: add ipi for %s, vector %x " 1547ff178cdSJimmy Vetayases "ipl %x\n", name, vector, ipl)); 1557ff178cdSJimmy Vetayases 1567ff178cdSJimmy Vetayases return (1); 1577ff178cdSJimmy Vetayases } 1587ff178cdSJimmy Vetayases 1597ff178cdSJimmy Vetayases /* 1607ff178cdSJimmy Vetayases * Find and return first free vector in range (start, end) 1617ff178cdSJimmy Vetayases */ 1627ff178cdSJimmy Vetayases static int 1637ff178cdSJimmy Vetayases apix_get_avail_vector_oncpu(uint32_t cpuid, int start, int end) 1647ff178cdSJimmy Vetayases { 1657ff178cdSJimmy Vetayases int i; 1667ff178cdSJimmy Vetayases apix_impl_t *apixp = apixs[cpuid]; 1677ff178cdSJimmy Vetayases 1687ff178cdSJimmy Vetayases for (i = start; i <= end; i++) { 1697ff178cdSJimmy Vetayases if (APIC_CHECK_RESERVE_VECTORS(i)) 1707ff178cdSJimmy Vetayases continue; 1717ff178cdSJimmy Vetayases if (IS_VECT_FREE(apixp->x_vectbl[i])) 1727ff178cdSJimmy Vetayases return (i); 1737ff178cdSJimmy Vetayases } 1747ff178cdSJimmy Vetayases 1757ff178cdSJimmy Vetayases return (0); 1767ff178cdSJimmy Vetayases } 1777ff178cdSJimmy Vetayases 1787ff178cdSJimmy Vetayases /* 1797ff178cdSJimmy Vetayases * Allocate a vector on specified cpu 1807ff178cdSJimmy Vetayases * 1817ff178cdSJimmy Vetayases * Return NULL on error 1827ff178cdSJimmy Vetayases */ 1837ff178cdSJimmy Vetayases static apix_vector_t * 1847ff178cdSJimmy Vetayases apix_alloc_vector_oncpu(uint32_t cpuid, dev_info_t *dip, int inum, int type) 1857ff178cdSJimmy Vetayases { 1867ff178cdSJimmy Vetayases processorid_t tocpu = cpuid & ~IRQ_USER_BOUND; 1877ff178cdSJimmy Vetayases apix_vector_t *vecp; 1887ff178cdSJimmy Vetayases int vector; 1897ff178cdSJimmy Vetayases 1907ff178cdSJimmy Vetayases ASSERT(APIX_CPU_LOCK_HELD(tocpu)); 1917ff178cdSJimmy Vetayases 1927ff178cdSJimmy Vetayases /* find free vector */ 1937ff178cdSJimmy Vetayases vector = apix_get_avail_vector_oncpu(tocpu, APIX_AVINTR_MIN, 1947ff178cdSJimmy Vetayases APIX_AVINTR_MAX); 1957ff178cdSJimmy Vetayases if (vector == 0) 1967ff178cdSJimmy Vetayases return (NULL); 1977ff178cdSJimmy Vetayases 1987ff178cdSJimmy Vetayases vecp = apix_init_vector(tocpu, vector); 1997ff178cdSJimmy Vetayases vecp->v_type = (ushort_t)type; 2007ff178cdSJimmy Vetayases vecp->v_inum = inum; 2017ff178cdSJimmy Vetayases vecp->v_flags = (cpuid & IRQ_USER_BOUND) ? APIX_VECT_USER_BOUND : 0; 2027ff178cdSJimmy Vetayases 2037ff178cdSJimmy Vetayases if (dip != NULL) 2047ff178cdSJimmy Vetayases apix_set_dev_map(vecp, dip, inum); 2057ff178cdSJimmy Vetayases 2067ff178cdSJimmy Vetayases return (vecp); 2077ff178cdSJimmy Vetayases } 2087ff178cdSJimmy Vetayases 2097ff178cdSJimmy Vetayases /* 2107ff178cdSJimmy Vetayases * Allocates "count" contiguous MSI vectors starting at the proper alignment. 2117ff178cdSJimmy Vetayases * Caller needs to make sure that count has to be power of 2 and should not 2127ff178cdSJimmy Vetayases * be < 1. 2137ff178cdSJimmy Vetayases * 2147ff178cdSJimmy Vetayases * Return first vector number 2157ff178cdSJimmy Vetayases */ 2167ff178cdSJimmy Vetayases apix_vector_t * 2177ff178cdSJimmy Vetayases apix_alloc_nvectors_oncpu(uint32_t cpuid, dev_info_t *dip, int inum, 2187ff178cdSJimmy Vetayases int count, int type) 2197ff178cdSJimmy Vetayases { 2207ff178cdSJimmy Vetayases int i, msibits, start = 0, navail = 0; 2217ff178cdSJimmy Vetayases apix_vector_t *vecp, *startp = NULL; 2227ff178cdSJimmy Vetayases processorid_t tocpu = cpuid & ~IRQ_USER_BOUND; 2237ff178cdSJimmy Vetayases uint_t flags; 2247ff178cdSJimmy Vetayases 2257ff178cdSJimmy Vetayases ASSERT(APIX_CPU_LOCK_HELD(tocpu)); 2267ff178cdSJimmy Vetayases 2277ff178cdSJimmy Vetayases /* 2287ff178cdSJimmy Vetayases * msibits is the no. of lower order message data bits for the 2297ff178cdSJimmy Vetayases * allocated MSI vectors and is used to calculate the aligned 2307ff178cdSJimmy Vetayases * starting vector 2317ff178cdSJimmy Vetayases */ 2327ff178cdSJimmy Vetayases msibits = count - 1; 2337ff178cdSJimmy Vetayases 2347ff178cdSJimmy Vetayases /* It has to be contiguous */ 2357ff178cdSJimmy Vetayases for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) { 2367ff178cdSJimmy Vetayases if (!IS_VECT_FREE(xv_vector(tocpu, i))) 2377ff178cdSJimmy Vetayases continue; 2387ff178cdSJimmy Vetayases 2397ff178cdSJimmy Vetayases /* 2407ff178cdSJimmy Vetayases * starting vector has to be aligned accordingly for 2417ff178cdSJimmy Vetayases * multiple MSIs 2427ff178cdSJimmy Vetayases */ 2437ff178cdSJimmy Vetayases if (msibits) 2447ff178cdSJimmy Vetayases i = (i + msibits) & ~msibits; 2457ff178cdSJimmy Vetayases 2467ff178cdSJimmy Vetayases for (navail = 0, start = i; i <= APIX_AVINTR_MAX; i++) { 2477ff178cdSJimmy Vetayases if (!IS_VECT_FREE(xv_vector(tocpu, i))) 2487ff178cdSJimmy Vetayases break; 2497ff178cdSJimmy Vetayases if (APIC_CHECK_RESERVE_VECTORS(i)) 2507ff178cdSJimmy Vetayases break; 2517ff178cdSJimmy Vetayases if (++navail == count) 2527ff178cdSJimmy Vetayases goto done; 2537ff178cdSJimmy Vetayases } 2547ff178cdSJimmy Vetayases } 2557ff178cdSJimmy Vetayases 2567ff178cdSJimmy Vetayases return (NULL); 2577ff178cdSJimmy Vetayases 2587ff178cdSJimmy Vetayases done: 2597ff178cdSJimmy Vetayases flags = (cpuid & IRQ_USER_BOUND) ? APIX_VECT_USER_BOUND : 0; 2607ff178cdSJimmy Vetayases 2617ff178cdSJimmy Vetayases for (i = 0; i < count; i++) { 2627ff178cdSJimmy Vetayases if ((vecp = apix_init_vector(tocpu, start + i)) == NULL) 2637ff178cdSJimmy Vetayases goto fail; 2647ff178cdSJimmy Vetayases 2657ff178cdSJimmy Vetayases vecp->v_type = (ushort_t)type; 2667ff178cdSJimmy Vetayases vecp->v_inum = inum + i; 2677ff178cdSJimmy Vetayases vecp->v_flags = flags; 2687ff178cdSJimmy Vetayases 2697ff178cdSJimmy Vetayases if (dip != NULL) 2707ff178cdSJimmy Vetayases apix_set_dev_map(vecp, dip, inum + i); 2717ff178cdSJimmy Vetayases 2727ff178cdSJimmy Vetayases if (i == 0) 2737ff178cdSJimmy Vetayases startp = vecp; 2747ff178cdSJimmy Vetayases } 2757ff178cdSJimmy Vetayases 2767ff178cdSJimmy Vetayases return (startp); 2777ff178cdSJimmy Vetayases 2787ff178cdSJimmy Vetayases fail: 2797ff178cdSJimmy Vetayases while (i-- > 0) { /* Free allocated vectors */ 2807ff178cdSJimmy Vetayases vecp = xv_vector(tocpu, start + i); 2817ff178cdSJimmy Vetayases apix_clear_dev_map(dip, inum + i, type); 2827ff178cdSJimmy Vetayases apix_cleanup_vector(vecp); 2837ff178cdSJimmy Vetayases } 2847ff178cdSJimmy Vetayases return (NULL); 2857ff178cdSJimmy Vetayases } 2867ff178cdSJimmy Vetayases 2877ff178cdSJimmy Vetayases #define APIX_WRITE_MSI_DATA(_hdl, _cap, _ctrl, _v)\ 2887ff178cdSJimmy Vetayases do {\ 2897ff178cdSJimmy Vetayases if ((_ctrl) & PCI_MSI_64BIT_MASK)\ 2907ff178cdSJimmy Vetayases pci_config_put16((_hdl), (_cap) + PCI_MSI_64BIT_DATA, (_v));\ 2917ff178cdSJimmy Vetayases else\ 2927ff178cdSJimmy Vetayases pci_config_put16((_hdl), (_cap) + PCI_MSI_32BIT_DATA, (_v));\ 2937ff178cdSJimmy Vetayases _NOTE(CONSTCOND)} while (0) 2947ff178cdSJimmy Vetayases 2957ff178cdSJimmy Vetayases static void 2967ff178cdSJimmy Vetayases apix_pci_msi_enable_vector(apix_vector_t *vecp, dev_info_t *dip, int type, 2977ff178cdSJimmy Vetayases int inum, int count, uchar_t vector, int target_apic_id) 2987ff178cdSJimmy Vetayases { 2997ff178cdSJimmy Vetayases uint64_t msi_addr, msi_data; 3007ff178cdSJimmy Vetayases ushort_t msi_ctrl; 3017ff178cdSJimmy Vetayases int i, cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip); 3027ff178cdSJimmy Vetayases ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(dip); 3037ff178cdSJimmy Vetayases msi_regs_t msi_regs; 3047ff178cdSJimmy Vetayases void *intrmap_tbl[PCI_MSI_MAX_INTRS]; 3057ff178cdSJimmy Vetayases 3067ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apix_pci_msi_enable_vector: dip=0x%p\n" 3077ff178cdSJimmy Vetayases "\tdriver = %s, inum=0x%x vector=0x%x apicid=0x%x\n", (void *)dip, 3087ff178cdSJimmy Vetayases ddi_driver_name(dip), inum, vector, target_apic_id)); 3097ff178cdSJimmy Vetayases 3107ff178cdSJimmy Vetayases ASSERT((handle != NULL) && (cap_ptr != 0)); 3117ff178cdSJimmy Vetayases 3127ff178cdSJimmy Vetayases msi_regs.mr_data = vector; 3137ff178cdSJimmy Vetayases msi_regs.mr_addr = target_apic_id; 3147ff178cdSJimmy Vetayases 3152edb3dccSJudy Chen for (i = 0; i < count; i++) 3162edb3dccSJudy Chen intrmap_tbl[i] = xv_intrmap_private(vecp->v_cpuid, vector + i); 3177ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_alloc_entry(intrmap_tbl, dip, type, 3187ff178cdSJimmy Vetayases count, 0xff); 3197ff178cdSJimmy Vetayases for (i = 0; i < count; i++) 3207ff178cdSJimmy Vetayases xv_intrmap_private(vecp->v_cpuid, vector + i) = intrmap_tbl[i]; 3217ff178cdSJimmy Vetayases 3227ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_map_entry(vecp->v_intrmap_private, 3237ff178cdSJimmy Vetayases (void *)&msi_regs, type, count); 3247ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_record_msi(vecp->v_intrmap_private, 3257ff178cdSJimmy Vetayases &msi_regs); 3267ff178cdSJimmy Vetayases 3277ff178cdSJimmy Vetayases /* MSI Address */ 3287ff178cdSJimmy Vetayases msi_addr = msi_regs.mr_addr; 3297ff178cdSJimmy Vetayases 3307ff178cdSJimmy Vetayases /* MSI Data: MSI is edge triggered according to spec */ 3317ff178cdSJimmy Vetayases msi_data = msi_regs.mr_data; 3327ff178cdSJimmy Vetayases 3337ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apix_pci_msi_enable_vector: addr=0x%lx " 3347ff178cdSJimmy Vetayases "data=0x%lx\n", (long)msi_addr, (long)msi_data)); 3357ff178cdSJimmy Vetayases 3367ff178cdSJimmy Vetayases if (type == APIX_TYPE_MSI) { 3377ff178cdSJimmy Vetayases msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL); 3387ff178cdSJimmy Vetayases 3397ff178cdSJimmy Vetayases /* Set the bits to inform how many MSIs are enabled */ 3407ff178cdSJimmy Vetayases msi_ctrl |= ((highbit(count) - 1) << PCI_MSI_MME_SHIFT); 3417ff178cdSJimmy Vetayases pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl); 3427ff178cdSJimmy Vetayases 3437ff178cdSJimmy Vetayases if ((vecp->v_flags & APIX_VECT_MASKABLE) == 0) 3447ff178cdSJimmy Vetayases APIX_WRITE_MSI_DATA(handle, cap_ptr, msi_ctrl, 3457ff178cdSJimmy Vetayases APIX_RESV_VECTOR); 3467ff178cdSJimmy Vetayases 3477ff178cdSJimmy Vetayases pci_config_put32(handle, 3487ff178cdSJimmy Vetayases cap_ptr + PCI_MSI_ADDR_OFFSET, msi_addr); 3497ff178cdSJimmy Vetayases if (msi_ctrl & PCI_MSI_64BIT_MASK) 3507ff178cdSJimmy Vetayases pci_config_put32(handle, 3517ff178cdSJimmy Vetayases cap_ptr + PCI_MSI_ADDR_OFFSET + 4, msi_addr >> 32); 3527ff178cdSJimmy Vetayases 3537ff178cdSJimmy Vetayases APIX_WRITE_MSI_DATA(handle, cap_ptr, msi_ctrl, msi_data); 3547ff178cdSJimmy Vetayases } else if (type == APIX_TYPE_MSIX) { 3557ff178cdSJimmy Vetayases uintptr_t off; 3567ff178cdSJimmy Vetayases ddi_intr_msix_t *msix_p = i_ddi_get_msix(dip); 3577ff178cdSJimmy Vetayases 3587ff178cdSJimmy Vetayases /* Offset into the "inum"th entry in the MSI-X table */ 3597ff178cdSJimmy Vetayases off = (uintptr_t)msix_p->msix_tbl_addr + 3607ff178cdSJimmy Vetayases (inum * PCI_MSIX_VECTOR_SIZE); 3617ff178cdSJimmy Vetayases 3627ff178cdSJimmy Vetayases ddi_put32(msix_p->msix_tbl_hdl, 3637ff178cdSJimmy Vetayases (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), msi_data); 3647ff178cdSJimmy Vetayases ddi_put64(msix_p->msix_tbl_hdl, 3657ff178cdSJimmy Vetayases (uint64_t *)(off + PCI_MSIX_LOWER_ADDR_OFFSET), msi_addr); 3667ff178cdSJimmy Vetayases } 3677ff178cdSJimmy Vetayases } 3687ff178cdSJimmy Vetayases 3697ff178cdSJimmy Vetayases static void 3707ff178cdSJimmy Vetayases apix_pci_msi_enable_mode(dev_info_t *dip, int type, int inum) 3717ff178cdSJimmy Vetayases { 3727ff178cdSJimmy Vetayases ushort_t msi_ctrl; 3737ff178cdSJimmy Vetayases int cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip); 3747ff178cdSJimmy Vetayases ddi_acc_handle_t handle = i_ddi_get_pci_config_handle(dip); 3757ff178cdSJimmy Vetayases 3767ff178cdSJimmy Vetayases ASSERT((handle != NULL) && (cap_ptr != 0)); 3777ff178cdSJimmy Vetayases 3787ff178cdSJimmy Vetayases if (type == APIX_TYPE_MSI) { 3797ff178cdSJimmy Vetayases msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL); 3807ff178cdSJimmy Vetayases if ((msi_ctrl & PCI_MSI_ENABLE_BIT)) 3817ff178cdSJimmy Vetayases return; 3827ff178cdSJimmy Vetayases 3837ff178cdSJimmy Vetayases msi_ctrl |= PCI_MSI_ENABLE_BIT; 3847ff178cdSJimmy Vetayases pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl); 3857ff178cdSJimmy Vetayases 3867ff178cdSJimmy Vetayases } else if (type == DDI_INTR_TYPE_MSIX) { 3877ff178cdSJimmy Vetayases uintptr_t off; 3887ff178cdSJimmy Vetayases uint32_t mask; 3897ff178cdSJimmy Vetayases ddi_intr_msix_t *msix_p; 3907ff178cdSJimmy Vetayases 3917ff178cdSJimmy Vetayases msix_p = i_ddi_get_msix(dip); 3927ff178cdSJimmy Vetayases 3937ff178cdSJimmy Vetayases /* Offset into "inum"th entry in the MSI-X table & clear mask */ 3947ff178cdSJimmy Vetayases off = (uintptr_t)msix_p->msix_tbl_addr + (inum * 3957ff178cdSJimmy Vetayases PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET; 3967ff178cdSJimmy Vetayases 3977ff178cdSJimmy Vetayases mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off); 3987ff178cdSJimmy Vetayases 3997ff178cdSJimmy Vetayases ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, (mask & ~1)); 4007ff178cdSJimmy Vetayases 4017ff178cdSJimmy Vetayases msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL); 4027ff178cdSJimmy Vetayases 4037ff178cdSJimmy Vetayases if (!(msi_ctrl & PCI_MSIX_ENABLE_BIT)) { 4047ff178cdSJimmy Vetayases msi_ctrl |= PCI_MSIX_ENABLE_BIT; 4057ff178cdSJimmy Vetayases pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL, 4067ff178cdSJimmy Vetayases msi_ctrl); 4077ff178cdSJimmy Vetayases } 4087ff178cdSJimmy Vetayases } 4097ff178cdSJimmy Vetayases } 4107ff178cdSJimmy Vetayases 4117ff178cdSJimmy Vetayases /* 4127ff178cdSJimmy Vetayases * Setup interrupt, pogramming IO-APIC or MSI/X address/data. 4137ff178cdSJimmy Vetayases */ 4147ff178cdSJimmy Vetayases void 4157ff178cdSJimmy Vetayases apix_enable_vector(apix_vector_t *vecp) 4167ff178cdSJimmy Vetayases { 4177ff178cdSJimmy Vetayases int tocpu = vecp->v_cpuid, type = vecp->v_type; 4187ff178cdSJimmy Vetayases apic_cpus_info_t *cpu_infop; 4197ff178cdSJimmy Vetayases ulong_t iflag; 4207ff178cdSJimmy Vetayases 4217ff178cdSJimmy Vetayases ASSERT(tocpu < apic_nproc); 4227ff178cdSJimmy Vetayases 4237ff178cdSJimmy Vetayases cpu_infop = &apic_cpus[tocpu]; 4247ff178cdSJimmy Vetayases if (vecp->v_flags & APIX_VECT_USER_BOUND) 4257ff178cdSJimmy Vetayases cpu_infop->aci_bound++; 4267ff178cdSJimmy Vetayases else 4277ff178cdSJimmy Vetayases cpu_infop->aci_temp_bound++; 4287ff178cdSJimmy Vetayases 4297ff178cdSJimmy Vetayases iflag = intr_clear(); 4307ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock); 4317ff178cdSJimmy Vetayases 4327ff178cdSJimmy Vetayases if (!DDI_INTR_IS_MSI_OR_MSIX(type)) { /* fixed */ 4337ff178cdSJimmy Vetayases apix_intx_enable(vecp->v_inum); 4347ff178cdSJimmy Vetayases } else { 4357ff178cdSJimmy Vetayases int inum = vecp->v_inum; 4367ff178cdSJimmy Vetayases dev_info_t *dip = APIX_GET_DIP(vecp); 4377ff178cdSJimmy Vetayases int count = i_ddi_intr_get_current_nintrs(dip); 4387ff178cdSJimmy Vetayases 4397ff178cdSJimmy Vetayases if (type == APIX_TYPE_MSI) { /* MSI */ 4407ff178cdSJimmy Vetayases if (inum == apix_get_max_dev_inum(dip, type)) { 4417ff178cdSJimmy Vetayases /* last one */ 4427ff178cdSJimmy Vetayases uchar_t start_inum = inum + 1 - count; 4437ff178cdSJimmy Vetayases uchar_t start_vect = vecp->v_vector + 1 - count; 4447ff178cdSJimmy Vetayases apix_vector_t *start_vecp = 4457ff178cdSJimmy Vetayases xv_vector(vecp->v_cpuid, start_vect); 4467ff178cdSJimmy Vetayases 4477ff178cdSJimmy Vetayases APIC_VERBOSE(INTR, (CE_CONT, "apix: call " 4487ff178cdSJimmy Vetayases "apix_pci_msi_enable_vector\n")); 4497ff178cdSJimmy Vetayases apix_pci_msi_enable_vector(start_vecp, dip, 4507ff178cdSJimmy Vetayases type, start_inum, count, start_vect, 4517ff178cdSJimmy Vetayases cpu_infop->aci_local_id); 4527ff178cdSJimmy Vetayases 4537ff178cdSJimmy Vetayases APIC_VERBOSE(INTR, (CE_CONT, "apix: call " 4547ff178cdSJimmy Vetayases "apix_pci_msi_enable_mode\n")); 4557ff178cdSJimmy Vetayases apix_pci_msi_enable_mode(dip, type, inum); 4567ff178cdSJimmy Vetayases } 4577ff178cdSJimmy Vetayases } else { /* MSI-X */ 4587ff178cdSJimmy Vetayases apix_pci_msi_enable_vector(vecp, dip, 4597ff178cdSJimmy Vetayases type, inum, 1, vecp->v_vector, 4607ff178cdSJimmy Vetayases cpu_infop->aci_local_id); 4617ff178cdSJimmy Vetayases apix_pci_msi_enable_mode(dip, type, inum); 4627ff178cdSJimmy Vetayases } 4637ff178cdSJimmy Vetayases } 4647ff178cdSJimmy Vetayases vecp->v_state = APIX_STATE_ENABLED; 4657ff178cdSJimmy Vetayases apic_redist_cpu_skip &= ~(1 << tocpu); 4667ff178cdSJimmy Vetayases 4677ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock); 4687ff178cdSJimmy Vetayases intr_restore(iflag); 4697ff178cdSJimmy Vetayases } 4707ff178cdSJimmy Vetayases 4717ff178cdSJimmy Vetayases /* 4727ff178cdSJimmy Vetayases * Disable the interrupt 4737ff178cdSJimmy Vetayases */ 4747ff178cdSJimmy Vetayases void 4757ff178cdSJimmy Vetayases apix_disable_vector(apix_vector_t *vecp) 4767ff178cdSJimmy Vetayases { 4777ff178cdSJimmy Vetayases struct autovec *avp = vecp->v_autovect; 4787ff178cdSJimmy Vetayases ulong_t iflag; 4797ff178cdSJimmy Vetayases 4807ff178cdSJimmy Vetayases ASSERT(avp != NULL); 4817ff178cdSJimmy Vetayases 4827ff178cdSJimmy Vetayases iflag = intr_clear(); 4837ff178cdSJimmy Vetayases lock_set(&apic_ioapic_lock); 4847ff178cdSJimmy Vetayases 4857ff178cdSJimmy Vetayases switch (vecp->v_type) { 4867ff178cdSJimmy Vetayases case APIX_TYPE_MSI: 4877ff178cdSJimmy Vetayases ASSERT(avp->av_vector != NULL && avp->av_dip != NULL); 4887ff178cdSJimmy Vetayases /* 4897ff178cdSJimmy Vetayases * Disable the MSI vector 4907ff178cdSJimmy Vetayases * Make sure we only disable on the last 4917ff178cdSJimmy Vetayases * of the multi-MSI support 4927ff178cdSJimmy Vetayases */ 4937ff178cdSJimmy Vetayases if (i_ddi_intr_get_current_nenables(avp->av_dip) == 1) { 4947ff178cdSJimmy Vetayases apic_pci_msi_disable_mode(avp->av_dip, 4957ff178cdSJimmy Vetayases DDI_INTR_TYPE_MSI); 4967ff178cdSJimmy Vetayases } 4977ff178cdSJimmy Vetayases break; 4987ff178cdSJimmy Vetayases case APIX_TYPE_MSIX: 4997ff178cdSJimmy Vetayases ASSERT(avp->av_vector != NULL && avp->av_dip != NULL); 5007ff178cdSJimmy Vetayases /* 5017ff178cdSJimmy Vetayases * Disable the MSI-X vector 5027ff178cdSJimmy Vetayases * needs to clear its mask and addr/data for each MSI-X 5037ff178cdSJimmy Vetayases */ 5047ff178cdSJimmy Vetayases apic_pci_msi_unconfigure(avp->av_dip, DDI_INTR_TYPE_MSIX, 5057ff178cdSJimmy Vetayases vecp->v_inum); 5067ff178cdSJimmy Vetayases /* 5077ff178cdSJimmy Vetayases * Make sure we only disable on the last MSI-X 5087ff178cdSJimmy Vetayases */ 5097ff178cdSJimmy Vetayases if (i_ddi_intr_get_current_nenables(avp->av_dip) == 1) { 5107ff178cdSJimmy Vetayases apic_pci_msi_disable_mode(avp->av_dip, 5117ff178cdSJimmy Vetayases DDI_INTR_TYPE_MSIX); 5127ff178cdSJimmy Vetayases } 5137ff178cdSJimmy Vetayases break; 5147ff178cdSJimmy Vetayases default: 5157ff178cdSJimmy Vetayases apix_intx_disable(vecp->v_inum); 5167ff178cdSJimmy Vetayases break; 5177ff178cdSJimmy Vetayases } 5187ff178cdSJimmy Vetayases 5197ff178cdSJimmy Vetayases if (!(apic_cpus[vecp->v_cpuid].aci_status & APIC_CPU_SUSPEND)) 5207ff178cdSJimmy Vetayases vecp->v_state = APIX_STATE_DISABLED; 5217ff178cdSJimmy Vetayases apic_vt_ops->apic_intrmap_free_entry(&vecp->v_intrmap_private); 5227ff178cdSJimmy Vetayases vecp->v_intrmap_private = NULL; 5237ff178cdSJimmy Vetayases 5247ff178cdSJimmy Vetayases lock_clear(&apic_ioapic_lock); 5257ff178cdSJimmy Vetayases intr_restore(iflag); 5267ff178cdSJimmy Vetayases } 5277ff178cdSJimmy Vetayases 5287ff178cdSJimmy Vetayases /* 5297ff178cdSJimmy Vetayases * Mark vector as obsoleted or freed. The vector is marked 5307ff178cdSJimmy Vetayases * obsoleted if there are pending requests on it. Otherwise, 5317ff178cdSJimmy Vetayases * free the vector. The obsoleted vectors get freed after 5327ff178cdSJimmy Vetayases * being serviced. 5337ff178cdSJimmy Vetayases * 5347ff178cdSJimmy Vetayases * Return 1 on being obosoleted and 0 on being freed. 5357ff178cdSJimmy Vetayases */ 5367ff178cdSJimmy Vetayases #define INTR_BUSY(_avp)\ 5377ff178cdSJimmy Vetayases ((((volatile ushort_t)(_avp)->av_flags) &\ 5387ff178cdSJimmy Vetayases (AV_PENTRY_PEND | AV_PENTRY_ONPROC)) != 0) 5397ff178cdSJimmy Vetayases #define LOCAL_WITH_INTR_DISABLED(_cpuid)\ 5407ff178cdSJimmy Vetayases ((_cpuid) == psm_get_cpu_id() && !interrupts_enabled()) 5417ff178cdSJimmy Vetayases static uint64_t dummy_tick; 5427ff178cdSJimmy Vetayases 5437ff178cdSJimmy Vetayases int 5447ff178cdSJimmy Vetayases apix_obsolete_vector(apix_vector_t *vecp) 5457ff178cdSJimmy Vetayases { 5467ff178cdSJimmy Vetayases struct autovec *avp = vecp->v_autovect; 5477ff178cdSJimmy Vetayases int repeats, tries, ipl, busy = 0, cpuid = vecp->v_cpuid; 5487ff178cdSJimmy Vetayases apix_impl_t *apixp = apixs[cpuid]; 5497ff178cdSJimmy Vetayases 5507ff178cdSJimmy Vetayases ASSERT(APIX_CPU_LOCK_HELD(cpuid)); 5517ff178cdSJimmy Vetayases 5527ff178cdSJimmy Vetayases for (avp = vecp->v_autovect; avp != NULL; avp = avp->av_link) { 5537ff178cdSJimmy Vetayases if (avp->av_vector == NULL) 5547ff178cdSJimmy Vetayases continue; 5557ff178cdSJimmy Vetayases 5567ff178cdSJimmy Vetayases if (LOCAL_WITH_INTR_DISABLED(cpuid)) { 5577ff178cdSJimmy Vetayases int bit, index, irr; 5587ff178cdSJimmy Vetayases 5597ff178cdSJimmy Vetayases if (INTR_BUSY(avp)) { 5607ff178cdSJimmy Vetayases busy++; 5617ff178cdSJimmy Vetayases continue; 5627ff178cdSJimmy Vetayases } 5637ff178cdSJimmy Vetayases 5647ff178cdSJimmy Vetayases /* check IRR for pending interrupts */ 5657ff178cdSJimmy Vetayases index = vecp->v_vector / 32; 5667ff178cdSJimmy Vetayases bit = vecp->v_vector % 32; 5677ff178cdSJimmy Vetayases irr = apic_reg_ops->apic_read(APIC_IRR_REG + index); 5687ff178cdSJimmy Vetayases if ((irr & (1 << bit)) != 0) 5697ff178cdSJimmy Vetayases busy++; 5707ff178cdSJimmy Vetayases 5717ff178cdSJimmy Vetayases if (!busy) 5727ff178cdSJimmy Vetayases apix_remove_av(vecp, avp); 5737ff178cdSJimmy Vetayases 5747ff178cdSJimmy Vetayases continue; 5757ff178cdSJimmy Vetayases } 5767ff178cdSJimmy Vetayases 5777ff178cdSJimmy Vetayases repeats = 0; 5787ff178cdSJimmy Vetayases do { 5797ff178cdSJimmy Vetayases repeats++; 5807ff178cdSJimmy Vetayases for (tries = 0; tries < apic_max_reps_clear_pending; 5817ff178cdSJimmy Vetayases tries++) 5827ff178cdSJimmy Vetayases if (!INTR_BUSY(avp)) 5837ff178cdSJimmy Vetayases break; 5847ff178cdSJimmy Vetayases } while (INTR_BUSY(avp) && 5857ff178cdSJimmy Vetayases (repeats < apic_max_reps_clear_pending)); 5867ff178cdSJimmy Vetayases 5877ff178cdSJimmy Vetayases if (INTR_BUSY(avp)) 5887ff178cdSJimmy Vetayases busy++; 5897ff178cdSJimmy Vetayases else { 5907ff178cdSJimmy Vetayases /* 5917ff178cdSJimmy Vetayases * Interrupt is not in pending list or being serviced. 5927ff178cdSJimmy Vetayases * However it might be cached in Local APIC's IRR 5937ff178cdSJimmy Vetayases * register. It's impossible to check another CPU's 5947ff178cdSJimmy Vetayases * IRR register. Then wait till lower levels finish 5957ff178cdSJimmy Vetayases * running. 5967ff178cdSJimmy Vetayases */ 5977ff178cdSJimmy Vetayases for (ipl = 1; ipl < MIN(LOCK_LEVEL, vecp->v_pri); ipl++) 5987ff178cdSJimmy Vetayases apix_wait_till_seen(cpuid, ipl); 5997ff178cdSJimmy Vetayases if (INTR_BUSY(avp)) 6007ff178cdSJimmy Vetayases busy++; 6017ff178cdSJimmy Vetayases } 6027ff178cdSJimmy Vetayases 6037ff178cdSJimmy Vetayases if (!busy) 6047ff178cdSJimmy Vetayases apix_remove_av(vecp, avp); 6057ff178cdSJimmy Vetayases } 6067ff178cdSJimmy Vetayases 6077ff178cdSJimmy Vetayases if (busy) { 6087ff178cdSJimmy Vetayases apix_vector_t *tp = apixp->x_obsoletes; 6097ff178cdSJimmy Vetayases 6107ff178cdSJimmy Vetayases if (vecp->v_state == APIX_STATE_OBSOLETED) 6117ff178cdSJimmy Vetayases return (1); 6127ff178cdSJimmy Vetayases 6137ff178cdSJimmy Vetayases vecp->v_state = APIX_STATE_OBSOLETED; 6147ff178cdSJimmy Vetayases vecp->v_next = NULL; 6157ff178cdSJimmy Vetayases if (tp == NULL) 6167ff178cdSJimmy Vetayases apixp->x_obsoletes = vecp; 6177ff178cdSJimmy Vetayases else { 6187ff178cdSJimmy Vetayases while (tp->v_next != NULL) 6197ff178cdSJimmy Vetayases tp = tp->v_next; 6207ff178cdSJimmy Vetayases tp->v_next = vecp; 6217ff178cdSJimmy Vetayases } 6227ff178cdSJimmy Vetayases return (1); 6237ff178cdSJimmy Vetayases } 6247ff178cdSJimmy Vetayases 6257ff178cdSJimmy Vetayases /* interrupt is not busy */ 6267ff178cdSJimmy Vetayases if (vecp->v_state == APIX_STATE_OBSOLETED) { 6277ff178cdSJimmy Vetayases /* remove from obsoleted list */ 6287ff178cdSJimmy Vetayases apixp->x_obsoletes = vecp->v_next; 6297ff178cdSJimmy Vetayases vecp->v_next = NULL; 6307ff178cdSJimmy Vetayases } 6317ff178cdSJimmy Vetayases apix_cleanup_vector(vecp); 6327ff178cdSJimmy Vetayases return (0); 6337ff178cdSJimmy Vetayases } 6347ff178cdSJimmy Vetayases 6357ff178cdSJimmy Vetayases /* 6367ff178cdSJimmy Vetayases * Duplicate number of continuous vectors to specified target vectors. 6377ff178cdSJimmy Vetayases */ 6387ff178cdSJimmy Vetayases static void 6397ff178cdSJimmy Vetayases apix_dup_vectors(apix_vector_t *oldp, apix_vector_t *newp, int count) 6407ff178cdSJimmy Vetayases { 6417ff178cdSJimmy Vetayases struct autovec *avp; 6427ff178cdSJimmy Vetayases apix_vector_t *fromp, *top; 6437ff178cdSJimmy Vetayases processorid_t oldcpu = oldp->v_cpuid, newcpu = newp->v_cpuid; 6447ff178cdSJimmy Vetayases uchar_t oldvec = oldp->v_vector, newvec = newp->v_vector; 6457ff178cdSJimmy Vetayases int i, inum; 6467ff178cdSJimmy Vetayases 6477ff178cdSJimmy Vetayases ASSERT(oldp->v_type != APIX_TYPE_IPI); 6487ff178cdSJimmy Vetayases 6497ff178cdSJimmy Vetayases for (i = 0; i < count; i++) { 6507ff178cdSJimmy Vetayases fromp = xv_vector(oldcpu, oldvec + i); 6517ff178cdSJimmy Vetayases top = xv_vector(newcpu, newvec + i); 6527ff178cdSJimmy Vetayases ASSERT(fromp != NULL && top != NULL); 6537ff178cdSJimmy Vetayases 6547ff178cdSJimmy Vetayases /* copy over original one */ 6557ff178cdSJimmy Vetayases top->v_state = fromp->v_state; 6567ff178cdSJimmy Vetayases top->v_type = fromp->v_type; 6577ff178cdSJimmy Vetayases top->v_bound_cpuid = fromp->v_bound_cpuid; 6587ff178cdSJimmy Vetayases top->v_inum = fromp->v_inum; 6597ff178cdSJimmy Vetayases top->v_flags = fromp->v_flags; 6607ff178cdSJimmy Vetayases top->v_intrmap_private = fromp->v_intrmap_private; 6617ff178cdSJimmy Vetayases 6627ff178cdSJimmy Vetayases for (avp = fromp->v_autovect; avp != NULL; avp = avp->av_link) { 6637ff178cdSJimmy Vetayases if (avp->av_vector == NULL) 6647ff178cdSJimmy Vetayases continue; 6657ff178cdSJimmy Vetayases 6667ff178cdSJimmy Vetayases apix_insert_av(top, avp->av_intr_id, avp->av_vector, 6677ff178cdSJimmy Vetayases avp->av_intarg1, avp->av_intarg2, avp->av_ticksp, 6687ff178cdSJimmy Vetayases avp->av_prilevel, avp->av_dip); 6697ff178cdSJimmy Vetayases 6707ff178cdSJimmy Vetayases if (fromp->v_type == APIX_TYPE_FIXED && 6717ff178cdSJimmy Vetayases avp->av_dip != NULL) { 6727ff178cdSJimmy Vetayases inum = GET_INTR_INUM(avp->av_intr_id); 6737ff178cdSJimmy Vetayases apix_set_dev_map(top, avp->av_dip, inum); 6747ff178cdSJimmy Vetayases } 6757ff178cdSJimmy Vetayases } 6767ff178cdSJimmy Vetayases 6777ff178cdSJimmy Vetayases if (DDI_INTR_IS_MSI_OR_MSIX(fromp->v_type) && 6787ff178cdSJimmy Vetayases fromp->v_devp != NULL) 6797ff178cdSJimmy Vetayases apix_set_dev_map(top, fromp->v_devp->dv_dip, 6807ff178cdSJimmy Vetayases fromp->v_devp->dv_inum); 6817ff178cdSJimmy Vetayases } 6827ff178cdSJimmy Vetayases } 6837ff178cdSJimmy Vetayases 6847ff178cdSJimmy Vetayases static apix_vector_t * 6857ff178cdSJimmy Vetayases apix_init_vector(processorid_t cpuid, uchar_t vector) 6867ff178cdSJimmy Vetayases { 6877ff178cdSJimmy Vetayases apix_impl_t *apixp = apixs[cpuid]; 6887ff178cdSJimmy Vetayases apix_vector_t *vecp = apixp->x_vectbl[vector]; 6897ff178cdSJimmy Vetayases 6907ff178cdSJimmy Vetayases ASSERT(IS_VECT_FREE(vecp)); 6917ff178cdSJimmy Vetayases 6927ff178cdSJimmy Vetayases if (vecp == NULL) { 6937ff178cdSJimmy Vetayases vecp = kmem_zalloc(sizeof (apix_vector_t), KM_NOSLEEP); 6947ff178cdSJimmy Vetayases if (vecp == NULL) { 6957ff178cdSJimmy Vetayases cmn_err(CE_WARN, "apix: no memory to allocate vector"); 6967ff178cdSJimmy Vetayases return (NULL); 6977ff178cdSJimmy Vetayases } 6987ff178cdSJimmy Vetayases apixp->x_vectbl[vector] = vecp; 6997ff178cdSJimmy Vetayases } 7007ff178cdSJimmy Vetayases vecp->v_state = APIX_STATE_ALLOCED; 7017ff178cdSJimmy Vetayases vecp->v_cpuid = vecp->v_bound_cpuid = cpuid; 7027ff178cdSJimmy Vetayases vecp->v_vector = vector; 7037ff178cdSJimmy Vetayases 7047ff178cdSJimmy Vetayases return (vecp); 7057ff178cdSJimmy Vetayases } 7067ff178cdSJimmy Vetayases 7077ff178cdSJimmy Vetayases static void 7087ff178cdSJimmy Vetayases apix_cleanup_vector(apix_vector_t *vecp) 7097ff178cdSJimmy Vetayases { 7107ff178cdSJimmy Vetayases ASSERT(vecp->v_share == 0); 7117ff178cdSJimmy Vetayases vecp->v_bound_cpuid = IRQ_UNINIT; 7127ff178cdSJimmy Vetayases vecp->v_state = APIX_STATE_FREED; 7137ff178cdSJimmy Vetayases vecp->v_type = 0; 7147ff178cdSJimmy Vetayases vecp->v_flags = 0; 7157ff178cdSJimmy Vetayases vecp->v_busy = 0; 716*1053f4b7SPrasad Singamsetty vecp->v_intrmap_private = NULL; 7177ff178cdSJimmy Vetayases } 7187ff178cdSJimmy Vetayases 7197ff178cdSJimmy Vetayases static void 7207ff178cdSJimmy Vetayases apix_dprint_vector(apix_vector_t *vecp, dev_info_t *dip, int count) 7217ff178cdSJimmy Vetayases { 7227ff178cdSJimmy Vetayases #ifdef DEBUG 7237ff178cdSJimmy Vetayases major_t major; 7247ff178cdSJimmy Vetayases char *name, *drv_name; 7257ff178cdSJimmy Vetayases int instance, len, t_len; 7267ff178cdSJimmy Vetayases char mesg[1024] = "apix: "; 7277ff178cdSJimmy Vetayases 7287ff178cdSJimmy Vetayases t_len = sizeof (mesg); 7297ff178cdSJimmy Vetayases len = strlen(mesg); 7307ff178cdSJimmy Vetayases if (dip != NULL) { 7317ff178cdSJimmy Vetayases name = ddi_get_name(dip); 7327ff178cdSJimmy Vetayases major = ddi_name_to_major(name); 7337ff178cdSJimmy Vetayases drv_name = ddi_major_to_name(major); 7347ff178cdSJimmy Vetayases instance = ddi_get_instance(dip); 7357ff178cdSJimmy Vetayases (void) snprintf(mesg + len, t_len - len, "%s (%s) instance %d ", 7367ff178cdSJimmy Vetayases name, drv_name, instance); 7377ff178cdSJimmy Vetayases } 7387ff178cdSJimmy Vetayases len = strlen(mesg); 7397ff178cdSJimmy Vetayases 7407ff178cdSJimmy Vetayases switch (vecp->v_type) { 7417ff178cdSJimmy Vetayases case APIX_TYPE_FIXED: 7427ff178cdSJimmy Vetayases (void) snprintf(mesg + len, t_len - len, "irqno %d", 7437ff178cdSJimmy Vetayases vecp->v_inum); 7447ff178cdSJimmy Vetayases break; 7457ff178cdSJimmy Vetayases case APIX_TYPE_MSI: 7467ff178cdSJimmy Vetayases (void) snprintf(mesg + len, t_len - len, 7477ff178cdSJimmy Vetayases "msi inum %d (count %d)", vecp->v_inum, count); 7487ff178cdSJimmy Vetayases break; 7497ff178cdSJimmy Vetayases case APIX_TYPE_MSIX: 7507ff178cdSJimmy Vetayases (void) snprintf(mesg + len, t_len - len, "msi-x inum %d", 7517ff178cdSJimmy Vetayases vecp->v_inum); 7527ff178cdSJimmy Vetayases break; 7537ff178cdSJimmy Vetayases default: 7547ff178cdSJimmy Vetayases break; 7557ff178cdSJimmy Vetayases 7567ff178cdSJimmy Vetayases } 7577ff178cdSJimmy Vetayases 7587ff178cdSJimmy Vetayases APIC_VERBOSE(ALLOC, (CE_CONT, "%s allocated with vector 0x%x on " 7597ff178cdSJimmy Vetayases "cpu %d\n", mesg, vecp->v_vector, vecp->v_cpuid)); 7607ff178cdSJimmy Vetayases #endif /* DEBUG */ 7617ff178cdSJimmy Vetayases } 7627ff178cdSJimmy Vetayases 7637ff178cdSJimmy Vetayases /* 7647ff178cdSJimmy Vetayases * Operations on avintr 7657ff178cdSJimmy Vetayases */ 7667ff178cdSJimmy Vetayases 7677ff178cdSJimmy Vetayases #define INIT_AUTOVEC(p, intr_id, f, arg1, arg2, ticksp, ipl, dip) \ 7687ff178cdSJimmy Vetayases do { \ 7697ff178cdSJimmy Vetayases (p)->av_intr_id = intr_id; \ 7707ff178cdSJimmy Vetayases (p)->av_vector = f; \ 7717ff178cdSJimmy Vetayases (p)->av_intarg1 = arg1; \ 7727ff178cdSJimmy Vetayases (p)->av_intarg2 = arg2; \ 7737ff178cdSJimmy Vetayases (p)->av_ticksp = ticksp; \ 7747ff178cdSJimmy Vetayases (p)->av_prilevel = ipl; \ 7757ff178cdSJimmy Vetayases (p)->av_dip = dip; \ 7767ff178cdSJimmy Vetayases (p)->av_flags = 0; \ 7777ff178cdSJimmy Vetayases _NOTE(CONSTCOND)} while (0) 7787ff178cdSJimmy Vetayases 7797ff178cdSJimmy Vetayases /* 7807ff178cdSJimmy Vetayases * Insert an interrupt service routine into chain by its priority from 7817ff178cdSJimmy Vetayases * high to low 7827ff178cdSJimmy Vetayases */ 7837ff178cdSJimmy Vetayases static void 7847ff178cdSJimmy Vetayases apix_insert_av(apix_vector_t *vecp, void *intr_id, avfunc f, caddr_t arg1, 7857ff178cdSJimmy Vetayases caddr_t arg2, uint64_t *ticksp, int ipl, dev_info_t *dip) 7867ff178cdSJimmy Vetayases { 7877ff178cdSJimmy Vetayases struct autovec *p, *prep, *mem; 7887ff178cdSJimmy Vetayases 7897ff178cdSJimmy Vetayases APIC_VERBOSE(INTR, (CE_CONT, "apix_insert_av: dip %p, vector 0x%x, " 7907ff178cdSJimmy Vetayases "cpu %d\n", (void *)dip, vecp->v_vector, vecp->v_cpuid)); 7917ff178cdSJimmy Vetayases 7927ff178cdSJimmy Vetayases mem = kmem_zalloc(sizeof (struct autovec), KM_SLEEP); 7937ff178cdSJimmy Vetayases INIT_AUTOVEC(mem, intr_id, f, arg1, arg2, ticksp, ipl, dip); 7947ff178cdSJimmy Vetayases if (vecp->v_type == APIX_TYPE_FIXED && apic_level_intr[vecp->v_inum]) 7957ff178cdSJimmy Vetayases mem->av_flags |= AV_PENTRY_LEVEL; 7967ff178cdSJimmy Vetayases 7977ff178cdSJimmy Vetayases vecp->v_share++; 7987ff178cdSJimmy Vetayases vecp->v_pri = (ipl > vecp->v_pri) ? ipl : vecp->v_pri; 7997ff178cdSJimmy Vetayases if (vecp->v_autovect == NULL) { /* Nothing on list - put it at head */ 8007ff178cdSJimmy Vetayases vecp->v_autovect = mem; 8017ff178cdSJimmy Vetayases return; 8027ff178cdSJimmy Vetayases } 8037ff178cdSJimmy Vetayases 8047ff178cdSJimmy Vetayases if (DDI_INTR_IS_MSI_OR_MSIX(vecp->v_type)) { /* MSI/X */ 8057ff178cdSJimmy Vetayases ASSERT(vecp->v_share == 1); /* No sharing for MSI/X */ 8067ff178cdSJimmy Vetayases 8077ff178cdSJimmy Vetayases INIT_AUTOVEC(vecp->v_autovect, intr_id, f, arg1, arg2, ticksp, 8087ff178cdSJimmy Vetayases ipl, dip); 8097ff178cdSJimmy Vetayases prep = vecp->v_autovect->av_link; 8107ff178cdSJimmy Vetayases vecp->v_autovect->av_link = NULL; 8117ff178cdSJimmy Vetayases 8127ff178cdSJimmy Vetayases /* Free the following autovect chain */ 8137ff178cdSJimmy Vetayases while (prep != NULL) { 8147ff178cdSJimmy Vetayases ASSERT(prep->av_vector == NULL); 8157ff178cdSJimmy Vetayases 8167ff178cdSJimmy Vetayases p = prep; 8177ff178cdSJimmy Vetayases prep = prep->av_link; 8187ff178cdSJimmy Vetayases kmem_free(p, sizeof (struct autovec)); 8197ff178cdSJimmy Vetayases } 8207ff178cdSJimmy Vetayases 8217ff178cdSJimmy Vetayases kmem_free(mem, sizeof (struct autovec)); 8227ff178cdSJimmy Vetayases return; 8237ff178cdSJimmy Vetayases } 8247ff178cdSJimmy Vetayases 8257ff178cdSJimmy Vetayases /* find where it goes in list */ 8267ff178cdSJimmy Vetayases prep = NULL; 8277ff178cdSJimmy Vetayases for (p = vecp->v_autovect; p != NULL; p = p->av_link) { 8287ff178cdSJimmy Vetayases if (p->av_vector && p->av_prilevel <= ipl) 8297ff178cdSJimmy Vetayases break; 8307ff178cdSJimmy Vetayases prep = p; 8317ff178cdSJimmy Vetayases } 8327ff178cdSJimmy Vetayases if (prep != NULL) { 8337ff178cdSJimmy Vetayases if (prep->av_vector == NULL) { /* freed struct available */ 8347ff178cdSJimmy Vetayases INIT_AUTOVEC(prep, intr_id, f, arg1, arg2, 8357ff178cdSJimmy Vetayases ticksp, ipl, dip); 8367ff178cdSJimmy Vetayases prep->av_flags = mem->av_flags; 8377ff178cdSJimmy Vetayases kmem_free(mem, sizeof (struct autovec)); 8387ff178cdSJimmy Vetayases return; 8397ff178cdSJimmy Vetayases } 8407ff178cdSJimmy Vetayases 8417ff178cdSJimmy Vetayases mem->av_link = prep->av_link; 8427ff178cdSJimmy Vetayases prep->av_link = mem; 8437ff178cdSJimmy Vetayases } else { 8447ff178cdSJimmy Vetayases /* insert new intpt at beginning of chain */ 8457ff178cdSJimmy Vetayases mem->av_link = vecp->v_autovect; 8467ff178cdSJimmy Vetayases vecp->v_autovect = mem; 8477ff178cdSJimmy Vetayases } 8487ff178cdSJimmy Vetayases } 8497ff178cdSJimmy Vetayases 8507ff178cdSJimmy Vetayases /* 8517ff178cdSJimmy Vetayases * After having made a change to an autovector list, wait until we have 8527ff178cdSJimmy Vetayases * seen specified cpu not executing an interrupt at that level--so we 8537ff178cdSJimmy Vetayases * know our change has taken effect completely (no old state in registers, 8547ff178cdSJimmy Vetayases * etc). 8557ff178cdSJimmy Vetayases */ 8567ff178cdSJimmy Vetayases #define APIX_CPU_ENABLED(_cp) \ 8577ff178cdSJimmy Vetayases (quiesce_active == 0 && \ 8587ff178cdSJimmy Vetayases (((_cp)->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) == 0)) 8597ff178cdSJimmy Vetayases 8607ff178cdSJimmy Vetayases static void 8617ff178cdSJimmy Vetayases apix_wait_till_seen(processorid_t cpuid, int ipl) 8627ff178cdSJimmy Vetayases { 8637ff178cdSJimmy Vetayases struct cpu *cp = cpu[cpuid]; 8647ff178cdSJimmy Vetayases 8657ff178cdSJimmy Vetayases if (cp == NULL || LOCAL_WITH_INTR_DISABLED(cpuid)) 8667ff178cdSJimmy Vetayases return; 8677ff178cdSJimmy Vetayases 8687ff178cdSJimmy Vetayases /* 8697ff178cdSJimmy Vetayases * Don't wait if the CPU is quiesced or offlined. This can happen 8707ff178cdSJimmy Vetayases * when a CPU is running pause thread but hardware triggered an 8717ff178cdSJimmy Vetayases * interrupt and the interrupt gets queued. 8727ff178cdSJimmy Vetayases */ 8737ff178cdSJimmy Vetayases for (;;) { 8747ff178cdSJimmy Vetayases if (!INTR_ACTIVE((volatile struct cpu *)cpu[cpuid], ipl) && 8757ff178cdSJimmy Vetayases (!APIX_CPU_ENABLED(cp) || 8767ff178cdSJimmy Vetayases !INTR_PENDING((volatile apix_impl_t *)apixs[cpuid], ipl))) 8777ff178cdSJimmy Vetayases return; 8787ff178cdSJimmy Vetayases } 8797ff178cdSJimmy Vetayases } 8807ff178cdSJimmy Vetayases 8817ff178cdSJimmy Vetayases static void 8827ff178cdSJimmy Vetayases apix_remove_av(apix_vector_t *vecp, struct autovec *target) 8837ff178cdSJimmy Vetayases { 8847ff178cdSJimmy Vetayases int hi_pri = 0; 8857ff178cdSJimmy Vetayases struct autovec *p; 8867ff178cdSJimmy Vetayases 8877ff178cdSJimmy Vetayases if (target == NULL) 8887ff178cdSJimmy Vetayases return; 8897ff178cdSJimmy Vetayases 8907ff178cdSJimmy Vetayases APIC_VERBOSE(INTR, (CE_CONT, "apix_remove_av: dip %p, vector 0x%x, " 8917ff178cdSJimmy Vetayases "cpu %d\n", (void *)target->av_dip, vecp->v_vector, vecp->v_cpuid)); 8927ff178cdSJimmy Vetayases 8937ff178cdSJimmy Vetayases for (p = vecp->v_autovect; p; p = p->av_link) { 8947ff178cdSJimmy Vetayases if (p == target || p->av_vector == NULL) 8957ff178cdSJimmy Vetayases continue; 8967ff178cdSJimmy Vetayases hi_pri = (p->av_prilevel > hi_pri) ? p->av_prilevel : hi_pri; 8977ff178cdSJimmy Vetayases } 8987ff178cdSJimmy Vetayases 8997ff178cdSJimmy Vetayases vecp->v_share--; 9007ff178cdSJimmy Vetayases vecp->v_pri = hi_pri; 9017ff178cdSJimmy Vetayases 9027ff178cdSJimmy Vetayases /* 9037ff178cdSJimmy Vetayases * This drops the handler from the chain, it can no longer be called. 9047ff178cdSJimmy Vetayases * However, there is no guarantee that the handler is not currently 9057ff178cdSJimmy Vetayases * still executing. 9067ff178cdSJimmy Vetayases */ 9077ff178cdSJimmy Vetayases target->av_vector = NULL; 9087ff178cdSJimmy Vetayases /* 9097ff178cdSJimmy Vetayases * There is a race where we could be just about to pick up the ticksp 9107ff178cdSJimmy Vetayases * pointer to increment it after returning from the service routine 9117ff178cdSJimmy Vetayases * in av_dispatch_autovect. Rather than NULL it out let's just point 9127ff178cdSJimmy Vetayases * it off to something safe so that any final tick update attempt 9137ff178cdSJimmy Vetayases * won't fault. 9147ff178cdSJimmy Vetayases */ 9157ff178cdSJimmy Vetayases target->av_ticksp = &dummy_tick; 9167ff178cdSJimmy Vetayases apix_wait_till_seen(vecp->v_cpuid, target->av_prilevel); 9177ff178cdSJimmy Vetayases } 9187ff178cdSJimmy Vetayases 9197ff178cdSJimmy Vetayases static struct autovec * 9207ff178cdSJimmy Vetayases apix_find_av(apix_vector_t *vecp, void *intr_id, avfunc f) 9217ff178cdSJimmy Vetayases { 9227ff178cdSJimmy Vetayases struct autovec *p; 9237ff178cdSJimmy Vetayases 9247ff178cdSJimmy Vetayases for (p = vecp->v_autovect; p; p = p->av_link) { 9257ff178cdSJimmy Vetayases if ((p->av_vector == f) && (p->av_intr_id == intr_id)) { 9267ff178cdSJimmy Vetayases /* found the handler */ 9277ff178cdSJimmy Vetayases return (p); 9287ff178cdSJimmy Vetayases } 9297ff178cdSJimmy Vetayases } 9307ff178cdSJimmy Vetayases 9317ff178cdSJimmy Vetayases return (NULL); 9327ff178cdSJimmy Vetayases } 9337ff178cdSJimmy Vetayases 9347ff178cdSJimmy Vetayases static apix_vector_t * 9357ff178cdSJimmy Vetayases apix_find_vector_by_avintr(void *intr_id, avfunc f) 9367ff178cdSJimmy Vetayases { 9377ff178cdSJimmy Vetayases apix_vector_t *vecp; 9387ff178cdSJimmy Vetayases processorid_t n; 9397ff178cdSJimmy Vetayases uchar_t v; 9407ff178cdSJimmy Vetayases 9417ff178cdSJimmy Vetayases for (n = 0; n < apic_nproc; n++) { 9427ff178cdSJimmy Vetayases if (!apix_is_cpu_enabled(n)) 9437ff178cdSJimmy Vetayases continue; 9447ff178cdSJimmy Vetayases 9457ff178cdSJimmy Vetayases for (v = APIX_AVINTR_MIN; v <= APIX_AVINTR_MIN; v++) { 9467ff178cdSJimmy Vetayases vecp = xv_vector(n, v); 9477ff178cdSJimmy Vetayases if (vecp == NULL || 9487ff178cdSJimmy Vetayases vecp->v_state <= APIX_STATE_OBSOLETED) 9497ff178cdSJimmy Vetayases continue; 9507ff178cdSJimmy Vetayases 9517ff178cdSJimmy Vetayases if (apix_find_av(vecp, intr_id, f) != NULL) 9527ff178cdSJimmy Vetayases return (vecp); 9537ff178cdSJimmy Vetayases } 9547ff178cdSJimmy Vetayases } 9557ff178cdSJimmy Vetayases 9567ff178cdSJimmy Vetayases return (NULL); 9577ff178cdSJimmy Vetayases } 9587ff178cdSJimmy Vetayases 9597ff178cdSJimmy Vetayases /* 9607ff178cdSJimmy Vetayases * Add interrupt service routine. 9617ff178cdSJimmy Vetayases * 9627ff178cdSJimmy Vetayases * For legacy interrupts (HPET timer, ACPI SCI), the vector is actually 9637ff178cdSJimmy Vetayases * IRQ no. A vector is then allocated. Otherwise, the vector is already 9647ff178cdSJimmy Vetayases * allocated. The input argument virt_vect is virtual vector of format 9657ff178cdSJimmy Vetayases * APIX_VIRTVEC_VECTOR(cpuid, vector). 9667ff178cdSJimmy Vetayases * 9677ff178cdSJimmy Vetayases * Return 1 on success, 0 on failure. 9687ff178cdSJimmy Vetayases */ 9697ff178cdSJimmy Vetayases int 9707ff178cdSJimmy Vetayases apix_add_avintr(void *intr_id, int ipl, avfunc xxintr, char *name, 9717ff178cdSJimmy Vetayases int virt_vect, caddr_t arg1, caddr_t arg2, uint64_t *ticksp, 9727ff178cdSJimmy Vetayases dev_info_t *dip) 9737ff178cdSJimmy Vetayases { 9747ff178cdSJimmy Vetayases int cpuid; 9757ff178cdSJimmy Vetayases uchar_t v = (uchar_t)APIX_VIRTVEC_VECTOR(virt_vect); 9767ff178cdSJimmy Vetayases apix_vector_t *vecp; 9777ff178cdSJimmy Vetayases 9787ff178cdSJimmy Vetayases if (xxintr == NULL) { 9797ff178cdSJimmy Vetayases cmn_err(CE_WARN, "Attempt to add null for %s " 9807ff178cdSJimmy Vetayases "on vector 0x%x,0x%x", name, 9817ff178cdSJimmy Vetayases APIX_VIRTVEC_CPU(virt_vect), 9827ff178cdSJimmy Vetayases APIX_VIRTVEC_VECTOR(virt_vect)); 9837ff178cdSJimmy Vetayases return (0); 9847ff178cdSJimmy Vetayases } 9857ff178cdSJimmy Vetayases 9867ff178cdSJimmy Vetayases if (v >= APIX_IPI_MIN) /* IPIs */ 9877ff178cdSJimmy Vetayases return (apix_add_ipi(ipl, xxintr, name, v, arg1, arg2)); 9887ff178cdSJimmy Vetayases 9897ff178cdSJimmy Vetayases if (!APIX_IS_VIRTVEC(virt_vect)) { /* got irq */ 9907ff178cdSJimmy Vetayases int irqno = virt_vect; 9917ff178cdSJimmy Vetayases int inum = GET_INTR_INUM(intr_id); 9927ff178cdSJimmy Vetayases 9937ff178cdSJimmy Vetayases /* 9947ff178cdSJimmy Vetayases * Senarios include: 9957ff178cdSJimmy Vetayases * a. add_avintr() is called before irqp initialized (legacy) 9967ff178cdSJimmy Vetayases * b. irqp is initialized, vector is not allocated (fixed) 9977ff178cdSJimmy Vetayases * c. irqp is initialized, vector is allocated (fixed & shared) 9987ff178cdSJimmy Vetayases */ 9997ff178cdSJimmy Vetayases if ((vecp = apix_alloc_intx(dip, inum, irqno)) == NULL) 10007ff178cdSJimmy Vetayases return (0); 10017ff178cdSJimmy Vetayases 10027ff178cdSJimmy Vetayases cpuid = vecp->v_cpuid; 10037ff178cdSJimmy Vetayases v = vecp->v_vector; 10047ff178cdSJimmy Vetayases virt_vect = APIX_VIRTVECTOR(cpuid, v); 10057ff178cdSJimmy Vetayases } else { /* got virtual vector */ 10067ff178cdSJimmy Vetayases cpuid = APIX_VIRTVEC_CPU(virt_vect); 10077ff178cdSJimmy Vetayases vecp = xv_vector(cpuid, v); 10087ff178cdSJimmy Vetayases ASSERT(vecp != NULL); 10097ff178cdSJimmy Vetayases } 10107ff178cdSJimmy Vetayases 10117ff178cdSJimmy Vetayases lock_set(&apix_lock); 10127ff178cdSJimmy Vetayases if (vecp->v_state <= APIX_STATE_OBSOLETED) { 10137ff178cdSJimmy Vetayases vecp = NULL; 10147ff178cdSJimmy Vetayases 10157ff178cdSJimmy Vetayases /* 10167ff178cdSJimmy Vetayases * Basically the allocated but not enabled interrupts 10177ff178cdSJimmy Vetayases * will not get re-targeted. But MSIs in allocated state 10187ff178cdSJimmy Vetayases * could be re-targeted due to group re-targeting. 10197ff178cdSJimmy Vetayases */ 10207ff178cdSJimmy Vetayases if (intr_id != NULL && dip != NULL) { 10217ff178cdSJimmy Vetayases ddi_intr_handle_impl_t *hdlp = intr_id; 10227ff178cdSJimmy Vetayases vecp = apix_get_dev_map(dip, hdlp->ih_inum, 10237ff178cdSJimmy Vetayases hdlp->ih_type); 10247ff178cdSJimmy Vetayases ASSERT(vecp->v_state == APIX_STATE_ALLOCED); 10257ff178cdSJimmy Vetayases } 10267ff178cdSJimmy Vetayases if (vecp == NULL) { 10277ff178cdSJimmy Vetayases lock_clear(&apix_lock); 10287ff178cdSJimmy Vetayases cmn_err(CE_WARN, "Invalid interrupt 0x%x,0x%x " 10297ff178cdSJimmy Vetayases " for %p to add", cpuid, v, intr_id); 10307ff178cdSJimmy Vetayases return (0); 10317ff178cdSJimmy Vetayases } 10327ff178cdSJimmy Vetayases cpuid = vecp->v_cpuid; 10337ff178cdSJimmy Vetayases virt_vect = APIX_VIRTVECTOR(cpuid, vecp->v_vector); 10347ff178cdSJimmy Vetayases } 10357ff178cdSJimmy Vetayases 10367ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(cpuid); 10377ff178cdSJimmy Vetayases apix_insert_av(vecp, intr_id, xxintr, arg1, arg2, ticksp, ipl, dip); 10387ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 10397ff178cdSJimmy Vetayases 10407ff178cdSJimmy Vetayases (void) apix_addspl(virt_vect, ipl, 0, 0); 10417ff178cdSJimmy Vetayases 10427ff178cdSJimmy Vetayases lock_clear(&apix_lock); 10437ff178cdSJimmy Vetayases 10447ff178cdSJimmy Vetayases return (1); 10457ff178cdSJimmy Vetayases } 10467ff178cdSJimmy Vetayases 10477ff178cdSJimmy Vetayases /* 10487ff178cdSJimmy Vetayases * Remove avintr 10497ff178cdSJimmy Vetayases * 10507ff178cdSJimmy Vetayases * For fixed, if it's the last one of shared interrupts, free the vector. 10517ff178cdSJimmy Vetayases * For msi/x, only disable the interrupt but not free the vector, which 10527ff178cdSJimmy Vetayases * is freed by PSM_XXX_FREE_XXX. 10537ff178cdSJimmy Vetayases */ 10547ff178cdSJimmy Vetayases void 10557ff178cdSJimmy Vetayases apix_rem_avintr(void *intr_id, int ipl, avfunc xxintr, int virt_vect) 10567ff178cdSJimmy Vetayases { 10577ff178cdSJimmy Vetayases avfunc f; 10587ff178cdSJimmy Vetayases apix_vector_t *vecp; 10597ff178cdSJimmy Vetayases struct autovec *avp; 10607ff178cdSJimmy Vetayases processorid_t cpuid; 10617ff178cdSJimmy Vetayases 10627ff178cdSJimmy Vetayases if ((f = xxintr) == NULL) 10637ff178cdSJimmy Vetayases return; 10647ff178cdSJimmy Vetayases 10657ff178cdSJimmy Vetayases lock_set(&apix_lock); 10667ff178cdSJimmy Vetayases 10677ff178cdSJimmy Vetayases if (!APIX_IS_VIRTVEC(virt_vect)) { /* got irq */ 10687ff178cdSJimmy Vetayases vecp = apix_intx_get_vector(virt_vect); 10697ff178cdSJimmy Vetayases virt_vect = APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector); 10707ff178cdSJimmy Vetayases } else /* got virtual vector */ 10717ff178cdSJimmy Vetayases vecp = xv_vector(APIX_VIRTVEC_CPU(virt_vect), 10727ff178cdSJimmy Vetayases APIX_VIRTVEC_VECTOR(virt_vect)); 10737ff178cdSJimmy Vetayases 10747ff178cdSJimmy Vetayases if (vecp == NULL) { 10757ff178cdSJimmy Vetayases lock_clear(&apix_lock); 10767ff178cdSJimmy Vetayases cmn_err(CE_CONT, "Invalid interrupt 0x%x,0x%x to remove", 10777ff178cdSJimmy Vetayases APIX_VIRTVEC_CPU(virt_vect), 10787ff178cdSJimmy Vetayases APIX_VIRTVEC_VECTOR(virt_vect)); 10797ff178cdSJimmy Vetayases return; 10807ff178cdSJimmy Vetayases } 10817ff178cdSJimmy Vetayases 10827ff178cdSJimmy Vetayases if (vecp->v_state <= APIX_STATE_OBSOLETED || 10837ff178cdSJimmy Vetayases ((avp = apix_find_av(vecp, intr_id, f)) == NULL)) { 10847ff178cdSJimmy Vetayases /* 10857ff178cdSJimmy Vetayases * It's possible that the interrupt is rebound to a 10867ff178cdSJimmy Vetayases * different cpu before rem_avintr() is called. Search 10877ff178cdSJimmy Vetayases * through all vectors once it happens. 10887ff178cdSJimmy Vetayases */ 10897ff178cdSJimmy Vetayases if ((vecp = apix_find_vector_by_avintr(intr_id, f)) 10907ff178cdSJimmy Vetayases == NULL) { 10917ff178cdSJimmy Vetayases lock_clear(&apix_lock); 10927ff178cdSJimmy Vetayases cmn_err(CE_CONT, "Unknown interrupt 0x%x,0x%x " 10937ff178cdSJimmy Vetayases "for %p to remove", APIX_VIRTVEC_CPU(virt_vect), 10947ff178cdSJimmy Vetayases APIX_VIRTVEC_VECTOR(virt_vect), intr_id); 10957ff178cdSJimmy Vetayases return; 10967ff178cdSJimmy Vetayases } 10977ff178cdSJimmy Vetayases virt_vect = APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector); 10987ff178cdSJimmy Vetayases avp = apix_find_av(vecp, intr_id, f); 10997ff178cdSJimmy Vetayases } 11007ff178cdSJimmy Vetayases cpuid = vecp->v_cpuid; 11017ff178cdSJimmy Vetayases 11027ff178cdSJimmy Vetayases /* disable interrupt */ 11037ff178cdSJimmy Vetayases (void) apix_delspl(virt_vect, ipl, 0, 0); 11047ff178cdSJimmy Vetayases 11057ff178cdSJimmy Vetayases /* remove ISR entry */ 11067ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(cpuid); 11077ff178cdSJimmy Vetayases apix_remove_av(vecp, avp); 11087ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 11097ff178cdSJimmy Vetayases 11107ff178cdSJimmy Vetayases lock_clear(&apix_lock); 11117ff178cdSJimmy Vetayases } 11127ff178cdSJimmy Vetayases 11137ff178cdSJimmy Vetayases /* 11147ff178cdSJimmy Vetayases * Device to vector mapping table 11157ff178cdSJimmy Vetayases */ 11167ff178cdSJimmy Vetayases 11177ff178cdSJimmy Vetayases static void 11187ff178cdSJimmy Vetayases apix_clear_dev_map(dev_info_t *dip, int inum, int type) 11197ff178cdSJimmy Vetayases { 11207ff178cdSJimmy Vetayases char *name; 11217ff178cdSJimmy Vetayases major_t major; 11227ff178cdSJimmy Vetayases apix_dev_vector_t *dvp, *prev = NULL; 11237ff178cdSJimmy Vetayases int found = 0; 11247ff178cdSJimmy Vetayases 11257ff178cdSJimmy Vetayases name = ddi_get_name(dip); 11267ff178cdSJimmy Vetayases major = ddi_name_to_major(name); 11277ff178cdSJimmy Vetayases 11287ff178cdSJimmy Vetayases mutex_enter(&apix_mutex); 11297ff178cdSJimmy Vetayases 11307ff178cdSJimmy Vetayases for (dvp = apix_dev_vector[major]; dvp != NULL; 11317ff178cdSJimmy Vetayases prev = dvp, dvp = dvp->dv_next) { 11327ff178cdSJimmy Vetayases if (dvp->dv_dip == dip && dvp->dv_inum == inum && 11337ff178cdSJimmy Vetayases dvp->dv_type == type) { 11347ff178cdSJimmy Vetayases found++; 11357ff178cdSJimmy Vetayases break; 11367ff178cdSJimmy Vetayases } 11377ff178cdSJimmy Vetayases } 11387ff178cdSJimmy Vetayases 11397ff178cdSJimmy Vetayases if (!found) { 11407ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 11417ff178cdSJimmy Vetayases return; 11427ff178cdSJimmy Vetayases } 11437ff178cdSJimmy Vetayases 11447ff178cdSJimmy Vetayases if (prev != NULL) 11457ff178cdSJimmy Vetayases prev->dv_next = dvp->dv_next; 11467ff178cdSJimmy Vetayases 11477ff178cdSJimmy Vetayases if (apix_dev_vector[major] == dvp) 11487ff178cdSJimmy Vetayases apix_dev_vector[major] = dvp->dv_next; 11497ff178cdSJimmy Vetayases 11507ff178cdSJimmy Vetayases dvp->dv_vector->v_devp = NULL; 11517ff178cdSJimmy Vetayases 11527ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 11537ff178cdSJimmy Vetayases 11547ff178cdSJimmy Vetayases kmem_free(dvp, sizeof (apix_dev_vector_t)); 11557ff178cdSJimmy Vetayases } 11567ff178cdSJimmy Vetayases 11577ff178cdSJimmy Vetayases void 11587ff178cdSJimmy Vetayases apix_set_dev_map(apix_vector_t *vecp, dev_info_t *dip, int inum) 11597ff178cdSJimmy Vetayases { 11607ff178cdSJimmy Vetayases apix_dev_vector_t *dvp; 11617ff178cdSJimmy Vetayases char *name; 11627ff178cdSJimmy Vetayases major_t major; 11637ff178cdSJimmy Vetayases uint32_t found = 0; 11647ff178cdSJimmy Vetayases 11657ff178cdSJimmy Vetayases ASSERT(dip != NULL); 11667ff178cdSJimmy Vetayases name = ddi_get_name(dip); 11677ff178cdSJimmy Vetayases major = ddi_name_to_major(name); 11687ff178cdSJimmy Vetayases 11697ff178cdSJimmy Vetayases mutex_enter(&apix_mutex); 11707ff178cdSJimmy Vetayases 11717ff178cdSJimmy Vetayases for (dvp = apix_dev_vector[major]; dvp != NULL; 11727ff178cdSJimmy Vetayases dvp = dvp->dv_next) { 11737ff178cdSJimmy Vetayases if (dvp->dv_dip == dip && dvp->dv_inum == inum && 11747ff178cdSJimmy Vetayases dvp->dv_type == vecp->v_type) { 11757ff178cdSJimmy Vetayases found++; 11767ff178cdSJimmy Vetayases break; 11777ff178cdSJimmy Vetayases } 11787ff178cdSJimmy Vetayases } 11797ff178cdSJimmy Vetayases 11807ff178cdSJimmy Vetayases if (found == 0) { /* not found */ 11817ff178cdSJimmy Vetayases dvp = kmem_zalloc(sizeof (apix_dev_vector_t), KM_SLEEP); 11827ff178cdSJimmy Vetayases dvp->dv_dip = dip; 11837ff178cdSJimmy Vetayases dvp->dv_inum = inum; 11847ff178cdSJimmy Vetayases dvp->dv_type = vecp->v_type; 11857ff178cdSJimmy Vetayases 11867ff178cdSJimmy Vetayases dvp->dv_next = apix_dev_vector[major]; 11877ff178cdSJimmy Vetayases apix_dev_vector[major] = dvp; 11887ff178cdSJimmy Vetayases } 11897ff178cdSJimmy Vetayases dvp->dv_vector = vecp; 11907ff178cdSJimmy Vetayases vecp->v_devp = dvp; 11917ff178cdSJimmy Vetayases 11927ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 11937ff178cdSJimmy Vetayases 11947ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apix_set_dev_map: dip=0x%p " 11957ff178cdSJimmy Vetayases "inum=0x%x vector=0x%x/0x%x\n", 11967ff178cdSJimmy Vetayases (void *)dip, inum, vecp->v_cpuid, vecp->v_vector)); 11977ff178cdSJimmy Vetayases } 11987ff178cdSJimmy Vetayases 11997ff178cdSJimmy Vetayases apix_vector_t * 12007ff178cdSJimmy Vetayases apix_get_dev_map(dev_info_t *dip, int inum, int type) 12017ff178cdSJimmy Vetayases { 12027ff178cdSJimmy Vetayases char *name; 12037ff178cdSJimmy Vetayases major_t major; 12047ff178cdSJimmy Vetayases apix_dev_vector_t *dvp; 12057ff178cdSJimmy Vetayases apix_vector_t *vecp; 12067ff178cdSJimmy Vetayases 12077ff178cdSJimmy Vetayases name = ddi_get_name(dip); 12087ff178cdSJimmy Vetayases if ((major = ddi_name_to_major(name)) == DDI_MAJOR_T_NONE) 12097ff178cdSJimmy Vetayases return (NULL); 12107ff178cdSJimmy Vetayases 12117ff178cdSJimmy Vetayases mutex_enter(&apix_mutex); 12127ff178cdSJimmy Vetayases for (dvp = apix_dev_vector[major]; dvp != NULL; 12137ff178cdSJimmy Vetayases dvp = dvp->dv_next) { 12147ff178cdSJimmy Vetayases if (dvp->dv_dip == dip && dvp->dv_inum == inum && 12157ff178cdSJimmy Vetayases dvp->dv_type == type) { 12167ff178cdSJimmy Vetayases vecp = dvp->dv_vector; 12177ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 12187ff178cdSJimmy Vetayases return (vecp); 12197ff178cdSJimmy Vetayases } 12207ff178cdSJimmy Vetayases } 12217ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 12227ff178cdSJimmy Vetayases 12237ff178cdSJimmy Vetayases return (NULL); 12247ff178cdSJimmy Vetayases } 12257ff178cdSJimmy Vetayases 12267ff178cdSJimmy Vetayases /* 12277ff178cdSJimmy Vetayases * Get minimum inum for specified device, used for MSI 12287ff178cdSJimmy Vetayases */ 12297ff178cdSJimmy Vetayases int 12307ff178cdSJimmy Vetayases apix_get_min_dev_inum(dev_info_t *dip, int type) 12317ff178cdSJimmy Vetayases { 12327ff178cdSJimmy Vetayases char *name; 12337ff178cdSJimmy Vetayases major_t major; 12347ff178cdSJimmy Vetayases apix_dev_vector_t *dvp; 12357ff178cdSJimmy Vetayases int inum = -1; 12367ff178cdSJimmy Vetayases 12377ff178cdSJimmy Vetayases name = ddi_get_name(dip); 12387ff178cdSJimmy Vetayases major = ddi_name_to_major(name); 12397ff178cdSJimmy Vetayases 12407ff178cdSJimmy Vetayases mutex_enter(&apix_mutex); 12417ff178cdSJimmy Vetayases for (dvp = apix_dev_vector[major]; dvp != NULL; 12427ff178cdSJimmy Vetayases dvp = dvp->dv_next) { 12437ff178cdSJimmy Vetayases if (dvp->dv_dip == dip && dvp->dv_type == type) { 12447ff178cdSJimmy Vetayases if (inum == -1) 12457ff178cdSJimmy Vetayases inum = dvp->dv_inum; 12467ff178cdSJimmy Vetayases else 12477ff178cdSJimmy Vetayases inum = (dvp->dv_inum < inum) ? 12487ff178cdSJimmy Vetayases dvp->dv_inum : inum; 12497ff178cdSJimmy Vetayases } 12507ff178cdSJimmy Vetayases } 12517ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 12527ff178cdSJimmy Vetayases 12537ff178cdSJimmy Vetayases return (inum); 12547ff178cdSJimmy Vetayases } 12557ff178cdSJimmy Vetayases 12567ff178cdSJimmy Vetayases int 12577ff178cdSJimmy Vetayases apix_get_max_dev_inum(dev_info_t *dip, int type) 12587ff178cdSJimmy Vetayases { 12597ff178cdSJimmy Vetayases char *name; 12607ff178cdSJimmy Vetayases major_t major; 12617ff178cdSJimmy Vetayases apix_dev_vector_t *dvp; 12627ff178cdSJimmy Vetayases int inum = -1; 12637ff178cdSJimmy Vetayases 12647ff178cdSJimmy Vetayases name = ddi_get_name(dip); 12657ff178cdSJimmy Vetayases major = ddi_name_to_major(name); 12667ff178cdSJimmy Vetayases 12677ff178cdSJimmy Vetayases mutex_enter(&apix_mutex); 12687ff178cdSJimmy Vetayases for (dvp = apix_dev_vector[major]; dvp != NULL; 12697ff178cdSJimmy Vetayases dvp = dvp->dv_next) { 12707ff178cdSJimmy Vetayases if (dvp->dv_dip == dip && dvp->dv_type == type) { 12717ff178cdSJimmy Vetayases if (inum == -1) 12727ff178cdSJimmy Vetayases inum = dvp->dv_inum; 12737ff178cdSJimmy Vetayases else 12747ff178cdSJimmy Vetayases inum = (dvp->dv_inum > inum) ? 12757ff178cdSJimmy Vetayases dvp->dv_inum : inum; 12767ff178cdSJimmy Vetayases } 12777ff178cdSJimmy Vetayases } 12787ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 12797ff178cdSJimmy Vetayases 12807ff178cdSJimmy Vetayases return (inum); 12817ff178cdSJimmy Vetayases } 12827ff178cdSJimmy Vetayases 12837ff178cdSJimmy Vetayases /* 12847ff178cdSJimmy Vetayases * Major to cpu binding, for INTR_ROUND_ROBIN_WITH_AFFINITY cpu 12857ff178cdSJimmy Vetayases * binding policy 12867ff178cdSJimmy Vetayases */ 12877ff178cdSJimmy Vetayases 12887ff178cdSJimmy Vetayases static uint32_t 12897ff178cdSJimmy Vetayases apix_get_dev_binding(dev_info_t *dip) 12907ff178cdSJimmy Vetayases { 12917ff178cdSJimmy Vetayases major_t major; 12927ff178cdSJimmy Vetayases char *name; 12937ff178cdSJimmy Vetayases uint32_t cpu = IRQ_UNINIT; 12947ff178cdSJimmy Vetayases 12957ff178cdSJimmy Vetayases name = ddi_get_name(dip); 12967ff178cdSJimmy Vetayases major = ddi_name_to_major(name); 12977ff178cdSJimmy Vetayases if (major < devcnt) { 12987ff178cdSJimmy Vetayases mutex_enter(&apix_mutex); 12997ff178cdSJimmy Vetayases cpu = apix_major_to_cpu[major]; 13007ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 13017ff178cdSJimmy Vetayases } 13027ff178cdSJimmy Vetayases 13037ff178cdSJimmy Vetayases return (cpu); 13047ff178cdSJimmy Vetayases } 13057ff178cdSJimmy Vetayases 13067ff178cdSJimmy Vetayases static void 13077ff178cdSJimmy Vetayases apix_set_dev_binding(dev_info_t *dip, uint32_t cpu) 13087ff178cdSJimmy Vetayases { 13097ff178cdSJimmy Vetayases major_t major; 13107ff178cdSJimmy Vetayases char *name; 13117ff178cdSJimmy Vetayases 13127ff178cdSJimmy Vetayases /* setup major to cpu mapping */ 13137ff178cdSJimmy Vetayases name = ddi_get_name(dip); 13147ff178cdSJimmy Vetayases major = ddi_name_to_major(name); 13157ff178cdSJimmy Vetayases if (apix_major_to_cpu[major] == IRQ_UNINIT) { 13167ff178cdSJimmy Vetayases mutex_enter(&apix_mutex); 13177ff178cdSJimmy Vetayases apix_major_to_cpu[major] = cpu; 13187ff178cdSJimmy Vetayases mutex_exit(&apix_mutex); 13197ff178cdSJimmy Vetayases } 13207ff178cdSJimmy Vetayases } 13217ff178cdSJimmy Vetayases 13227ff178cdSJimmy Vetayases /* 13237ff178cdSJimmy Vetayases * return the cpu to which this intr should be bound. 13247ff178cdSJimmy Vetayases * Check properties or any other mechanism to see if user wants it 13257ff178cdSJimmy Vetayases * bound to a specific CPU. If so, return the cpu id with high bit set. 13267ff178cdSJimmy Vetayases * If not, use the policy to choose a cpu and return the id. 13277ff178cdSJimmy Vetayases */ 13287ff178cdSJimmy Vetayases uint32_t 13297ff178cdSJimmy Vetayases apix_bind_cpu(dev_info_t *dip) 13307ff178cdSJimmy Vetayases { 13317ff178cdSJimmy Vetayases int instance, instno, prop_len, bind_cpu, count; 13327ff178cdSJimmy Vetayases uint_t i, rc; 13337ff178cdSJimmy Vetayases major_t major; 13347ff178cdSJimmy Vetayases char *name, *drv_name, *prop_val, *cptr; 13357ff178cdSJimmy Vetayases char prop_name[32]; 13367ff178cdSJimmy Vetayases 13377ff178cdSJimmy Vetayases lock_set(&apix_lock); 13387ff178cdSJimmy Vetayases 13397ff178cdSJimmy Vetayases if (apic_intr_policy == INTR_LOWEST_PRIORITY) { 13407ff178cdSJimmy Vetayases cmn_err(CE_WARN, "apix: unsupported interrupt binding policy " 13417ff178cdSJimmy Vetayases "LOWEST PRIORITY, use ROUND ROBIN instead"); 13427ff178cdSJimmy Vetayases apic_intr_policy = INTR_ROUND_ROBIN; 13437ff178cdSJimmy Vetayases } 13447ff178cdSJimmy Vetayases 13457ff178cdSJimmy Vetayases if (apic_nproc == 1) { 13467ff178cdSJimmy Vetayases lock_clear(&apix_lock); 13477ff178cdSJimmy Vetayases return (0); 13487ff178cdSJimmy Vetayases } 13497ff178cdSJimmy Vetayases 13507ff178cdSJimmy Vetayases drv_name = NULL; 13517ff178cdSJimmy Vetayases rc = DDI_PROP_NOT_FOUND; 13527ff178cdSJimmy Vetayases major = (major_t)-1; 13537ff178cdSJimmy Vetayases if (dip != NULL) { 13547ff178cdSJimmy Vetayases name = ddi_get_name(dip); 13557ff178cdSJimmy Vetayases major = ddi_name_to_major(name); 13567ff178cdSJimmy Vetayases drv_name = ddi_major_to_name(major); 13577ff178cdSJimmy Vetayases instance = ddi_get_instance(dip); 13587ff178cdSJimmy Vetayases if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) { 13597ff178cdSJimmy Vetayases bind_cpu = apix_get_dev_binding(dip); 13607ff178cdSJimmy Vetayases if (bind_cpu != IRQ_UNINIT) { 13617ff178cdSJimmy Vetayases lock_clear(&apix_lock); 13627ff178cdSJimmy Vetayases return (bind_cpu); 13637ff178cdSJimmy Vetayases } 13647ff178cdSJimmy Vetayases } 13657ff178cdSJimmy Vetayases /* 13667ff178cdSJimmy Vetayases * search for "drvname"_intpt_bind_cpus property first, the 13677ff178cdSJimmy Vetayases * syntax of the property should be "a[,b,c,...]" where 13687ff178cdSJimmy Vetayases * instance 0 binds to cpu a, instance 1 binds to cpu b, 13697ff178cdSJimmy Vetayases * instance 3 binds to cpu c... 13707ff178cdSJimmy Vetayases * ddi_getlongprop() will search /option first, then / 13717ff178cdSJimmy Vetayases * if "drvname"_intpt_bind_cpus doesn't exist, then find 13727ff178cdSJimmy Vetayases * intpt_bind_cpus property. The syntax is the same, and 13737ff178cdSJimmy Vetayases * it applies to all the devices if its "drvname" specific 13747ff178cdSJimmy Vetayases * property doesn't exist 13757ff178cdSJimmy Vetayases */ 13767ff178cdSJimmy Vetayases (void) strcpy(prop_name, drv_name); 13777ff178cdSJimmy Vetayases (void) strcat(prop_name, "_intpt_bind_cpus"); 13787ff178cdSJimmy Vetayases rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name, 13797ff178cdSJimmy Vetayases (caddr_t)&prop_val, &prop_len); 13807ff178cdSJimmy Vetayases if (rc != DDI_PROP_SUCCESS) { 13817ff178cdSJimmy Vetayases rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, 13827ff178cdSJimmy Vetayases "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len); 13837ff178cdSJimmy Vetayases } 13847ff178cdSJimmy Vetayases } 13857ff178cdSJimmy Vetayases if (rc == DDI_PROP_SUCCESS) { 13867ff178cdSJimmy Vetayases for (i = count = 0; i < (prop_len - 1); i++) 13877ff178cdSJimmy Vetayases if (prop_val[i] == ',') 13887ff178cdSJimmy Vetayases count++; 13897ff178cdSJimmy Vetayases if (prop_val[i-1] != ',') 13907ff178cdSJimmy Vetayases count++; 13917ff178cdSJimmy Vetayases /* 13927ff178cdSJimmy Vetayases * if somehow the binding instances defined in the 13937ff178cdSJimmy Vetayases * property are not enough for this instno., then 13947ff178cdSJimmy Vetayases * reuse the pattern for the next instance until 13957ff178cdSJimmy Vetayases * it reaches the requested instno 13967ff178cdSJimmy Vetayases */ 13977ff178cdSJimmy Vetayases instno = instance % count; 13987ff178cdSJimmy Vetayases i = 0; 13997ff178cdSJimmy Vetayases cptr = prop_val; 14007ff178cdSJimmy Vetayases while (i < instno) 14017ff178cdSJimmy Vetayases if (*cptr++ == ',') 14027ff178cdSJimmy Vetayases i++; 14037ff178cdSJimmy Vetayases bind_cpu = stoi(&cptr); 14047ff178cdSJimmy Vetayases kmem_free(prop_val, prop_len); 14057ff178cdSJimmy Vetayases /* if specific cpu is bogus, then default to cpu 0 */ 14067ff178cdSJimmy Vetayases if (bind_cpu >= apic_nproc) { 14077ff178cdSJimmy Vetayases cmn_err(CE_WARN, "apix: %s=%s: CPU %d not present", 14087ff178cdSJimmy Vetayases prop_name, prop_val, bind_cpu); 14097ff178cdSJimmy Vetayases bind_cpu = 0; 14107ff178cdSJimmy Vetayases } else { 14117ff178cdSJimmy Vetayases /* indicate that we are bound at user request */ 14127ff178cdSJimmy Vetayases bind_cpu |= IRQ_USER_BOUND; 14137ff178cdSJimmy Vetayases } 14147ff178cdSJimmy Vetayases /* 14157ff178cdSJimmy Vetayases * no need to check apic_cpus[].aci_status, if specific cpu is 14167ff178cdSJimmy Vetayases * not up, then post_cpu_start will handle it. 14177ff178cdSJimmy Vetayases */ 14187ff178cdSJimmy Vetayases } else { 14197ff178cdSJimmy Vetayases bind_cpu = apic_get_next_bind_cpu(); 14207ff178cdSJimmy Vetayases } 14217ff178cdSJimmy Vetayases 14227ff178cdSJimmy Vetayases lock_clear(&apix_lock); 14237ff178cdSJimmy Vetayases 14247ff178cdSJimmy Vetayases return ((uint32_t)bind_cpu); 14257ff178cdSJimmy Vetayases } 14267ff178cdSJimmy Vetayases 14277ff178cdSJimmy Vetayases static boolean_t 14287ff178cdSJimmy Vetayases apix_is_cpu_enabled(processorid_t cpuid) 14297ff178cdSJimmy Vetayases { 14307ff178cdSJimmy Vetayases apic_cpus_info_t *cpu_infop; 14317ff178cdSJimmy Vetayases 14327ff178cdSJimmy Vetayases cpu_infop = &apic_cpus[cpuid]; 14337ff178cdSJimmy Vetayases 14347ff178cdSJimmy Vetayases if ((cpu_infop->aci_status & APIC_CPU_INTR_ENABLE) == 0) 14357ff178cdSJimmy Vetayases return (B_FALSE); 14367ff178cdSJimmy Vetayases 14377ff178cdSJimmy Vetayases return (B_TRUE); 14387ff178cdSJimmy Vetayases } 14397ff178cdSJimmy Vetayases 14407ff178cdSJimmy Vetayases /* 14417ff178cdSJimmy Vetayases * Must be called with apix_lock held. This function can be 14427ff178cdSJimmy Vetayases * called from above lock level by apix_intr_redistribute(). 14437ff178cdSJimmy Vetayases * 14447ff178cdSJimmy Vetayases * Arguments: 14457ff178cdSJimmy Vetayases * vecp : Vector to be rebound 14467ff178cdSJimmy Vetayases * tocpu : Target cpu. IRQ_UNINIT means target is vecp->v_cpuid. 14477ff178cdSJimmy Vetayases * count : Number of continuous vectors 14487ff178cdSJimmy Vetayases * 14497ff178cdSJimmy Vetayases * Return new vector being bound to 14507ff178cdSJimmy Vetayases */ 14517ff178cdSJimmy Vetayases apix_vector_t * 14527ff178cdSJimmy Vetayases apix_rebind(apix_vector_t *vecp, processorid_t newcpu, int count) 14537ff178cdSJimmy Vetayases { 14547ff178cdSJimmy Vetayases apix_vector_t *newp, *oldp; 14557ff178cdSJimmy Vetayases processorid_t oldcpu = vecp->v_cpuid; 14567ff178cdSJimmy Vetayases uchar_t newvec, oldvec = vecp->v_vector; 14577ff178cdSJimmy Vetayases int i; 14587ff178cdSJimmy Vetayases 14597ff178cdSJimmy Vetayases ASSERT(LOCK_HELD(&apix_lock) && count > 0); 14607ff178cdSJimmy Vetayases 14617ff178cdSJimmy Vetayases if (!apix_is_cpu_enabled(newcpu)) 14627ff178cdSJimmy Vetayases return (NULL); 14637ff178cdSJimmy Vetayases 14647ff178cdSJimmy Vetayases if (vecp->v_cpuid == newcpu) /* rebind to the same cpu */ 14657ff178cdSJimmy Vetayases return (vecp); 14667ff178cdSJimmy Vetayases 14677ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(oldcpu); 14687ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(newcpu); 14697ff178cdSJimmy Vetayases 14707ff178cdSJimmy Vetayases /* allocate vector */ 14717ff178cdSJimmy Vetayases if (count == 1) 14727ff178cdSJimmy Vetayases newp = apix_alloc_vector_oncpu(newcpu, NULL, 0, vecp->v_type); 14737ff178cdSJimmy Vetayases else { 14747ff178cdSJimmy Vetayases ASSERT(vecp->v_type == APIX_TYPE_MSI); 14757ff178cdSJimmy Vetayases newp = apix_alloc_nvectors_oncpu(newcpu, NULL, 0, count, 14767ff178cdSJimmy Vetayases vecp->v_type); 14777ff178cdSJimmy Vetayases } 14787ff178cdSJimmy Vetayases if (newp == NULL) { 14797ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(newcpu); 14807ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(oldcpu); 14817ff178cdSJimmy Vetayases return (NULL); 14827ff178cdSJimmy Vetayases } 14837ff178cdSJimmy Vetayases 14847ff178cdSJimmy Vetayases newvec = newp->v_vector; 14857ff178cdSJimmy Vetayases apix_dup_vectors(vecp, newp, count); 14867ff178cdSJimmy Vetayases 14877ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(newcpu); 14887ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(oldcpu); 14897ff178cdSJimmy Vetayases 14907ff178cdSJimmy Vetayases if (!DDI_INTR_IS_MSI_OR_MSIX(vecp->v_type)) { 14917ff178cdSJimmy Vetayases ASSERT(count == 1); 14927ff178cdSJimmy Vetayases if (apix_intx_rebind(vecp->v_inum, newcpu, newvec) != 0) { 14937ff178cdSJimmy Vetayases struct autovec *avp; 14947ff178cdSJimmy Vetayases int inum; 14957ff178cdSJimmy Vetayases 14967ff178cdSJimmy Vetayases /* undo duplication */ 14977ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(oldcpu); 14987ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(newcpu); 14997ff178cdSJimmy Vetayases for (avp = newp->v_autovect; avp != NULL; 15007ff178cdSJimmy Vetayases avp = avp->av_link) { 15017ff178cdSJimmy Vetayases if (avp->av_dip != NULL) { 15027ff178cdSJimmy Vetayases inum = GET_INTR_INUM(avp->av_intr_id); 15037ff178cdSJimmy Vetayases apix_set_dev_map(vecp, avp->av_dip, 15047ff178cdSJimmy Vetayases inum); 15057ff178cdSJimmy Vetayases } 15067ff178cdSJimmy Vetayases apix_remove_av(newp, avp); 15077ff178cdSJimmy Vetayases } 15087ff178cdSJimmy Vetayases apix_cleanup_vector(newp); 15097ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(newcpu); 15107ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(oldcpu); 15117ff178cdSJimmy Vetayases APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind fixed " 15127ff178cdSJimmy Vetayases "interrupt 0x%x to cpu %d failed\n", 15137ff178cdSJimmy Vetayases vecp->v_inum, newcpu)); 15147ff178cdSJimmy Vetayases return (NULL); 15157ff178cdSJimmy Vetayases } 15167ff178cdSJimmy Vetayases 15177ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(oldcpu); 15187ff178cdSJimmy Vetayases (void) apix_obsolete_vector(vecp); 15197ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(oldcpu); 15207ff178cdSJimmy Vetayases APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind fixed interrupt" 15217ff178cdSJimmy Vetayases " 0x%x/0x%x to 0x%x/0x%x\n", 15227ff178cdSJimmy Vetayases oldcpu, oldvec, newcpu, newvec)); 15237ff178cdSJimmy Vetayases return (newp); 15247ff178cdSJimmy Vetayases } 15257ff178cdSJimmy Vetayases 15267ff178cdSJimmy Vetayases for (i = 0; i < count; i++) { 15277ff178cdSJimmy Vetayases oldp = xv_vector(oldcpu, oldvec + i); 15287ff178cdSJimmy Vetayases newp = xv_vector(newcpu, newvec + i); 15297ff178cdSJimmy Vetayases 15307ff178cdSJimmy Vetayases if (newp->v_share > 0) { 15317ff178cdSJimmy Vetayases APIX_SET_REBIND_INFO(oldp, newp); 15327ff178cdSJimmy Vetayases 15337ff178cdSJimmy Vetayases apix_enable_vector(newp); 15347ff178cdSJimmy Vetayases 15357ff178cdSJimmy Vetayases APIX_CLR_REBIND_INFO(); 15367ff178cdSJimmy Vetayases } 15377ff178cdSJimmy Vetayases 15387ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(oldcpu); 15397ff178cdSJimmy Vetayases (void) apix_obsolete_vector(oldp); 15407ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(oldcpu); 15417ff178cdSJimmy Vetayases } 15427ff178cdSJimmy Vetayases APIC_VERBOSE(REBIND, (CE_CONT, "apix: rebind vector 0x%x/0x%x " 15437ff178cdSJimmy Vetayases "to 0x%x/0x%x, count=%d\n", 15447ff178cdSJimmy Vetayases oldcpu, oldvec, newcpu, newvec, count)); 15457ff178cdSJimmy Vetayases 15467ff178cdSJimmy Vetayases return (xv_vector(newcpu, newvec)); 15477ff178cdSJimmy Vetayases } 15487ff178cdSJimmy Vetayases 15497ff178cdSJimmy Vetayases /* 15507ff178cdSJimmy Vetayases * Senarios include: 15517ff178cdSJimmy Vetayases * a. add_avintr() is called before irqp initialized (legacy) 15527ff178cdSJimmy Vetayases * b. irqp is initialized, vector is not allocated (fixed interrupts) 15537ff178cdSJimmy Vetayases * c. irqp is initialized, vector is allocated (shared interrupts) 15547ff178cdSJimmy Vetayases */ 15557ff178cdSJimmy Vetayases apix_vector_t * 15567ff178cdSJimmy Vetayases apix_alloc_intx(dev_info_t *dip, int inum, int irqno) 15577ff178cdSJimmy Vetayases { 15587ff178cdSJimmy Vetayases apic_irq_t *irqp; 15597ff178cdSJimmy Vetayases apix_vector_t *vecp; 15607ff178cdSJimmy Vetayases 15617ff178cdSJimmy Vetayases /* 15627ff178cdSJimmy Vetayases * Allocate IRQ. Caller is later responsible for the 15637ff178cdSJimmy Vetayases * initialization 15647ff178cdSJimmy Vetayases */ 15657ff178cdSJimmy Vetayases mutex_enter(&airq_mutex); 15667ff178cdSJimmy Vetayases if ((irqp = apic_irq_table[irqno]) == NULL) { 15677ff178cdSJimmy Vetayases /* allocate irq */ 15687ff178cdSJimmy Vetayases irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP); 15697ff178cdSJimmy Vetayases irqp->airq_mps_intr_index = FREE_INDEX; 15707ff178cdSJimmy Vetayases apic_irq_table[irqno] = irqp; 15717ff178cdSJimmy Vetayases } 15727ff178cdSJimmy Vetayases if (irqp->airq_mps_intr_index == FREE_INDEX) { 15737ff178cdSJimmy Vetayases irqp->airq_mps_intr_index = DEFAULT_INDEX; 15747ff178cdSJimmy Vetayases irqp->airq_cpu = IRQ_UNINIT; 15757ff178cdSJimmy Vetayases irqp->airq_origirq = (uchar_t)irqno; 15767ff178cdSJimmy Vetayases } 15777ff178cdSJimmy Vetayases 15787ff178cdSJimmy Vetayases mutex_exit(&airq_mutex); 15797ff178cdSJimmy Vetayases 15807ff178cdSJimmy Vetayases /* 15817ff178cdSJimmy Vetayases * allocate vector 15827ff178cdSJimmy Vetayases */ 15837ff178cdSJimmy Vetayases if (irqp->airq_cpu == IRQ_UNINIT) { 15847ff178cdSJimmy Vetayases uint32_t bindcpu, cpuid; 15857ff178cdSJimmy Vetayases 15867ff178cdSJimmy Vetayases /* select cpu by system policy */ 15877ff178cdSJimmy Vetayases bindcpu = apix_bind_cpu(dip); 15887ff178cdSJimmy Vetayases cpuid = bindcpu & ~IRQ_USER_BOUND; 15897ff178cdSJimmy Vetayases 15907ff178cdSJimmy Vetayases /* allocate vector */ 15917ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(cpuid); 15927ff178cdSJimmy Vetayases 15937ff178cdSJimmy Vetayases if ((vecp = apix_alloc_vector_oncpu(bindcpu, dip, inum, 15947ff178cdSJimmy Vetayases APIX_TYPE_FIXED)) == NULL) { 15957ff178cdSJimmy Vetayases cmn_err(CE_WARN, "No interrupt vector for irq %x", 15967ff178cdSJimmy Vetayases irqno); 15977ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 15987ff178cdSJimmy Vetayases return (NULL); 15997ff178cdSJimmy Vetayases } 16007ff178cdSJimmy Vetayases vecp->v_inum = irqno; 16017ff178cdSJimmy Vetayases vecp->v_flags |= APIX_VECT_MASKABLE; 16027ff178cdSJimmy Vetayases 16037ff178cdSJimmy Vetayases apix_intx_set_vector(irqno, vecp->v_cpuid, vecp->v_vector); 16047ff178cdSJimmy Vetayases 16057ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 16067ff178cdSJimmy Vetayases } else { 16077ff178cdSJimmy Vetayases vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector); 16087ff178cdSJimmy Vetayases ASSERT(!IS_VECT_FREE(vecp)); 16097ff178cdSJimmy Vetayases 16107ff178cdSJimmy Vetayases if (dip != NULL) 16117ff178cdSJimmy Vetayases apix_set_dev_map(vecp, dip, inum); 16127ff178cdSJimmy Vetayases } 16137ff178cdSJimmy Vetayases 16147ff178cdSJimmy Vetayases if ((dip != NULL) && 16157ff178cdSJimmy Vetayases (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) && 16167ff178cdSJimmy Vetayases ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0)) 16177ff178cdSJimmy Vetayases apix_set_dev_binding(dip, vecp->v_cpuid); 16187ff178cdSJimmy Vetayases 16197ff178cdSJimmy Vetayases apix_dprint_vector(vecp, dip, 1); 16207ff178cdSJimmy Vetayases 16217ff178cdSJimmy Vetayases return (vecp); 16227ff178cdSJimmy Vetayases } 16237ff178cdSJimmy Vetayases 16247ff178cdSJimmy Vetayases int 16257ff178cdSJimmy Vetayases apix_alloc_msi(dev_info_t *dip, int inum, int count, int behavior) 16267ff178cdSJimmy Vetayases { 16277ff178cdSJimmy Vetayases int i, cap_ptr, rcount = count; 16287ff178cdSJimmy Vetayases apix_vector_t *vecp; 16297ff178cdSJimmy Vetayases processorid_t bindcpu, cpuid; 16307ff178cdSJimmy Vetayases ushort_t msi_ctrl; 16317ff178cdSJimmy Vetayases ddi_acc_handle_t handle; 16327ff178cdSJimmy Vetayases 16337ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apix_alloc_msi_vectors: dip=0x%p " 16347ff178cdSJimmy Vetayases "inum=0x%x count=0x%x behavior=%d\n", 16357ff178cdSJimmy Vetayases (void *)dip, inum, count, behavior)); 16367ff178cdSJimmy Vetayases 16377ff178cdSJimmy Vetayases if (count > 1) { 16387ff178cdSJimmy Vetayases if (behavior == DDI_INTR_ALLOC_STRICT && 16397ff178cdSJimmy Vetayases apic_multi_msi_enable == 0) 16407ff178cdSJimmy Vetayases return (0); 16417ff178cdSJimmy Vetayases if (apic_multi_msi_enable == 0) 16427ff178cdSJimmy Vetayases count = 1; 16437ff178cdSJimmy Vetayases } 16447ff178cdSJimmy Vetayases 16457ff178cdSJimmy Vetayases /* Check whether it supports per-vector masking */ 16467ff178cdSJimmy Vetayases cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip); 16477ff178cdSJimmy Vetayases handle = i_ddi_get_pci_config_handle(dip); 16487ff178cdSJimmy Vetayases msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL); 16497ff178cdSJimmy Vetayases 16507ff178cdSJimmy Vetayases /* bind to cpu */ 16517ff178cdSJimmy Vetayases bindcpu = apix_bind_cpu(dip); 16527ff178cdSJimmy Vetayases cpuid = bindcpu & ~IRQ_USER_BOUND; 16537ff178cdSJimmy Vetayases 16547ff178cdSJimmy Vetayases /* if not ISP2, then round it down */ 16557ff178cdSJimmy Vetayases if (!ISP2(rcount)) 16567ff178cdSJimmy Vetayases rcount = 1 << (highbit(rcount) - 1); 16577ff178cdSJimmy Vetayases 16587ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(cpuid); 16597ff178cdSJimmy Vetayases for (vecp = NULL; rcount > 0; rcount >>= 1) { 16607ff178cdSJimmy Vetayases vecp = apix_alloc_nvectors_oncpu(bindcpu, dip, inum, rcount, 16617ff178cdSJimmy Vetayases APIX_TYPE_MSI); 16627ff178cdSJimmy Vetayases if (vecp != NULL || behavior == DDI_INTR_ALLOC_STRICT) 16637ff178cdSJimmy Vetayases break; 16647ff178cdSJimmy Vetayases } 16657ff178cdSJimmy Vetayases for (i = 0; vecp && i < rcount; i++) 16667ff178cdSJimmy Vetayases xv_vector(vecp->v_cpuid, vecp->v_vector + i)->v_flags |= 16677ff178cdSJimmy Vetayases (msi_ctrl & PCI_MSI_PVM_MASK) ? APIX_VECT_MASKABLE : 0; 16687ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 16697ff178cdSJimmy Vetayases if (vecp == NULL) { 16707ff178cdSJimmy Vetayases APIC_VERBOSE(INTR, (CE_CONT, 16717ff178cdSJimmy Vetayases "apix_alloc_msi: no %d cont vectors found on cpu 0x%x\n", 16727ff178cdSJimmy Vetayases count, bindcpu)); 16737ff178cdSJimmy Vetayases return (0); 16747ff178cdSJimmy Vetayases } 16757ff178cdSJimmy Vetayases 16767ff178cdSJimmy Vetayases /* major to cpu binding */ 16777ff178cdSJimmy Vetayases if ((apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) && 16787ff178cdSJimmy Vetayases ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0)) 16797ff178cdSJimmy Vetayases apix_set_dev_binding(dip, vecp->v_cpuid); 16807ff178cdSJimmy Vetayases 16817ff178cdSJimmy Vetayases apix_dprint_vector(vecp, dip, rcount); 16827ff178cdSJimmy Vetayases 16837ff178cdSJimmy Vetayases return (rcount); 16847ff178cdSJimmy Vetayases } 16857ff178cdSJimmy Vetayases 16867ff178cdSJimmy Vetayases int 16877ff178cdSJimmy Vetayases apix_alloc_msix(dev_info_t *dip, int inum, int count, int behavior) 16887ff178cdSJimmy Vetayases { 16897ff178cdSJimmy Vetayases apix_vector_t *vecp; 16907ff178cdSJimmy Vetayases processorid_t bindcpu, cpuid; 16917ff178cdSJimmy Vetayases int i; 16927ff178cdSJimmy Vetayases 16937ff178cdSJimmy Vetayases for (i = 0; i < count; i++) { 16947ff178cdSJimmy Vetayases /* select cpu by system policy */ 16957ff178cdSJimmy Vetayases bindcpu = apix_bind_cpu(dip); 16967ff178cdSJimmy Vetayases cpuid = bindcpu & ~IRQ_USER_BOUND; 16977ff178cdSJimmy Vetayases 16987ff178cdSJimmy Vetayases /* allocate vector */ 16997ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(cpuid); 17007ff178cdSJimmy Vetayases if ((vecp = apix_alloc_vector_oncpu(bindcpu, dip, inum + i, 17017ff178cdSJimmy Vetayases APIX_TYPE_MSIX)) == NULL) { 17027ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 17037ff178cdSJimmy Vetayases APIC_VERBOSE(INTR, (CE_CONT, "apix_alloc_msix: " 17047ff178cdSJimmy Vetayases "allocate msix for device dip=%p, inum=%d on" 17057ff178cdSJimmy Vetayases " cpu %d failed", (void *)dip, inum + i, bindcpu)); 17067ff178cdSJimmy Vetayases break; 17077ff178cdSJimmy Vetayases } 17087ff178cdSJimmy Vetayases vecp->v_flags |= APIX_VECT_MASKABLE; 17097ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 17107ff178cdSJimmy Vetayases 17117ff178cdSJimmy Vetayases /* major to cpu mapping */ 17127ff178cdSJimmy Vetayases if ((i == 0) && 17137ff178cdSJimmy Vetayases (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) && 17147ff178cdSJimmy Vetayases ((vecp->v_flags & APIX_VECT_USER_BOUND) == 0)) 17157ff178cdSJimmy Vetayases apix_set_dev_binding(dip, vecp->v_cpuid); 17167ff178cdSJimmy Vetayases 17177ff178cdSJimmy Vetayases apix_dprint_vector(vecp, dip, 1); 17187ff178cdSJimmy Vetayases } 17197ff178cdSJimmy Vetayases 17207ff178cdSJimmy Vetayases if (i < count && behavior == DDI_INTR_ALLOC_STRICT) { 17217ff178cdSJimmy Vetayases APIC_VERBOSE(INTR, (CE_WARN, "apix_alloc_msix: " 17227ff178cdSJimmy Vetayases "strictly allocate %d vectors failed, got %d\n", 17237ff178cdSJimmy Vetayases count, i)); 17247ff178cdSJimmy Vetayases apix_free_vectors(dip, inum, i, APIX_TYPE_MSIX); 17257ff178cdSJimmy Vetayases i = 0; 17267ff178cdSJimmy Vetayases } 17277ff178cdSJimmy Vetayases 17287ff178cdSJimmy Vetayases return (i); 17297ff178cdSJimmy Vetayases } 17307ff178cdSJimmy Vetayases 17317ff178cdSJimmy Vetayases /* 17327ff178cdSJimmy Vetayases * A rollback free for vectors allocated by apix_alloc_xxx(). 17337ff178cdSJimmy Vetayases */ 17347ff178cdSJimmy Vetayases void 17357ff178cdSJimmy Vetayases apix_free_vectors(dev_info_t *dip, int inum, int count, int type) 17367ff178cdSJimmy Vetayases { 17377ff178cdSJimmy Vetayases int i, cpuid; 17387ff178cdSJimmy Vetayases apix_vector_t *vecp; 17397ff178cdSJimmy Vetayases 17407ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: dip: %p inum: %x " 17417ff178cdSJimmy Vetayases "count: %x type: %x\n", 17427ff178cdSJimmy Vetayases (void *)dip, inum, count, type)); 17437ff178cdSJimmy Vetayases 17447ff178cdSJimmy Vetayases lock_set(&apix_lock); 17457ff178cdSJimmy Vetayases 17467ff178cdSJimmy Vetayases for (i = 0; i < count; i++, inum++) { 17477ff178cdSJimmy Vetayases if ((vecp = apix_get_dev_map(dip, inum, type)) == NULL) { 17487ff178cdSJimmy Vetayases lock_clear(&apix_lock); 17497ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: " 17507ff178cdSJimmy Vetayases "dip=0x%p inum=0x%x type=0x%x apix_find_intr() " 17517ff178cdSJimmy Vetayases "failed\n", (void *)dip, inum, type)); 17527ff178cdSJimmy Vetayases continue; 17537ff178cdSJimmy Vetayases } 17547ff178cdSJimmy Vetayases 17557ff178cdSJimmy Vetayases APIX_ENTER_CPU_LOCK(vecp->v_cpuid); 17567ff178cdSJimmy Vetayases cpuid = vecp->v_cpuid; 17577ff178cdSJimmy Vetayases 17587ff178cdSJimmy Vetayases DDI_INTR_IMPLDBG((CE_CONT, "apix_free_vectors: " 17597ff178cdSJimmy Vetayases "dip=0x%p inum=0x%x type=0x%x vector 0x%x (share %d)\n", 17607ff178cdSJimmy Vetayases (void *)dip, inum, type, vecp->v_vector, vecp->v_share)); 17617ff178cdSJimmy Vetayases 17627ff178cdSJimmy Vetayases /* tear down device interrupt to vector mapping */ 17637ff178cdSJimmy Vetayases apix_clear_dev_map(dip, inum, type); 17647ff178cdSJimmy Vetayases 17657ff178cdSJimmy Vetayases if (vecp->v_type == APIX_TYPE_FIXED) { 17667ff178cdSJimmy Vetayases if (vecp->v_share > 0) { /* share IRQ line */ 17677ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 17687ff178cdSJimmy Vetayases continue; 17697ff178cdSJimmy Vetayases } 17707ff178cdSJimmy Vetayases 17717ff178cdSJimmy Vetayases /* Free apic_irq_table entry */ 17727ff178cdSJimmy Vetayases apix_intx_free(vecp->v_inum); 17737ff178cdSJimmy Vetayases } 17747ff178cdSJimmy Vetayases 17757ff178cdSJimmy Vetayases /* free vector */ 17767ff178cdSJimmy Vetayases apix_cleanup_vector(vecp); 17777ff178cdSJimmy Vetayases 17787ff178cdSJimmy Vetayases APIX_LEAVE_CPU_LOCK(cpuid); 17797ff178cdSJimmy Vetayases } 17807ff178cdSJimmy Vetayases 17817ff178cdSJimmy Vetayases lock_clear(&apix_lock); 17827ff178cdSJimmy Vetayases } 17837ff178cdSJimmy Vetayases 17847ff178cdSJimmy Vetayases /* 17857ff178cdSJimmy Vetayases * Must be called with apix_lock held 17867ff178cdSJimmy Vetayases */ 17877ff178cdSJimmy Vetayases apix_vector_t * 17887ff178cdSJimmy Vetayases apix_setup_io_intr(apix_vector_t *vecp) 17897ff178cdSJimmy Vetayases { 17907ff178cdSJimmy Vetayases processorid_t bindcpu; 17917ff178cdSJimmy Vetayases int ret; 17927ff178cdSJimmy Vetayases 17937ff178cdSJimmy Vetayases ASSERT(LOCK_HELD(&apix_lock)); 17947ff178cdSJimmy Vetayases 17957ff178cdSJimmy Vetayases /* 17967ff178cdSJimmy Vetayases * Interrupts are enabled on the CPU, programme IOAPIC RDT 17977ff178cdSJimmy Vetayases * entry or MSI/X address/data to enable the interrupt. 17987ff178cdSJimmy Vetayases */ 17997ff178cdSJimmy Vetayases if (apix_is_cpu_enabled(vecp->v_cpuid)) { 18007ff178cdSJimmy Vetayases apix_enable_vector(vecp); 18017ff178cdSJimmy Vetayases return (vecp); 18027ff178cdSJimmy Vetayases } 18037ff178cdSJimmy Vetayases 18047ff178cdSJimmy Vetayases /* 18057ff178cdSJimmy Vetayases * CPU is not up or interrupts are disabled. Fall back to the 18067ff178cdSJimmy Vetayases * first avialable CPU. 18077ff178cdSJimmy Vetayases */ 18087ff178cdSJimmy Vetayases bindcpu = apic_find_cpu(APIC_CPU_INTR_ENABLE); 18097ff178cdSJimmy Vetayases 18107ff178cdSJimmy Vetayases if (vecp->v_type == APIX_TYPE_MSI) 18117ff178cdSJimmy Vetayases return (apix_grp_set_cpu(vecp, bindcpu, &ret)); 18127ff178cdSJimmy Vetayases 18137ff178cdSJimmy Vetayases return (apix_set_cpu(vecp, bindcpu, &ret)); 18147ff178cdSJimmy Vetayases } 18157ff178cdSJimmy Vetayases 18167ff178cdSJimmy Vetayases /* 18177ff178cdSJimmy Vetayases * For interrupts which call add_avintr() before apic is initialized. 18187ff178cdSJimmy Vetayases * ioapix_setup_intr() will 18197ff178cdSJimmy Vetayases * - allocate vector 18207ff178cdSJimmy Vetayases * - copy over ISR 18217ff178cdSJimmy Vetayases */ 18227ff178cdSJimmy Vetayases static void 18237ff178cdSJimmy Vetayases ioapix_setup_intr(int irqno, iflag_t *flagp) 18247ff178cdSJimmy Vetayases { 18257ff178cdSJimmy Vetayases extern struct av_head autovect[]; 18267ff178cdSJimmy Vetayases apix_vector_t *vecp; 18277ff178cdSJimmy Vetayases apic_irq_t *irqp; 18287ff178cdSJimmy Vetayases uchar_t ioapicindex, ipin; 18297ff178cdSJimmy Vetayases ulong_t iflag; 18307ff178cdSJimmy Vetayases struct autovec *avp; 18317ff178cdSJimmy Vetayases 18327ff178cdSJimmy Vetayases irqp = apic_irq_table[irqno]; 18337ff178cdSJimmy Vetayases ioapicindex = acpi_find_ioapic(irqno); 18347ff178cdSJimmy Vetayases ASSERT(ioapicindex != 0xFF); 18357ff178cdSJimmy Vetayases ipin = irqno - apic_io_vectbase[ioapicindex]; 18367ff178cdSJimmy Vetayases 18377ff178cdSJimmy Vetayases if ((irqp != NULL) && (irqp->airq_mps_intr_index == ACPI_INDEX)) { 18387ff178cdSJimmy Vetayases ASSERT(irqp->airq_intin_no == ipin && 18397ff178cdSJimmy Vetayases irqp->airq_ioapicindex == ioapicindex); 18407ff178cdSJimmy Vetayases vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector); 18417ff178cdSJimmy Vetayases ASSERT(!IS_VECT_FREE(vecp)); 18427ff178cdSJimmy Vetayases } else { 18437ff178cdSJimmy Vetayases vecp = apix_alloc_intx(NULL, 0, irqno); 18447ff178cdSJimmy Vetayases 18457ff178cdSJimmy Vetayases irqp = apic_irq_table[irqno]; 18467ff178cdSJimmy Vetayases irqp->airq_mps_intr_index = ACPI_INDEX; 18477ff178cdSJimmy Vetayases irqp->airq_ioapicindex = ioapicindex; 18487ff178cdSJimmy Vetayases irqp->airq_intin_no = ipin; 18497ff178cdSJimmy Vetayases irqp->airq_iflag = *flagp; 18507ff178cdSJimmy Vetayases irqp->airq_share++; 18517ff178cdSJimmy Vetayases apic_record_rdt_entry(irqp, irqno); 18527ff178cdSJimmy Vetayases } 18537ff178cdSJimmy Vetayases 18547ff178cdSJimmy Vetayases /* copy over autovect */ 18557ff178cdSJimmy Vetayases for (avp = autovect[irqno].avh_link; avp; avp = avp->av_link) 18567ff178cdSJimmy Vetayases apix_insert_av(vecp, avp->av_intr_id, avp->av_vector, 18577ff178cdSJimmy Vetayases avp->av_intarg1, avp->av_intarg2, avp->av_ticksp, 18587ff178cdSJimmy Vetayases avp->av_prilevel, avp->av_dip); 18597ff178cdSJimmy Vetayases 18607ff178cdSJimmy Vetayases /* Program I/O APIC */ 18617ff178cdSJimmy Vetayases iflag = intr_clear(); 18627ff178cdSJimmy Vetayases lock_set(&apix_lock); 18637ff178cdSJimmy Vetayases 18647ff178cdSJimmy Vetayases (void) apix_setup_io_intr(vecp); 18657ff178cdSJimmy Vetayases 18667ff178cdSJimmy Vetayases lock_clear(&apix_lock); 18677ff178cdSJimmy Vetayases intr_restore(iflag); 18687ff178cdSJimmy Vetayases 18697ff178cdSJimmy Vetayases APIC_VERBOSE_IOAPIC((CE_CONT, "apix: setup ioapic, irqno %x " 18707ff178cdSJimmy Vetayases "(ioapic %x, ipin %x) is bound to cpu %x, vector %x\n", 18717ff178cdSJimmy Vetayases irqno, ioapicindex, ipin, irqp->airq_cpu, irqp->airq_vector)); 18727ff178cdSJimmy Vetayases } 18737ff178cdSJimmy Vetayases 18747ff178cdSJimmy Vetayases void 18757ff178cdSJimmy Vetayases ioapix_init_intr(int mask_apic) 18767ff178cdSJimmy Vetayases { 18777ff178cdSJimmy Vetayases int ioapicindex; 18787ff178cdSJimmy Vetayases int i, j; 18797ff178cdSJimmy Vetayases 18807ff178cdSJimmy Vetayases /* mask interrupt vectors */ 18817ff178cdSJimmy Vetayases for (j = 0; j < apic_io_max && mask_apic; j++) { 18827ff178cdSJimmy Vetayases int intin_max; 18837ff178cdSJimmy Vetayases 18847ff178cdSJimmy Vetayases ioapicindex = j; 18857ff178cdSJimmy Vetayases /* Bits 23-16 define the maximum redirection entries */ 18867ff178cdSJimmy Vetayases intin_max = (ioapic_read(ioapicindex, APIC_VERS_CMD) >> 16) 18877ff178cdSJimmy Vetayases & 0xff; 18887ff178cdSJimmy Vetayases for (i = 0; i <= intin_max; i++) 18897ff178cdSJimmy Vetayases ioapic_write(ioapicindex, APIC_RDT_CMD + 2 * i, 18907ff178cdSJimmy Vetayases AV_MASK); 18917ff178cdSJimmy Vetayases } 18927ff178cdSJimmy Vetayases 18937ff178cdSJimmy Vetayases /* 18947ff178cdSJimmy Vetayases * Hack alert: deal with ACPI SCI interrupt chicken/egg here 18957ff178cdSJimmy Vetayases */ 18967ff178cdSJimmy Vetayases if (apic_sci_vect > 0) 18977ff178cdSJimmy Vetayases ioapix_setup_intr(apic_sci_vect, &apic_sci_flags); 18987ff178cdSJimmy Vetayases 18997ff178cdSJimmy Vetayases /* 19007ff178cdSJimmy Vetayases * Hack alert: deal with ACPI HPET interrupt chicken/egg here. 19017ff178cdSJimmy Vetayases */ 19027ff178cdSJimmy Vetayases if (apic_hpet_vect > 0) 19037ff178cdSJimmy Vetayases ioapix_setup_intr(apic_hpet_vect, &apic_hpet_flags); 19047ff178cdSJimmy Vetayases } 1905