1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org> 6 * Copyright (c) 2000, BSDi 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_bus.h" 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/malloc.h> 39 #include <sys/module.h> 40 #include <sys/limits.h> 41 #include <sys/linker.h> 42 #include <sys/fcntl.h> 43 #include <sys/conf.h> 44 #include <sys/kernel.h> 45 #include <sys/queue.h> 46 #include <sys/sysctl.h> 47 #include <sys/endian.h> 48 49 #include <vm/vm.h> 50 #include <vm/pmap.h> 51 #include <vm/vm_extern.h> 52 53 #include <sys/bus.h> 54 #include <machine/bus.h> 55 #include <sys/rman.h> 56 #include <machine/resource.h> 57 #include <machine/stdarg.h> 58 59 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) 60 #include <machine/intr_machdep.h> 61 #endif 62 63 #include <sys/pciio.h> 64 #include <dev/pci/pcireg.h> 65 #include <dev/pci/pcivar.h> 66 #include <dev/pci/pci_private.h> 67 68 #ifdef PCI_IOV 69 #include <sys/nv.h> 70 #include <dev/pci/pci_iov_private.h> 71 #endif 72 73 #include <dev/usb/controller/xhcireg.h> 74 #include <dev/usb/controller/ehcireg.h> 75 #include <dev/usb/controller/ohcireg.h> 76 #include <dev/usb/controller/uhcireg.h> 77 78 #include "pcib_if.h" 79 #include "pci_if.h" 80 81 #define PCIR_IS_BIOS(cfg, reg) \ 82 (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \ 83 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1)) 84 85 static int pci_has_quirk(uint32_t devid, int quirk); 86 static pci_addr_t pci_mapbase(uint64_t mapreg); 87 static const char *pci_maptype(uint64_t mapreg); 88 static int pci_maprange(uint64_t mapreg); 89 static pci_addr_t pci_rombase(uint64_t mapreg); 90 static int pci_romsize(uint64_t testval); 91 static void pci_fixancient(pcicfgregs *cfg); 92 static int pci_printf(pcicfgregs *cfg, const char *fmt, ...); 93 94 static int pci_porten(device_t dev); 95 static int pci_memen(device_t dev); 96 static void pci_assign_interrupt(device_t bus, device_t dev, 97 int force_route); 98 static int pci_add_map(device_t bus, device_t dev, int reg, 99 struct resource_list *rl, int force, int prefetch); 100 static int pci_probe(device_t dev); 101 static int pci_attach(device_t dev); 102 static int pci_detach(device_t dev); 103 static void pci_load_vendor_data(void); 104 static int pci_describe_parse_line(char **ptr, int *vendor, 105 int *device, char **desc); 106 static char *pci_describe_device(device_t dev); 107 static int pci_modevent(module_t mod, int what, void *arg); 108 static void pci_hdrtypedata(device_t pcib, int b, int s, int f, 109 pcicfgregs *cfg); 110 static void pci_read_cap(device_t pcib, pcicfgregs *cfg); 111 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, 112 int reg, uint32_t *data); 113 #if 0 114 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, 115 int reg, uint32_t data); 116 #endif 117 static void pci_read_vpd(device_t pcib, pcicfgregs *cfg); 118 static void pci_mask_msix(device_t dev, u_int index); 119 static void pci_unmask_msix(device_t dev, u_int index); 120 static int pci_msi_blacklisted(void); 121 static int pci_msix_blacklisted(void); 122 static void pci_resume_msi(device_t dev); 123 static void pci_resume_msix(device_t dev); 124 static int pci_remap_intr_method(device_t bus, device_t dev, 125 u_int irq); 126 static void pci_hint_device_unit(device_t acdev, device_t child, 127 const char *name, int *unitp); 128 129 static int pci_get_id_method(device_t dev, device_t child, 130 enum pci_id_type type, uintptr_t *rid); 131 132 static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d, 133 int b, int s, int f, uint16_t vid, uint16_t did); 134 135 static device_method_t pci_methods[] = { 136 /* Device interface */ 137 DEVMETHOD(device_probe, pci_probe), 138 DEVMETHOD(device_attach, pci_attach), 139 DEVMETHOD(device_detach, pci_detach), 140 DEVMETHOD(device_shutdown, bus_generic_shutdown), 141 DEVMETHOD(device_suspend, bus_generic_suspend), 142 DEVMETHOD(device_resume, pci_resume), 143 144 /* Bus interface */ 145 DEVMETHOD(bus_print_child, pci_print_child), 146 DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch), 147 DEVMETHOD(bus_read_ivar, pci_read_ivar), 148 DEVMETHOD(bus_write_ivar, pci_write_ivar), 149 DEVMETHOD(bus_driver_added, pci_driver_added), 150 DEVMETHOD(bus_setup_intr, pci_setup_intr), 151 DEVMETHOD(bus_teardown_intr, pci_teardown_intr), 152 153 DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag), 154 DEVMETHOD(bus_get_resource_list,pci_get_resource_list), 155 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), 156 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), 157 DEVMETHOD(bus_delete_resource, pci_delete_resource), 158 DEVMETHOD(bus_alloc_resource, pci_alloc_resource), 159 DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource), 160 DEVMETHOD(bus_release_resource, pci_release_resource), 161 DEVMETHOD(bus_activate_resource, pci_activate_resource), 162 DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource), 163 DEVMETHOD(bus_child_deleted, pci_child_deleted), 164 DEVMETHOD(bus_child_detached, pci_child_detached), 165 DEVMETHOD(bus_child_pnpinfo_str, pci_child_pnpinfo_str_method), 166 DEVMETHOD(bus_child_location_str, pci_child_location_str_method), 167 DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit), 168 DEVMETHOD(bus_remap_intr, pci_remap_intr_method), 169 DEVMETHOD(bus_suspend_child, pci_suspend_child), 170 DEVMETHOD(bus_resume_child, pci_resume_child), 171 DEVMETHOD(bus_rescan, pci_rescan_method), 172 173 /* PCI interface */ 174 DEVMETHOD(pci_read_config, pci_read_config_method), 175 DEVMETHOD(pci_write_config, pci_write_config_method), 176 DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method), 177 DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method), 178 DEVMETHOD(pci_enable_io, pci_enable_io_method), 179 DEVMETHOD(pci_disable_io, pci_disable_io_method), 180 DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method), 181 DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method), 182 DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method), 183 DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method), 184 DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method), 185 DEVMETHOD(pci_find_cap, pci_find_cap_method), 186 DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method), 187 DEVMETHOD(pci_find_extcap, pci_find_extcap_method), 188 DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method), 189 DEVMETHOD(pci_find_htcap, pci_find_htcap_method), 190 DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method), 191 DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method), 192 DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method), 193 DEVMETHOD(pci_enable_msi, pci_enable_msi_method), 194 DEVMETHOD(pci_enable_msix, pci_enable_msix_method), 195 DEVMETHOD(pci_disable_msi, pci_disable_msi_method), 196 DEVMETHOD(pci_remap_msix, pci_remap_msix_method), 197 DEVMETHOD(pci_release_msi, pci_release_msi_method), 198 DEVMETHOD(pci_msi_count, pci_msi_count_method), 199 DEVMETHOD(pci_msix_count, pci_msix_count_method), 200 DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method), 201 DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method), 202 DEVMETHOD(pci_get_id, pci_get_id_method), 203 DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method), 204 DEVMETHOD(pci_child_added, pci_child_added_method), 205 #ifdef PCI_IOV 206 DEVMETHOD(pci_iov_attach, pci_iov_attach_method), 207 DEVMETHOD(pci_iov_detach, pci_iov_detach_method), 208 DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method), 209 #endif 210 211 DEVMETHOD_END 212 }; 213 214 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc)); 215 216 static devclass_t pci_devclass; 217 EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_devclass, pci_modevent, NULL, 218 BUS_PASS_BUS); 219 MODULE_VERSION(pci, 1); 220 221 static char *pci_vendordata; 222 static size_t pci_vendordata_size; 223 224 struct pci_quirk { 225 uint32_t devid; /* Vendor/device of the card */ 226 int type; 227 #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */ 228 #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */ 229 #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */ 230 #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */ 231 #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */ 232 #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */ 233 #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */ 234 int arg1; 235 int arg2; 236 }; 237 238 static const struct pci_quirk pci_quirks[] = { 239 /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */ 240 { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 }, 241 { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 }, 242 /* As does the Serverworks OSB4 (the SMBus mapping register) */ 243 { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 }, 244 245 /* 246 * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge 247 * or the CMIC-SL (AKA ServerWorks GC_LE). 248 */ 249 { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 250 { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 251 252 /* 253 * MSI doesn't work on earlier Intel chipsets including 254 * E7500, E7501, E7505, 845, 865, 875/E7210, and 855. 255 */ 256 { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 257 { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 258 { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 259 { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 260 { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 261 { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 262 { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 263 264 /* 265 * MSI doesn't work with devices behind the AMD 8131 HT-PCIX 266 * bridge. 267 */ 268 { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 }, 269 270 /* 271 * MSI-X allocation doesn't work properly for devices passed through 272 * by VMware up to at least ESXi 5.1. 273 */ 274 { 0x079015ad, PCI_QUIRK_DISABLE_MSIX, 0, 0 }, /* PCI/PCI-X */ 275 { 0x07a015ad, PCI_QUIRK_DISABLE_MSIX, 0, 0 }, /* PCIe */ 276 277 /* 278 * Some virtualization environments emulate an older chipset 279 * but support MSI just fine. QEMU uses the Intel 82440. 280 */ 281 { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 }, 282 283 /* 284 * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus 285 * controller depending on SoftPciRst register (PM_IO 0x55 [7]). 286 * It prevents us from attaching hpet(4) when the bit is unset. 287 * Note this quirk only affects SB600 revision A13 and earlier. 288 * For SB600 A21 and later, firmware must set the bit to hide it. 289 * For SB700 and later, it is unused and hardcoded to zero. 290 */ 291 { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 }, 292 293 /* 294 * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have 295 * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit 296 * of the command register is set. 297 */ 298 { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, 299 { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, 300 { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, 301 { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, 302 { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, 303 304 /* 305 * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't 306 * issue MSI interrupts with PCIM_CMD_INTxDIS set either. 307 */ 308 { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */ 309 { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */ 310 { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */ 311 { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */ 312 { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */ 313 { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */ 314 315 /* 316 * HPE Gen 10 VGA has a memory range that can't be allocated in the 317 * expected place. 318 */ 319 { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 }, 320 321 { 0 } 322 }; 323 324 /* map register information */ 325 #define PCI_MAPMEM 0x01 /* memory map */ 326 #define PCI_MAPMEMP 0x02 /* prefetchable memory map */ 327 #define PCI_MAPPORT 0x04 /* port map */ 328 329 struct devlist pci_devq; 330 uint32_t pci_generation; 331 uint32_t pci_numdevs = 0; 332 static int pcie_chipset, pcix_chipset; 333 334 /* sysctl vars */ 335 SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD, 0, "PCI bus tuning parameters"); 336 337 static int pci_enable_io_modes = 1; 338 SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN, 339 &pci_enable_io_modes, 1, 340 "Enable I/O and memory bits in the config register. Some BIOSes do not" 341 " enable these bits correctly. We'd like to do this all the time, but" 342 " there are some peripherals that this causes problems with."); 343 344 static int pci_do_realloc_bars = 1; 345 SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN, 346 &pci_do_realloc_bars, 0, 347 "Attempt to allocate a new range for any BARs whose original " 348 "firmware-assigned ranges fail to allocate during the initial device scan."); 349 350 static int pci_do_power_nodriver = 0; 351 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN, 352 &pci_do_power_nodriver, 0, 353 "Place a function into D3 state when no driver attaches to it. 0 means" 354 " disable. 1 means conservatively place devices into D3 state. 2 means" 355 " aggressively place devices into D3 state. 3 means put absolutely" 356 " everything in D3 state."); 357 358 int pci_do_power_resume = 1; 359 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN, 360 &pci_do_power_resume, 1, 361 "Transition from D3 -> D0 on resume."); 362 363 int pci_do_power_suspend = 1; 364 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN, 365 &pci_do_power_suspend, 1, 366 "Transition from D0 -> D3 on suspend."); 367 368 static int pci_do_msi = 1; 369 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1, 370 "Enable support for MSI interrupts"); 371 372 static int pci_do_msix = 1; 373 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1, 374 "Enable support for MSI-X interrupts"); 375 376 static int pci_msix_rewrite_table = 0; 377 SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN, 378 &pci_msix_rewrite_table, 0, 379 "Rewrite entire MSI-X table when updating MSI-X entries"); 380 381 static int pci_honor_msi_blacklist = 1; 382 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN, 383 &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X"); 384 385 #if defined(__i386__) || defined(__amd64__) 386 static int pci_usb_takeover = 1; 387 #else 388 static int pci_usb_takeover = 0; 389 #endif 390 SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN, 391 &pci_usb_takeover, 1, 392 "Enable early takeover of USB controllers. Disable this if you depend on" 393 " BIOS emulation of USB devices, that is you use USB devices (like" 394 " keyboard or mouse) but do not load USB drivers"); 395 396 static int pci_clear_bars; 397 SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0, 398 "Ignore firmware-assigned resources for BARs."); 399 400 #if defined(NEW_PCIB) && defined(PCI_RES_BUS) 401 static int pci_clear_buses; 402 SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0, 403 "Ignore firmware-assigned bus numbers."); 404 #endif 405 406 static int pci_enable_ari = 1; 407 SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari, 408 0, "Enable support for PCIe Alternative RID Interpretation"); 409 410 static int pci_clear_aer_on_attach = 0; 411 SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN, 412 &pci_clear_aer_on_attach, 0, 413 "Clear port and device AER state on driver attach"); 414 415 static int 416 pci_has_quirk(uint32_t devid, int quirk) 417 { 418 const struct pci_quirk *q; 419 420 for (q = &pci_quirks[0]; q->devid; q++) { 421 if (q->devid == devid && q->type == quirk) 422 return (1); 423 } 424 return (0); 425 } 426 427 /* Find a device_t by bus/slot/function in domain 0 */ 428 429 device_t 430 pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func) 431 { 432 433 return (pci_find_dbsf(0, bus, slot, func)); 434 } 435 436 /* Find a device_t by domain/bus/slot/function */ 437 438 device_t 439 pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func) 440 { 441 struct pci_devinfo *dinfo; 442 443 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { 444 if ((dinfo->cfg.domain == domain) && 445 (dinfo->cfg.bus == bus) && 446 (dinfo->cfg.slot == slot) && 447 (dinfo->cfg.func == func)) { 448 return (dinfo->cfg.dev); 449 } 450 } 451 452 return (NULL); 453 } 454 455 /* Find a device_t by vendor/device ID */ 456 457 device_t 458 pci_find_device(uint16_t vendor, uint16_t device) 459 { 460 struct pci_devinfo *dinfo; 461 462 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { 463 if ((dinfo->cfg.vendor == vendor) && 464 (dinfo->cfg.device == device)) { 465 return (dinfo->cfg.dev); 466 } 467 } 468 469 return (NULL); 470 } 471 472 device_t 473 pci_find_class(uint8_t class, uint8_t subclass) 474 { 475 struct pci_devinfo *dinfo; 476 477 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { 478 if (dinfo->cfg.baseclass == class && 479 dinfo->cfg.subclass == subclass) { 480 return (dinfo->cfg.dev); 481 } 482 } 483 484 return (NULL); 485 } 486 487 static int 488 pci_printf(pcicfgregs *cfg, const char *fmt, ...) 489 { 490 va_list ap; 491 int retval; 492 493 retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot, 494 cfg->func); 495 va_start(ap, fmt); 496 retval += vprintf(fmt, ap); 497 va_end(ap); 498 return (retval); 499 } 500 501 /* return base address of memory or port map */ 502 503 static pci_addr_t 504 pci_mapbase(uint64_t mapreg) 505 { 506 507 if (PCI_BAR_MEM(mapreg)) 508 return (mapreg & PCIM_BAR_MEM_BASE); 509 else 510 return (mapreg & PCIM_BAR_IO_BASE); 511 } 512 513 /* return map type of memory or port map */ 514 515 static const char * 516 pci_maptype(uint64_t mapreg) 517 { 518 519 if (PCI_BAR_IO(mapreg)) 520 return ("I/O Port"); 521 if (mapreg & PCIM_BAR_MEM_PREFETCH) 522 return ("Prefetchable Memory"); 523 return ("Memory"); 524 } 525 526 /* return log2 of map size decoded for memory or port map */ 527 528 int 529 pci_mapsize(uint64_t testval) 530 { 531 int ln2size; 532 533 testval = pci_mapbase(testval); 534 ln2size = 0; 535 if (testval != 0) { 536 while ((testval & 1) == 0) 537 { 538 ln2size++; 539 testval >>= 1; 540 } 541 } 542 return (ln2size); 543 } 544 545 /* return base address of device ROM */ 546 547 static pci_addr_t 548 pci_rombase(uint64_t mapreg) 549 { 550 551 return (mapreg & PCIM_BIOS_ADDR_MASK); 552 } 553 554 /* return log2 of map size decided for device ROM */ 555 556 static int 557 pci_romsize(uint64_t testval) 558 { 559 int ln2size; 560 561 testval = pci_rombase(testval); 562 ln2size = 0; 563 if (testval != 0) { 564 while ((testval & 1) == 0) 565 { 566 ln2size++; 567 testval >>= 1; 568 } 569 } 570 return (ln2size); 571 } 572 573 /* return log2 of address range supported by map register */ 574 575 static int 576 pci_maprange(uint64_t mapreg) 577 { 578 int ln2range = 0; 579 580 if (PCI_BAR_IO(mapreg)) 581 ln2range = 32; 582 else 583 switch (mapreg & PCIM_BAR_MEM_TYPE) { 584 case PCIM_BAR_MEM_32: 585 ln2range = 32; 586 break; 587 case PCIM_BAR_MEM_1MB: 588 ln2range = 20; 589 break; 590 case PCIM_BAR_MEM_64: 591 ln2range = 64; 592 break; 593 } 594 return (ln2range); 595 } 596 597 /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */ 598 599 static void 600 pci_fixancient(pcicfgregs *cfg) 601 { 602 if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL) 603 return; 604 605 /* PCI to PCI bridges use header type 1 */ 606 if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI) 607 cfg->hdrtype = PCIM_HDRTYPE_BRIDGE; 608 } 609 610 /* extract header type specific config data */ 611 612 static void 613 pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg) 614 { 615 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w) 616 switch (cfg->hdrtype & PCIM_HDRTYPE) { 617 case PCIM_HDRTYPE_NORMAL: 618 cfg->subvendor = REG(PCIR_SUBVEND_0, 2); 619 cfg->subdevice = REG(PCIR_SUBDEV_0, 2); 620 cfg->mingnt = REG(PCIR_MINGNT, 1); 621 cfg->maxlat = REG(PCIR_MAXLAT, 1); 622 cfg->nummaps = PCI_MAXMAPS_0; 623 break; 624 case PCIM_HDRTYPE_BRIDGE: 625 cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1); 626 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1); 627 cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1); 628 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1); 629 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2); 630 cfg->nummaps = PCI_MAXMAPS_1; 631 break; 632 case PCIM_HDRTYPE_CARDBUS: 633 cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1); 634 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1); 635 cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1); 636 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1); 637 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2); 638 cfg->subvendor = REG(PCIR_SUBVEND_2, 2); 639 cfg->subdevice = REG(PCIR_SUBDEV_2, 2); 640 cfg->nummaps = PCI_MAXMAPS_2; 641 break; 642 } 643 #undef REG 644 } 645 646 /* read configuration header into pcicfgregs structure */ 647 struct pci_devinfo * 648 pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f) 649 { 650 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w) 651 uint16_t vid, did; 652 653 vid = REG(PCIR_VENDOR, 2); 654 did = REG(PCIR_DEVICE, 2); 655 if (vid != 0xffff) 656 return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did)); 657 658 return (NULL); 659 } 660 661 struct pci_devinfo * 662 pci_alloc_devinfo_method(device_t dev) 663 { 664 665 return (malloc(sizeof(struct pci_devinfo), M_DEVBUF, 666 M_WAITOK | M_ZERO)); 667 } 668 669 static struct pci_devinfo * 670 pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f, 671 uint16_t vid, uint16_t did) 672 { 673 struct pci_devinfo *devlist_entry; 674 pcicfgregs *cfg; 675 676 devlist_entry = PCI_ALLOC_DEVINFO(bus); 677 678 cfg = &devlist_entry->cfg; 679 680 cfg->domain = d; 681 cfg->bus = b; 682 cfg->slot = s; 683 cfg->func = f; 684 cfg->vendor = vid; 685 cfg->device = did; 686 cfg->cmdreg = REG(PCIR_COMMAND, 2); 687 cfg->statreg = REG(PCIR_STATUS, 2); 688 cfg->baseclass = REG(PCIR_CLASS, 1); 689 cfg->subclass = REG(PCIR_SUBCLASS, 1); 690 cfg->progif = REG(PCIR_PROGIF, 1); 691 cfg->revid = REG(PCIR_REVID, 1); 692 cfg->hdrtype = REG(PCIR_HDRTYPE, 1); 693 cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1); 694 cfg->lattimer = REG(PCIR_LATTIMER, 1); 695 cfg->intpin = REG(PCIR_INTPIN, 1); 696 cfg->intline = REG(PCIR_INTLINE, 1); 697 698 cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0; 699 cfg->hdrtype &= ~PCIM_MFDEV; 700 STAILQ_INIT(&cfg->maps); 701 702 cfg->iov = NULL; 703 704 pci_fixancient(cfg); 705 pci_hdrtypedata(pcib, b, s, f, cfg); 706 707 if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT) 708 pci_read_cap(pcib, cfg); 709 710 STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links); 711 712 devlist_entry->conf.pc_sel.pc_domain = cfg->domain; 713 devlist_entry->conf.pc_sel.pc_bus = cfg->bus; 714 devlist_entry->conf.pc_sel.pc_dev = cfg->slot; 715 devlist_entry->conf.pc_sel.pc_func = cfg->func; 716 devlist_entry->conf.pc_hdr = cfg->hdrtype; 717 718 devlist_entry->conf.pc_subvendor = cfg->subvendor; 719 devlist_entry->conf.pc_subdevice = cfg->subdevice; 720 devlist_entry->conf.pc_vendor = cfg->vendor; 721 devlist_entry->conf.pc_device = cfg->device; 722 723 devlist_entry->conf.pc_class = cfg->baseclass; 724 devlist_entry->conf.pc_subclass = cfg->subclass; 725 devlist_entry->conf.pc_progif = cfg->progif; 726 devlist_entry->conf.pc_revid = cfg->revid; 727 728 pci_numdevs++; 729 pci_generation++; 730 731 return (devlist_entry); 732 } 733 #undef REG 734 735 static void 736 pci_ea_fill_info(device_t pcib, pcicfgregs *cfg) 737 { 738 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \ 739 cfg->ea.ea_location + (n), w) 740 int num_ent; 741 int ptr; 742 int a, b; 743 uint32_t val; 744 int ent_size; 745 uint32_t dw[4]; 746 uint64_t base, max_offset; 747 struct pci_ea_entry *eae; 748 749 if (cfg->ea.ea_location == 0) 750 return; 751 752 STAILQ_INIT(&cfg->ea.ea_entries); 753 754 /* Determine the number of entries */ 755 num_ent = REG(PCIR_EA_NUM_ENT, 2); 756 num_ent &= PCIM_EA_NUM_ENT_MASK; 757 758 /* Find the first entry to care of */ 759 ptr = PCIR_EA_FIRST_ENT; 760 761 /* Skip DWORD 2 for type 1 functions */ 762 if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE) 763 ptr += 4; 764 765 for (a = 0; a < num_ent; a++) { 766 767 eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO); 768 eae->eae_cfg_offset = cfg->ea.ea_location + ptr; 769 770 /* Read a number of dwords in the entry */ 771 val = REG(ptr, 4); 772 ptr += 4; 773 ent_size = (val & PCIM_EA_ES); 774 775 for (b = 0; b < ent_size; b++) { 776 dw[b] = REG(ptr, 4); 777 ptr += 4; 778 } 779 780 eae->eae_flags = val; 781 eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET; 782 783 base = dw[0] & PCIM_EA_FIELD_MASK; 784 max_offset = dw[1] | ~PCIM_EA_FIELD_MASK; 785 b = 2; 786 if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) { 787 base |= (uint64_t)dw[b] << 32UL; 788 b++; 789 } 790 if (((dw[1] & PCIM_EA_IS_64) != 0) 791 && (b < ent_size)) { 792 max_offset |= (uint64_t)dw[b] << 32UL; 793 b++; 794 } 795 796 eae->eae_base = base; 797 eae->eae_max_offset = max_offset; 798 799 STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link); 800 801 if (bootverbose) { 802 printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n", 803 cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags, 804 (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset); 805 } 806 } 807 } 808 #undef REG 809 810 static void 811 pci_read_cap(device_t pcib, pcicfgregs *cfg) 812 { 813 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w) 814 #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w) 815 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) 816 uint64_t addr; 817 #endif 818 uint32_t val; 819 int ptr, nextptr, ptrptr; 820 821 switch (cfg->hdrtype & PCIM_HDRTYPE) { 822 case PCIM_HDRTYPE_NORMAL: 823 case PCIM_HDRTYPE_BRIDGE: 824 ptrptr = PCIR_CAP_PTR; 825 break; 826 case PCIM_HDRTYPE_CARDBUS: 827 ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */ 828 break; 829 default: 830 return; /* no extended capabilities support */ 831 } 832 nextptr = REG(ptrptr, 1); /* sanity check? */ 833 834 /* 835 * Read capability entries. 836 */ 837 while (nextptr != 0) { 838 /* Sanity check */ 839 if (nextptr > 255) { 840 printf("illegal PCI extended capability offset %d\n", 841 nextptr); 842 return; 843 } 844 /* Find the next entry */ 845 ptr = nextptr; 846 nextptr = REG(ptr + PCICAP_NEXTPTR, 1); 847 848 /* Process this entry */ 849 switch (REG(ptr + PCICAP_ID, 1)) { 850 case PCIY_PMG: /* PCI power management */ 851 if (cfg->pp.pp_cap == 0) { 852 cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2); 853 cfg->pp.pp_status = ptr + PCIR_POWER_STATUS; 854 cfg->pp.pp_bse = ptr + PCIR_POWER_BSE; 855 if ((nextptr - ptr) > PCIR_POWER_DATA) 856 cfg->pp.pp_data = ptr + PCIR_POWER_DATA; 857 } 858 break; 859 case PCIY_HT: /* HyperTransport */ 860 /* Determine HT-specific capability type. */ 861 val = REG(ptr + PCIR_HT_COMMAND, 2); 862 863 if ((val & 0xe000) == PCIM_HTCAP_SLAVE) 864 cfg->ht.ht_slave = ptr; 865 866 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) 867 switch (val & PCIM_HTCMD_CAP_MASK) { 868 case PCIM_HTCAP_MSI_MAPPING: 869 if (!(val & PCIM_HTCMD_MSI_FIXED)) { 870 /* Sanity check the mapping window. */ 871 addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI, 872 4); 873 addr <<= 32; 874 addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO, 875 4); 876 if (addr != MSI_INTEL_ADDR_BASE) 877 device_printf(pcib, 878 "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n", 879 cfg->domain, cfg->bus, 880 cfg->slot, cfg->func, 881 (long long)addr); 882 } else 883 addr = MSI_INTEL_ADDR_BASE; 884 885 cfg->ht.ht_msimap = ptr; 886 cfg->ht.ht_msictrl = val; 887 cfg->ht.ht_msiaddr = addr; 888 break; 889 } 890 #endif 891 break; 892 case PCIY_MSI: /* PCI MSI */ 893 cfg->msi.msi_location = ptr; 894 cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2); 895 cfg->msi.msi_msgnum = 1 << ((cfg->msi.msi_ctrl & 896 PCIM_MSICTRL_MMC_MASK)>>1); 897 break; 898 case PCIY_MSIX: /* PCI MSI-X */ 899 cfg->msix.msix_location = ptr; 900 cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2); 901 cfg->msix.msix_msgnum = (cfg->msix.msix_ctrl & 902 PCIM_MSIXCTRL_TABLE_SIZE) + 1; 903 val = REG(ptr + PCIR_MSIX_TABLE, 4); 904 cfg->msix.msix_table_bar = PCIR_BAR(val & 905 PCIM_MSIX_BIR_MASK); 906 cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK; 907 val = REG(ptr + PCIR_MSIX_PBA, 4); 908 cfg->msix.msix_pba_bar = PCIR_BAR(val & 909 PCIM_MSIX_BIR_MASK); 910 cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK; 911 break; 912 case PCIY_VPD: /* PCI Vital Product Data */ 913 cfg->vpd.vpd_reg = ptr; 914 break; 915 case PCIY_SUBVENDOR: 916 /* Should always be true. */ 917 if ((cfg->hdrtype & PCIM_HDRTYPE) == 918 PCIM_HDRTYPE_BRIDGE) { 919 val = REG(ptr + PCIR_SUBVENDCAP_ID, 4); 920 cfg->subvendor = val & 0xffff; 921 cfg->subdevice = val >> 16; 922 } 923 break; 924 case PCIY_PCIX: /* PCI-X */ 925 /* 926 * Assume we have a PCI-X chipset if we have 927 * at least one PCI-PCI bridge with a PCI-X 928 * capability. Note that some systems with 929 * PCI-express or HT chipsets might match on 930 * this check as well. 931 */ 932 if ((cfg->hdrtype & PCIM_HDRTYPE) == 933 PCIM_HDRTYPE_BRIDGE) 934 pcix_chipset = 1; 935 cfg->pcix.pcix_location = ptr; 936 break; 937 case PCIY_EXPRESS: /* PCI-express */ 938 /* 939 * Assume we have a PCI-express chipset if we have 940 * at least one PCI-express device. 941 */ 942 pcie_chipset = 1; 943 cfg->pcie.pcie_location = ptr; 944 val = REG(ptr + PCIER_FLAGS, 2); 945 cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE; 946 break; 947 case PCIY_EA: /* Enhanced Allocation */ 948 cfg->ea.ea_location = ptr; 949 pci_ea_fill_info(pcib, cfg); 950 break; 951 default: 952 break; 953 } 954 } 955 956 #if defined(__powerpc__) 957 /* 958 * Enable the MSI mapping window for all HyperTransport 959 * slaves. PCI-PCI bridges have their windows enabled via 960 * PCIB_MAP_MSI(). 961 */ 962 if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 && 963 !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) { 964 device_printf(pcib, 965 "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n", 966 cfg->domain, cfg->bus, cfg->slot, cfg->func); 967 cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE; 968 WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl, 969 2); 970 } 971 #endif 972 /* REG and WREG use carry through to next functions */ 973 } 974 975 /* 976 * PCI Vital Product Data 977 */ 978 979 #define PCI_VPD_TIMEOUT 1000000 980 981 static int 982 pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data) 983 { 984 int count = PCI_VPD_TIMEOUT; 985 986 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned")); 987 988 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2); 989 990 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) { 991 if (--count < 0) 992 return (ENXIO); 993 DELAY(1); /* limit looping */ 994 } 995 *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4)); 996 997 return (0); 998 } 999 1000 #if 0 1001 static int 1002 pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data) 1003 { 1004 int count = PCI_VPD_TIMEOUT; 1005 1006 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned")); 1007 1008 WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4); 1009 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2); 1010 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) { 1011 if (--count < 0) 1012 return (ENXIO); 1013 DELAY(1); /* limit looping */ 1014 } 1015 1016 return (0); 1017 } 1018 #endif 1019 1020 #undef PCI_VPD_TIMEOUT 1021 1022 struct vpd_readstate { 1023 device_t pcib; 1024 pcicfgregs *cfg; 1025 uint32_t val; 1026 int bytesinval; 1027 int off; 1028 uint8_t cksum; 1029 }; 1030 1031 static int 1032 vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data) 1033 { 1034 uint32_t reg; 1035 uint8_t byte; 1036 1037 if (vrs->bytesinval == 0) { 1038 if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®)) 1039 return (ENXIO); 1040 vrs->val = le32toh(reg); 1041 vrs->off += 4; 1042 byte = vrs->val & 0xff; 1043 vrs->bytesinval = 3; 1044 } else { 1045 vrs->val = vrs->val >> 8; 1046 byte = vrs->val & 0xff; 1047 vrs->bytesinval--; 1048 } 1049 1050 vrs->cksum += byte; 1051 *data = byte; 1052 return (0); 1053 } 1054 1055 static void 1056 pci_read_vpd(device_t pcib, pcicfgregs *cfg) 1057 { 1058 struct vpd_readstate vrs; 1059 int state; 1060 int name; 1061 int remain; 1062 int i; 1063 int alloc, off; /* alloc/off for RO/W arrays */ 1064 int cksumvalid; 1065 int dflen; 1066 uint8_t byte; 1067 uint8_t byte2; 1068 1069 /* init vpd reader */ 1070 vrs.bytesinval = 0; 1071 vrs.off = 0; 1072 vrs.pcib = pcib; 1073 vrs.cfg = cfg; 1074 vrs.cksum = 0; 1075 1076 state = 0; 1077 name = remain = i = 0; /* shut up stupid gcc */ 1078 alloc = off = 0; /* shut up stupid gcc */ 1079 dflen = 0; /* shut up stupid gcc */ 1080 cksumvalid = -1; 1081 while (state >= 0) { 1082 if (vpd_nextbyte(&vrs, &byte)) { 1083 state = -2; 1084 break; 1085 } 1086 #if 0 1087 printf("vpd: val: %#x, off: %d, bytesinval: %d, byte: %#hhx, " \ 1088 "state: %d, remain: %d, name: %#x, i: %d\n", vrs.val, 1089 vrs.off, vrs.bytesinval, byte, state, remain, name, i); 1090 #endif 1091 switch (state) { 1092 case 0: /* item name */ 1093 if (byte & 0x80) { 1094 if (vpd_nextbyte(&vrs, &byte2)) { 1095 state = -2; 1096 break; 1097 } 1098 remain = byte2; 1099 if (vpd_nextbyte(&vrs, &byte2)) { 1100 state = -2; 1101 break; 1102 } 1103 remain |= byte2 << 8; 1104 if (remain > (0x7f*4 - vrs.off)) { 1105 state = -1; 1106 pci_printf(cfg, 1107 "invalid VPD data, remain %#x\n", 1108 remain); 1109 } 1110 name = byte & 0x7f; 1111 } else { 1112 remain = byte & 0x7; 1113 name = (byte >> 3) & 0xf; 1114 } 1115 switch (name) { 1116 case 0x2: /* String */ 1117 cfg->vpd.vpd_ident = malloc(remain + 1, 1118 M_DEVBUF, M_WAITOK); 1119 i = 0; 1120 state = 1; 1121 break; 1122 case 0xf: /* End */ 1123 state = -1; 1124 break; 1125 case 0x10: /* VPD-R */ 1126 alloc = 8; 1127 off = 0; 1128 cfg->vpd.vpd_ros = malloc(alloc * 1129 sizeof(*cfg->vpd.vpd_ros), M_DEVBUF, 1130 M_WAITOK | M_ZERO); 1131 state = 2; 1132 break; 1133 case 0x11: /* VPD-W */ 1134 alloc = 8; 1135 off = 0; 1136 cfg->vpd.vpd_w = malloc(alloc * 1137 sizeof(*cfg->vpd.vpd_w), M_DEVBUF, 1138 M_WAITOK | M_ZERO); 1139 state = 5; 1140 break; 1141 default: /* Invalid data, abort */ 1142 state = -1; 1143 break; 1144 } 1145 break; 1146 1147 case 1: /* Identifier String */ 1148 cfg->vpd.vpd_ident[i++] = byte; 1149 remain--; 1150 if (remain == 0) { 1151 cfg->vpd.vpd_ident[i] = '\0'; 1152 state = 0; 1153 } 1154 break; 1155 1156 case 2: /* VPD-R Keyword Header */ 1157 if (off == alloc) { 1158 cfg->vpd.vpd_ros = reallocf(cfg->vpd.vpd_ros, 1159 (alloc *= 2) * sizeof(*cfg->vpd.vpd_ros), 1160 M_DEVBUF, M_WAITOK | M_ZERO); 1161 } 1162 cfg->vpd.vpd_ros[off].keyword[0] = byte; 1163 if (vpd_nextbyte(&vrs, &byte2)) { 1164 state = -2; 1165 break; 1166 } 1167 cfg->vpd.vpd_ros[off].keyword[1] = byte2; 1168 if (vpd_nextbyte(&vrs, &byte2)) { 1169 state = -2; 1170 break; 1171 } 1172 cfg->vpd.vpd_ros[off].len = dflen = byte2; 1173 if (dflen == 0 && 1174 strncmp(cfg->vpd.vpd_ros[off].keyword, "RV", 1175 2) == 0) { 1176 /* 1177 * if this happens, we can't trust the rest 1178 * of the VPD. 1179 */ 1180 pci_printf(cfg, "bad keyword length: %d\n", 1181 dflen); 1182 cksumvalid = 0; 1183 state = -1; 1184 break; 1185 } else if (dflen == 0) { 1186 cfg->vpd.vpd_ros[off].value = malloc(1 * 1187 sizeof(*cfg->vpd.vpd_ros[off].value), 1188 M_DEVBUF, M_WAITOK); 1189 cfg->vpd.vpd_ros[off].value[0] = '\x00'; 1190 } else 1191 cfg->vpd.vpd_ros[off].value = malloc( 1192 (dflen + 1) * 1193 sizeof(*cfg->vpd.vpd_ros[off].value), 1194 M_DEVBUF, M_WAITOK); 1195 remain -= 3; 1196 i = 0; 1197 /* keep in sync w/ state 3's transistions */ 1198 if (dflen == 0 && remain == 0) 1199 state = 0; 1200 else if (dflen == 0) 1201 state = 2; 1202 else 1203 state = 3; 1204 break; 1205 1206 case 3: /* VPD-R Keyword Value */ 1207 cfg->vpd.vpd_ros[off].value[i++] = byte; 1208 if (strncmp(cfg->vpd.vpd_ros[off].keyword, 1209 "RV", 2) == 0 && cksumvalid == -1) { 1210 if (vrs.cksum == 0) 1211 cksumvalid = 1; 1212 else { 1213 if (bootverbose) 1214 pci_printf(cfg, 1215 "bad VPD cksum, remain %hhu\n", 1216 vrs.cksum); 1217 cksumvalid = 0; 1218 state = -1; 1219 break; 1220 } 1221 } 1222 dflen--; 1223 remain--; 1224 /* keep in sync w/ state 2's transistions */ 1225 if (dflen == 0) 1226 cfg->vpd.vpd_ros[off++].value[i++] = '\0'; 1227 if (dflen == 0 && remain == 0) { 1228 cfg->vpd.vpd_rocnt = off; 1229 cfg->vpd.vpd_ros = reallocf(cfg->vpd.vpd_ros, 1230 off * sizeof(*cfg->vpd.vpd_ros), 1231 M_DEVBUF, M_WAITOK | M_ZERO); 1232 state = 0; 1233 } else if (dflen == 0) 1234 state = 2; 1235 break; 1236 1237 case 4: 1238 remain--; 1239 if (remain == 0) 1240 state = 0; 1241 break; 1242 1243 case 5: /* VPD-W Keyword Header */ 1244 if (off == alloc) { 1245 cfg->vpd.vpd_w = reallocf(cfg->vpd.vpd_w, 1246 (alloc *= 2) * sizeof(*cfg->vpd.vpd_w), 1247 M_DEVBUF, M_WAITOK | M_ZERO); 1248 } 1249 cfg->vpd.vpd_w[off].keyword[0] = byte; 1250 if (vpd_nextbyte(&vrs, &byte2)) { 1251 state = -2; 1252 break; 1253 } 1254 cfg->vpd.vpd_w[off].keyword[1] = byte2; 1255 if (vpd_nextbyte(&vrs, &byte2)) { 1256 state = -2; 1257 break; 1258 } 1259 cfg->vpd.vpd_w[off].len = dflen = byte2; 1260 cfg->vpd.vpd_w[off].start = vrs.off - vrs.bytesinval; 1261 cfg->vpd.vpd_w[off].value = malloc((dflen + 1) * 1262 sizeof(*cfg->vpd.vpd_w[off].value), 1263 M_DEVBUF, M_WAITOK); 1264 remain -= 3; 1265 i = 0; 1266 /* keep in sync w/ state 6's transistions */ 1267 if (dflen == 0 && remain == 0) 1268 state = 0; 1269 else if (dflen == 0) 1270 state = 5; 1271 else 1272 state = 6; 1273 break; 1274 1275 case 6: /* VPD-W Keyword Value */ 1276 cfg->vpd.vpd_w[off].value[i++] = byte; 1277 dflen--; 1278 remain--; 1279 /* keep in sync w/ state 5's transistions */ 1280 if (dflen == 0) 1281 cfg->vpd.vpd_w[off++].value[i++] = '\0'; 1282 if (dflen == 0 && remain == 0) { 1283 cfg->vpd.vpd_wcnt = off; 1284 cfg->vpd.vpd_w = reallocf(cfg->vpd.vpd_w, 1285 off * sizeof(*cfg->vpd.vpd_w), 1286 M_DEVBUF, M_WAITOK | M_ZERO); 1287 state = 0; 1288 } else if (dflen == 0) 1289 state = 5; 1290 break; 1291 1292 default: 1293 pci_printf(cfg, "invalid state: %d\n", state); 1294 state = -1; 1295 break; 1296 } 1297 } 1298 1299 if (cksumvalid == 0 || state < -1) { 1300 /* read-only data bad, clean up */ 1301 if (cfg->vpd.vpd_ros != NULL) { 1302 for (off = 0; cfg->vpd.vpd_ros[off].value; off++) 1303 free(cfg->vpd.vpd_ros[off].value, M_DEVBUF); 1304 free(cfg->vpd.vpd_ros, M_DEVBUF); 1305 cfg->vpd.vpd_ros = NULL; 1306 } 1307 } 1308 if (state < -1) { 1309 /* I/O error, clean up */ 1310 pci_printf(cfg, "failed to read VPD data.\n"); 1311 if (cfg->vpd.vpd_ident != NULL) { 1312 free(cfg->vpd.vpd_ident, M_DEVBUF); 1313 cfg->vpd.vpd_ident = NULL; 1314 } 1315 if (cfg->vpd.vpd_w != NULL) { 1316 for (off = 0; cfg->vpd.vpd_w[off].value; off++) 1317 free(cfg->vpd.vpd_w[off].value, M_DEVBUF); 1318 free(cfg->vpd.vpd_w, M_DEVBUF); 1319 cfg->vpd.vpd_w = NULL; 1320 } 1321 } 1322 cfg->vpd.vpd_cached = 1; 1323 #undef REG 1324 #undef WREG 1325 } 1326 1327 int 1328 pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr) 1329 { 1330 struct pci_devinfo *dinfo = device_get_ivars(child); 1331 pcicfgregs *cfg = &dinfo->cfg; 1332 1333 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) 1334 pci_read_vpd(device_get_parent(dev), cfg); 1335 1336 *identptr = cfg->vpd.vpd_ident; 1337 1338 if (*identptr == NULL) 1339 return (ENXIO); 1340 1341 return (0); 1342 } 1343 1344 int 1345 pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw, 1346 const char **vptr) 1347 { 1348 struct pci_devinfo *dinfo = device_get_ivars(child); 1349 pcicfgregs *cfg = &dinfo->cfg; 1350 int i; 1351 1352 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) 1353 pci_read_vpd(device_get_parent(dev), cfg); 1354 1355 for (i = 0; i < cfg->vpd.vpd_rocnt; i++) 1356 if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword, 1357 sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) { 1358 *vptr = cfg->vpd.vpd_ros[i].value; 1359 return (0); 1360 } 1361 1362 *vptr = NULL; 1363 return (ENXIO); 1364 } 1365 1366 struct pcicfg_vpd * 1367 pci_fetch_vpd_list(device_t dev) 1368 { 1369 struct pci_devinfo *dinfo = device_get_ivars(dev); 1370 pcicfgregs *cfg = &dinfo->cfg; 1371 1372 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) 1373 pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg); 1374 return (&cfg->vpd); 1375 } 1376 1377 /* 1378 * Find the requested HyperTransport capability and return the offset 1379 * in configuration space via the pointer provided. The function 1380 * returns 0 on success and an error code otherwise. 1381 */ 1382 int 1383 pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg) 1384 { 1385 int ptr, error; 1386 uint16_t val; 1387 1388 error = pci_find_cap(child, PCIY_HT, &ptr); 1389 if (error) 1390 return (error); 1391 1392 /* 1393 * Traverse the capabilities list checking each HT capability 1394 * to see if it matches the requested HT capability. 1395 */ 1396 for (;;) { 1397 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2); 1398 if (capability == PCIM_HTCAP_SLAVE || 1399 capability == PCIM_HTCAP_HOST) 1400 val &= 0xe000; 1401 else 1402 val &= PCIM_HTCMD_CAP_MASK; 1403 if (val == capability) { 1404 if (capreg != NULL) 1405 *capreg = ptr; 1406 return (0); 1407 } 1408 1409 /* Skip to the next HT capability. */ 1410 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0) 1411 break; 1412 } 1413 1414 return (ENOENT); 1415 } 1416 1417 /* 1418 * Find the next requested HyperTransport capability after start and return 1419 * the offset in configuration space via the pointer provided. The function 1420 * returns 0 on success and an error code otherwise. 1421 */ 1422 int 1423 pci_find_next_htcap_method(device_t dev, device_t child, int capability, 1424 int start, int *capreg) 1425 { 1426 int ptr; 1427 uint16_t val; 1428 1429 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT, 1430 ("start capability is not HyperTransport capability")); 1431 ptr = start; 1432 1433 /* 1434 * Traverse the capabilities list checking each HT capability 1435 * to see if it matches the requested HT capability. 1436 */ 1437 for (;;) { 1438 /* Skip to the next HT capability. */ 1439 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0) 1440 break; 1441 1442 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2); 1443 if (capability == PCIM_HTCAP_SLAVE || 1444 capability == PCIM_HTCAP_HOST) 1445 val &= 0xe000; 1446 else 1447 val &= PCIM_HTCMD_CAP_MASK; 1448 if (val == capability) { 1449 if (capreg != NULL) 1450 *capreg = ptr; 1451 return (0); 1452 } 1453 } 1454 1455 return (ENOENT); 1456 } 1457 1458 /* 1459 * Find the requested capability and return the offset in 1460 * configuration space via the pointer provided. The function returns 1461 * 0 on success and an error code otherwise. 1462 */ 1463 int 1464 pci_find_cap_method(device_t dev, device_t child, int capability, 1465 int *capreg) 1466 { 1467 struct pci_devinfo *dinfo = device_get_ivars(child); 1468 pcicfgregs *cfg = &dinfo->cfg; 1469 uint32_t status; 1470 uint8_t ptr; 1471 1472 /* 1473 * Check the CAP_LIST bit of the PCI status register first. 1474 */ 1475 status = pci_read_config(child, PCIR_STATUS, 2); 1476 if (!(status & PCIM_STATUS_CAPPRESENT)) 1477 return (ENXIO); 1478 1479 /* 1480 * Determine the start pointer of the capabilities list. 1481 */ 1482 switch (cfg->hdrtype & PCIM_HDRTYPE) { 1483 case PCIM_HDRTYPE_NORMAL: 1484 case PCIM_HDRTYPE_BRIDGE: 1485 ptr = PCIR_CAP_PTR; 1486 break; 1487 case PCIM_HDRTYPE_CARDBUS: 1488 ptr = PCIR_CAP_PTR_2; 1489 break; 1490 default: 1491 /* XXX: panic? */ 1492 return (ENXIO); /* no extended capabilities support */ 1493 } 1494 ptr = pci_read_config(child, ptr, 1); 1495 1496 /* 1497 * Traverse the capabilities list. 1498 */ 1499 while (ptr != 0) { 1500 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) { 1501 if (capreg != NULL) 1502 *capreg = ptr; 1503 return (0); 1504 } 1505 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1); 1506 } 1507 1508 return (ENOENT); 1509 } 1510 1511 /* 1512 * Find the next requested capability after start and return the offset in 1513 * configuration space via the pointer provided. The function returns 1514 * 0 on success and an error code otherwise. 1515 */ 1516 int 1517 pci_find_next_cap_method(device_t dev, device_t child, int capability, 1518 int start, int *capreg) 1519 { 1520 uint8_t ptr; 1521 1522 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability, 1523 ("start capability is not expected capability")); 1524 1525 ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1); 1526 while (ptr != 0) { 1527 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) { 1528 if (capreg != NULL) 1529 *capreg = ptr; 1530 return (0); 1531 } 1532 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1); 1533 } 1534 1535 return (ENOENT); 1536 } 1537 1538 /* 1539 * Find the requested extended capability and return the offset in 1540 * configuration space via the pointer provided. The function returns 1541 * 0 on success and an error code otherwise. 1542 */ 1543 int 1544 pci_find_extcap_method(device_t dev, device_t child, int capability, 1545 int *capreg) 1546 { 1547 struct pci_devinfo *dinfo = device_get_ivars(child); 1548 pcicfgregs *cfg = &dinfo->cfg; 1549 uint32_t ecap; 1550 uint16_t ptr; 1551 1552 /* Only supported for PCI-express devices. */ 1553 if (cfg->pcie.pcie_location == 0) 1554 return (ENXIO); 1555 1556 ptr = PCIR_EXTCAP; 1557 ecap = pci_read_config(child, ptr, 4); 1558 if (ecap == 0xffffffff || ecap == 0) 1559 return (ENOENT); 1560 for (;;) { 1561 if (PCI_EXTCAP_ID(ecap) == capability) { 1562 if (capreg != NULL) 1563 *capreg = ptr; 1564 return (0); 1565 } 1566 ptr = PCI_EXTCAP_NEXTPTR(ecap); 1567 if (ptr == 0) 1568 break; 1569 ecap = pci_read_config(child, ptr, 4); 1570 } 1571 1572 return (ENOENT); 1573 } 1574 1575 /* 1576 * Find the next requested extended capability after start and return the 1577 * offset in configuration space via the pointer provided. The function 1578 * returns 0 on success and an error code otherwise. 1579 */ 1580 int 1581 pci_find_next_extcap_method(device_t dev, device_t child, int capability, 1582 int start, int *capreg) 1583 { 1584 struct pci_devinfo *dinfo = device_get_ivars(child); 1585 pcicfgregs *cfg = &dinfo->cfg; 1586 uint32_t ecap; 1587 uint16_t ptr; 1588 1589 /* Only supported for PCI-express devices. */ 1590 if (cfg->pcie.pcie_location == 0) 1591 return (ENXIO); 1592 1593 ecap = pci_read_config(child, start, 4); 1594 KASSERT(PCI_EXTCAP_ID(ecap) == capability, 1595 ("start extended capability is not expected capability")); 1596 ptr = PCI_EXTCAP_NEXTPTR(ecap); 1597 while (ptr != 0) { 1598 ecap = pci_read_config(child, ptr, 4); 1599 if (PCI_EXTCAP_ID(ecap) == capability) { 1600 if (capreg != NULL) 1601 *capreg = ptr; 1602 return (0); 1603 } 1604 ptr = PCI_EXTCAP_NEXTPTR(ecap); 1605 } 1606 1607 return (ENOENT); 1608 } 1609 1610 /* 1611 * Support for MSI-X message interrupts. 1612 */ 1613 static void 1614 pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data) 1615 { 1616 struct pci_devinfo *dinfo = device_get_ivars(dev); 1617 struct pcicfg_msix *msix = &dinfo->cfg.msix; 1618 uint32_t offset; 1619 1620 KASSERT(msix->msix_table_len > index, ("bogus index")); 1621 offset = msix->msix_table_offset + index * 16; 1622 bus_write_4(msix->msix_table_res, offset, address & 0xffffffff); 1623 bus_write_4(msix->msix_table_res, offset + 4, address >> 32); 1624 bus_write_4(msix->msix_table_res, offset + 8, data); 1625 } 1626 1627 void 1628 pci_enable_msix_method(device_t dev, device_t child, u_int index, 1629 uint64_t address, uint32_t data) 1630 { 1631 1632 if (pci_msix_rewrite_table) { 1633 struct pci_devinfo *dinfo = device_get_ivars(child); 1634 struct pcicfg_msix *msix = &dinfo->cfg.msix; 1635 1636 /* 1637 * Some VM hosts require MSIX to be disabled in the 1638 * control register before updating the MSIX table 1639 * entries are allowed. It is not enough to only 1640 * disable MSIX while updating a single entry. MSIX 1641 * must be disabled while updating all entries in the 1642 * table. 1643 */ 1644 pci_write_config(child, 1645 msix->msix_location + PCIR_MSIX_CTRL, 1646 msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2); 1647 pci_resume_msix(child); 1648 } else 1649 pci_write_msix_entry(child, index, address, data); 1650 1651 /* Enable MSI -> HT mapping. */ 1652 pci_ht_map_msi(child, address); 1653 } 1654 1655 void 1656 pci_mask_msix(device_t dev, u_int index) 1657 { 1658 struct pci_devinfo *dinfo = device_get_ivars(dev); 1659 struct pcicfg_msix *msix = &dinfo->cfg.msix; 1660 uint32_t offset, val; 1661 1662 KASSERT(msix->msix_msgnum > index, ("bogus index")); 1663 offset = msix->msix_table_offset + index * 16 + 12; 1664 val = bus_read_4(msix->msix_table_res, offset); 1665 if (!(val & PCIM_MSIX_VCTRL_MASK)) { 1666 val |= PCIM_MSIX_VCTRL_MASK; 1667 bus_write_4(msix->msix_table_res, offset, val); 1668 } 1669 } 1670 1671 void 1672 pci_unmask_msix(device_t dev, u_int index) 1673 { 1674 struct pci_devinfo *dinfo = device_get_ivars(dev); 1675 struct pcicfg_msix *msix = &dinfo->cfg.msix; 1676 uint32_t offset, val; 1677 1678 KASSERT(msix->msix_table_len > index, ("bogus index")); 1679 offset = msix->msix_table_offset + index * 16 + 12; 1680 val = bus_read_4(msix->msix_table_res, offset); 1681 if (val & PCIM_MSIX_VCTRL_MASK) { 1682 val &= ~PCIM_MSIX_VCTRL_MASK; 1683 bus_write_4(msix->msix_table_res, offset, val); 1684 } 1685 } 1686 1687 int 1688 pci_pending_msix(device_t dev, u_int index) 1689 { 1690 struct pci_devinfo *dinfo = device_get_ivars(dev); 1691 struct pcicfg_msix *msix = &dinfo->cfg.msix; 1692 uint32_t offset, bit; 1693 1694 KASSERT(msix->msix_table_len > index, ("bogus index")); 1695 offset = msix->msix_pba_offset + (index / 32) * 4; 1696 bit = 1 << index % 32; 1697 return (bus_read_4(msix->msix_pba_res, offset) & bit); 1698 } 1699 1700 /* 1701 * Restore MSI-X registers and table during resume. If MSI-X is 1702 * enabled then walk the virtual table to restore the actual MSI-X 1703 * table. 1704 */ 1705 static void 1706 pci_resume_msix(device_t dev) 1707 { 1708 struct pci_devinfo *dinfo = device_get_ivars(dev); 1709 struct pcicfg_msix *msix = &dinfo->cfg.msix; 1710 struct msix_table_entry *mte; 1711 struct msix_vector *mv; 1712 int i; 1713 1714 if (msix->msix_alloc > 0) { 1715 /* First, mask all vectors. */ 1716 for (i = 0; i < msix->msix_msgnum; i++) 1717 pci_mask_msix(dev, i); 1718 1719 /* Second, program any messages with at least one handler. */ 1720 for (i = 0; i < msix->msix_table_len; i++) { 1721 mte = &msix->msix_table[i]; 1722 if (mte->mte_vector == 0 || mte->mte_handlers == 0) 1723 continue; 1724 mv = &msix->msix_vectors[mte->mte_vector - 1]; 1725 pci_write_msix_entry(dev, i, mv->mv_address, 1726 mv->mv_data); 1727 pci_unmask_msix(dev, i); 1728 } 1729 } 1730 pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL, 1731 msix->msix_ctrl, 2); 1732 } 1733 1734 /* 1735 * Attempt to allocate *count MSI-X messages. The actual number allocated is 1736 * returned in *count. After this function returns, each message will be 1737 * available to the driver as SYS_RES_IRQ resources starting at rid 1. 1738 */ 1739 int 1740 pci_alloc_msix_method(device_t dev, device_t child, int *count) 1741 { 1742 struct pci_devinfo *dinfo = device_get_ivars(child); 1743 pcicfgregs *cfg = &dinfo->cfg; 1744 struct resource_list_entry *rle; 1745 int actual, error, i, irq, max; 1746 1747 /* Don't let count == 0 get us into trouble. */ 1748 if (*count == 0) 1749 return (EINVAL); 1750 1751 /* If rid 0 is allocated, then fail. */ 1752 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); 1753 if (rle != NULL && rle->res != NULL) 1754 return (ENXIO); 1755 1756 /* Already have allocated messages? */ 1757 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) 1758 return (ENXIO); 1759 1760 /* If MSI-X is blacklisted for this system, fail. */ 1761 if (pci_msix_blacklisted()) 1762 return (ENXIO); 1763 1764 /* MSI-X capability present? */ 1765 if (cfg->msix.msix_location == 0 || !pci_do_msix) 1766 return (ENODEV); 1767 1768 /* Make sure the appropriate BARs are mapped. */ 1769 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY, 1770 cfg->msix.msix_table_bar); 1771 if (rle == NULL || rle->res == NULL || 1772 !(rman_get_flags(rle->res) & RF_ACTIVE)) 1773 return (ENXIO); 1774 cfg->msix.msix_table_res = rle->res; 1775 if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) { 1776 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY, 1777 cfg->msix.msix_pba_bar); 1778 if (rle == NULL || rle->res == NULL || 1779 !(rman_get_flags(rle->res) & RF_ACTIVE)) 1780 return (ENXIO); 1781 } 1782 cfg->msix.msix_pba_res = rle->res; 1783 1784 if (bootverbose) 1785 device_printf(child, 1786 "attempting to allocate %d MSI-X vectors (%d supported)\n", 1787 *count, cfg->msix.msix_msgnum); 1788 max = min(*count, cfg->msix.msix_msgnum); 1789 for (i = 0; i < max; i++) { 1790 /* Allocate a message. */ 1791 error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq); 1792 if (error) { 1793 if (i == 0) 1794 return (error); 1795 break; 1796 } 1797 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq, 1798 irq, 1); 1799 } 1800 actual = i; 1801 1802 if (bootverbose) { 1803 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1); 1804 if (actual == 1) 1805 device_printf(child, "using IRQ %ju for MSI-X\n", 1806 rle->start); 1807 else { 1808 int run; 1809 1810 /* 1811 * Be fancy and try to print contiguous runs of 1812 * IRQ values as ranges. 'irq' is the previous IRQ. 1813 * 'run' is true if we are in a range. 1814 */ 1815 device_printf(child, "using IRQs %ju", rle->start); 1816 irq = rle->start; 1817 run = 0; 1818 for (i = 1; i < actual; i++) { 1819 rle = resource_list_find(&dinfo->resources, 1820 SYS_RES_IRQ, i + 1); 1821 1822 /* Still in a run? */ 1823 if (rle->start == irq + 1) { 1824 run = 1; 1825 irq++; 1826 continue; 1827 } 1828 1829 /* Finish previous range. */ 1830 if (run) { 1831 printf("-%d", irq); 1832 run = 0; 1833 } 1834 1835 /* Start new range. */ 1836 printf(",%ju", rle->start); 1837 irq = rle->start; 1838 } 1839 1840 /* Unfinished range? */ 1841 if (run) 1842 printf("-%d", irq); 1843 printf(" for MSI-X\n"); 1844 } 1845 } 1846 1847 /* Mask all vectors. */ 1848 for (i = 0; i < cfg->msix.msix_msgnum; i++) 1849 pci_mask_msix(child, i); 1850 1851 /* Allocate and initialize vector data and virtual table. */ 1852 cfg->msix.msix_vectors = malloc(sizeof(struct msix_vector) * actual, 1853 M_DEVBUF, M_WAITOK | M_ZERO); 1854 cfg->msix.msix_table = malloc(sizeof(struct msix_table_entry) * actual, 1855 M_DEVBUF, M_WAITOK | M_ZERO); 1856 for (i = 0; i < actual; i++) { 1857 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); 1858 cfg->msix.msix_vectors[i].mv_irq = rle->start; 1859 cfg->msix.msix_table[i].mte_vector = i + 1; 1860 } 1861 1862 /* Update control register to enable MSI-X. */ 1863 cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1864 pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL, 1865 cfg->msix.msix_ctrl, 2); 1866 1867 /* Update counts of alloc'd messages. */ 1868 cfg->msix.msix_alloc = actual; 1869 cfg->msix.msix_table_len = actual; 1870 *count = actual; 1871 return (0); 1872 } 1873 1874 /* 1875 * By default, pci_alloc_msix() will assign the allocated IRQ 1876 * resources consecutively to the first N messages in the MSI-X table. 1877 * However, device drivers may want to use different layouts if they 1878 * either receive fewer messages than they asked for, or they wish to 1879 * populate the MSI-X table sparsely. This method allows the driver 1880 * to specify what layout it wants. It must be called after a 1881 * successful pci_alloc_msix() but before any of the associated 1882 * SYS_RES_IRQ resources are allocated via bus_alloc_resource(). 1883 * 1884 * The 'vectors' array contains 'count' message vectors. The array 1885 * maps directly to the MSI-X table in that index 0 in the array 1886 * specifies the vector for the first message in the MSI-X table, etc. 1887 * The vector value in each array index can either be 0 to indicate 1888 * that no vector should be assigned to a message slot, or it can be a 1889 * number from 1 to N (where N is the count returned from a 1890 * succcessful call to pci_alloc_msix()) to indicate which message 1891 * vector (IRQ) to be used for the corresponding message. 1892 * 1893 * On successful return, each message with a non-zero vector will have 1894 * an associated SYS_RES_IRQ whose rid is equal to the array index + 1895 * 1. Additionally, if any of the IRQs allocated via the previous 1896 * call to pci_alloc_msix() are not used in the mapping, those IRQs 1897 * will be freed back to the system automatically. 1898 * 1899 * For example, suppose a driver has a MSI-X table with 6 messages and 1900 * asks for 6 messages, but pci_alloc_msix() only returns a count of 1901 * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and 1902 * C. After the call to pci_alloc_msix(), the device will be setup to 1903 * have an MSI-X table of ABC--- (where - means no vector assigned). 1904 * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 }, 1905 * then the MSI-X table will look like A-AB-B, and the 'C' vector will 1906 * be freed back to the system. This device will also have valid 1907 * SYS_RES_IRQ rids of 1, 3, 4, and 6. 1908 * 1909 * In any case, the SYS_RES_IRQ rid X will always map to the message 1910 * at MSI-X table index X - 1 and will only be valid if a vector is 1911 * assigned to that table entry. 1912 */ 1913 int 1914 pci_remap_msix_method(device_t dev, device_t child, int count, 1915 const u_int *vectors) 1916 { 1917 struct pci_devinfo *dinfo = device_get_ivars(child); 1918 struct pcicfg_msix *msix = &dinfo->cfg.msix; 1919 struct resource_list_entry *rle; 1920 int i, irq, j, *used; 1921 1922 /* 1923 * Have to have at least one message in the table but the 1924 * table can't be bigger than the actual MSI-X table in the 1925 * device. 1926 */ 1927 if (count == 0 || count > msix->msix_msgnum) 1928 return (EINVAL); 1929 1930 /* Sanity check the vectors. */ 1931 for (i = 0; i < count; i++) 1932 if (vectors[i] > msix->msix_alloc) 1933 return (EINVAL); 1934 1935 /* 1936 * Make sure there aren't any holes in the vectors to be used. 1937 * It's a big pain to support it, and it doesn't really make 1938 * sense anyway. Also, at least one vector must be used. 1939 */ 1940 used = malloc(sizeof(int) * msix->msix_alloc, M_DEVBUF, M_WAITOK | 1941 M_ZERO); 1942 for (i = 0; i < count; i++) 1943 if (vectors[i] != 0) 1944 used[vectors[i] - 1] = 1; 1945 for (i = 0; i < msix->msix_alloc - 1; i++) 1946 if (used[i] == 0 && used[i + 1] == 1) { 1947 free(used, M_DEVBUF); 1948 return (EINVAL); 1949 } 1950 if (used[0] != 1) { 1951 free(used, M_DEVBUF); 1952 return (EINVAL); 1953 } 1954 1955 /* Make sure none of the resources are allocated. */ 1956 for (i = 0; i < msix->msix_table_len; i++) { 1957 if (msix->msix_table[i].mte_vector == 0) 1958 continue; 1959 if (msix->msix_table[i].mte_handlers > 0) { 1960 free(used, M_DEVBUF); 1961 return (EBUSY); 1962 } 1963 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); 1964 KASSERT(rle != NULL, ("missing resource")); 1965 if (rle->res != NULL) { 1966 free(used, M_DEVBUF); 1967 return (EBUSY); 1968 } 1969 } 1970 1971 /* Free the existing resource list entries. */ 1972 for (i = 0; i < msix->msix_table_len; i++) { 1973 if (msix->msix_table[i].mte_vector == 0) 1974 continue; 1975 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); 1976 } 1977 1978 /* 1979 * Build the new virtual table keeping track of which vectors are 1980 * used. 1981 */ 1982 free(msix->msix_table, M_DEVBUF); 1983 msix->msix_table = malloc(sizeof(struct msix_table_entry) * count, 1984 M_DEVBUF, M_WAITOK | M_ZERO); 1985 for (i = 0; i < count; i++) 1986 msix->msix_table[i].mte_vector = vectors[i]; 1987 msix->msix_table_len = count; 1988 1989 /* Free any unused IRQs and resize the vectors array if necessary. */ 1990 j = msix->msix_alloc - 1; 1991 if (used[j] == 0) { 1992 struct msix_vector *vec; 1993 1994 while (used[j] == 0) { 1995 PCIB_RELEASE_MSIX(device_get_parent(dev), child, 1996 msix->msix_vectors[j].mv_irq); 1997 j--; 1998 } 1999 vec = malloc(sizeof(struct msix_vector) * (j + 1), M_DEVBUF, 2000 M_WAITOK); 2001 bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) * 2002 (j + 1)); 2003 free(msix->msix_vectors, M_DEVBUF); 2004 msix->msix_vectors = vec; 2005 msix->msix_alloc = j + 1; 2006 } 2007 free(used, M_DEVBUF); 2008 2009 /* Map the IRQs onto the rids. */ 2010 for (i = 0; i < count; i++) { 2011 if (vectors[i] == 0) 2012 continue; 2013 irq = msix->msix_vectors[vectors[i] - 1].mv_irq; 2014 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq, 2015 irq, 1); 2016 } 2017 2018 if (bootverbose) { 2019 device_printf(child, "Remapped MSI-X IRQs as: "); 2020 for (i = 0; i < count; i++) { 2021 if (i != 0) 2022 printf(", "); 2023 if (vectors[i] == 0) 2024 printf("---"); 2025 else 2026 printf("%d", 2027 msix->msix_vectors[vectors[i] - 1].mv_irq); 2028 } 2029 printf("\n"); 2030 } 2031 2032 return (0); 2033 } 2034 2035 static int 2036 pci_release_msix(device_t dev, device_t child) 2037 { 2038 struct pci_devinfo *dinfo = device_get_ivars(child); 2039 struct pcicfg_msix *msix = &dinfo->cfg.msix; 2040 struct resource_list_entry *rle; 2041 int i; 2042 2043 /* Do we have any messages to release? */ 2044 if (msix->msix_alloc == 0) 2045 return (ENODEV); 2046 2047 /* Make sure none of the resources are allocated. */ 2048 for (i = 0; i < msix->msix_table_len; i++) { 2049 if (msix->msix_table[i].mte_vector == 0) 2050 continue; 2051 if (msix->msix_table[i].mte_handlers > 0) 2052 return (EBUSY); 2053 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); 2054 KASSERT(rle != NULL, ("missing resource")); 2055 if (rle->res != NULL) 2056 return (EBUSY); 2057 } 2058 2059 /* Update control register to disable MSI-X. */ 2060 msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE; 2061 pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL, 2062 msix->msix_ctrl, 2); 2063 2064 /* Free the resource list entries. */ 2065 for (i = 0; i < msix->msix_table_len; i++) { 2066 if (msix->msix_table[i].mte_vector == 0) 2067 continue; 2068 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); 2069 } 2070 free(msix->msix_table, M_DEVBUF); 2071 msix->msix_table_len = 0; 2072 2073 /* Release the IRQs. */ 2074 for (i = 0; i < msix->msix_alloc; i++) 2075 PCIB_RELEASE_MSIX(device_get_parent(dev), child, 2076 msix->msix_vectors[i].mv_irq); 2077 free(msix->msix_vectors, M_DEVBUF); 2078 msix->msix_alloc = 0; 2079 return (0); 2080 } 2081 2082 /* 2083 * Return the max supported MSI-X messages this device supports. 2084 * Basically, assuming the MD code can alloc messages, this function 2085 * should return the maximum value that pci_alloc_msix() can return. 2086 * Thus, it is subject to the tunables, etc. 2087 */ 2088 int 2089 pci_msix_count_method(device_t dev, device_t child) 2090 { 2091 struct pci_devinfo *dinfo = device_get_ivars(child); 2092 struct pcicfg_msix *msix = &dinfo->cfg.msix; 2093 2094 if (pci_do_msix && msix->msix_location != 0) 2095 return (msix->msix_msgnum); 2096 return (0); 2097 } 2098 2099 int 2100 pci_msix_pba_bar_method(device_t dev, device_t child) 2101 { 2102 struct pci_devinfo *dinfo = device_get_ivars(child); 2103 struct pcicfg_msix *msix = &dinfo->cfg.msix; 2104 2105 if (pci_do_msix && msix->msix_location != 0) 2106 return (msix->msix_pba_bar); 2107 return (-1); 2108 } 2109 2110 int 2111 pci_msix_table_bar_method(device_t dev, device_t child) 2112 { 2113 struct pci_devinfo *dinfo = device_get_ivars(child); 2114 struct pcicfg_msix *msix = &dinfo->cfg.msix; 2115 2116 if (pci_do_msix && msix->msix_location != 0) 2117 return (msix->msix_table_bar); 2118 return (-1); 2119 } 2120 2121 /* 2122 * HyperTransport MSI mapping control 2123 */ 2124 void 2125 pci_ht_map_msi(device_t dev, uint64_t addr) 2126 { 2127 struct pci_devinfo *dinfo = device_get_ivars(dev); 2128 struct pcicfg_ht *ht = &dinfo->cfg.ht; 2129 2130 if (!ht->ht_msimap) 2131 return; 2132 2133 if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) && 2134 ht->ht_msiaddr >> 20 == addr >> 20) { 2135 /* Enable MSI -> HT mapping. */ 2136 ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE; 2137 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND, 2138 ht->ht_msictrl, 2); 2139 } 2140 2141 if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) { 2142 /* Disable MSI -> HT mapping. */ 2143 ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE; 2144 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND, 2145 ht->ht_msictrl, 2); 2146 } 2147 } 2148 2149 int 2150 pci_get_max_payload(device_t dev) 2151 { 2152 struct pci_devinfo *dinfo = device_get_ivars(dev); 2153 int cap; 2154 uint16_t val; 2155 2156 cap = dinfo->cfg.pcie.pcie_location; 2157 if (cap == 0) 2158 return (0); 2159 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); 2160 val &= PCIEM_CTL_MAX_PAYLOAD; 2161 val >>= 5; 2162 return (1 << (val + 7)); 2163 } 2164 2165 int 2166 pci_get_max_read_req(device_t dev) 2167 { 2168 struct pci_devinfo *dinfo = device_get_ivars(dev); 2169 int cap; 2170 uint16_t val; 2171 2172 cap = dinfo->cfg.pcie.pcie_location; 2173 if (cap == 0) 2174 return (0); 2175 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); 2176 val &= PCIEM_CTL_MAX_READ_REQUEST; 2177 val >>= 12; 2178 return (1 << (val + 7)); 2179 } 2180 2181 int 2182 pci_set_max_read_req(device_t dev, int size) 2183 { 2184 struct pci_devinfo *dinfo = device_get_ivars(dev); 2185 int cap; 2186 uint16_t val; 2187 2188 cap = dinfo->cfg.pcie.pcie_location; 2189 if (cap == 0) 2190 return (0); 2191 if (size < 128) 2192 size = 128; 2193 if (size > 4096) 2194 size = 4096; 2195 size = (1 << (fls(size) - 1)); 2196 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); 2197 val &= ~PCIEM_CTL_MAX_READ_REQUEST; 2198 val |= (fls(size) - 8) << 12; 2199 pci_write_config(dev, cap + PCIER_DEVICE_CTL, val, 2); 2200 return (size); 2201 } 2202 2203 uint32_t 2204 pcie_read_config(device_t dev, int reg, int width) 2205 { 2206 struct pci_devinfo *dinfo = device_get_ivars(dev); 2207 int cap; 2208 2209 cap = dinfo->cfg.pcie.pcie_location; 2210 if (cap == 0) { 2211 if (width == 2) 2212 return (0xffff); 2213 return (0xffffffff); 2214 } 2215 2216 return (pci_read_config(dev, cap + reg, width)); 2217 } 2218 2219 void 2220 pcie_write_config(device_t dev, int reg, uint32_t value, int width) 2221 { 2222 struct pci_devinfo *dinfo = device_get_ivars(dev); 2223 int cap; 2224 2225 cap = dinfo->cfg.pcie.pcie_location; 2226 if (cap == 0) 2227 return; 2228 pci_write_config(dev, cap + reg, value, width); 2229 } 2230 2231 /* 2232 * Adjusts a PCI-e capability register by clearing the bits in mask 2233 * and setting the bits in (value & mask). Bits not set in mask are 2234 * not adjusted. 2235 * 2236 * Returns the old value on success or all ones on failure. 2237 */ 2238 uint32_t 2239 pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value, 2240 int width) 2241 { 2242 struct pci_devinfo *dinfo = device_get_ivars(dev); 2243 uint32_t old, new; 2244 int cap; 2245 2246 cap = dinfo->cfg.pcie.pcie_location; 2247 if (cap == 0) { 2248 if (width == 2) 2249 return (0xffff); 2250 return (0xffffffff); 2251 } 2252 2253 old = pci_read_config(dev, cap + reg, width); 2254 new = old & ~mask; 2255 new |= (value & mask); 2256 pci_write_config(dev, cap + reg, new, width); 2257 return (old); 2258 } 2259 2260 /* 2261 * Support for MSI message signalled interrupts. 2262 */ 2263 void 2264 pci_enable_msi_method(device_t dev, device_t child, uint64_t address, 2265 uint16_t data) 2266 { 2267 struct pci_devinfo *dinfo = device_get_ivars(child); 2268 struct pcicfg_msi *msi = &dinfo->cfg.msi; 2269 2270 /* Write data and address values. */ 2271 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR, 2272 address & 0xffffffff, 4); 2273 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) { 2274 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH, 2275 address >> 32, 4); 2276 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT, 2277 data, 2); 2278 } else 2279 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data, 2280 2); 2281 2282 /* Enable MSI in the control register. */ 2283 msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE; 2284 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, 2285 msi->msi_ctrl, 2); 2286 2287 /* Enable MSI -> HT mapping. */ 2288 pci_ht_map_msi(child, address); 2289 } 2290 2291 void 2292 pci_disable_msi_method(device_t dev, device_t child) 2293 { 2294 struct pci_devinfo *dinfo = device_get_ivars(child); 2295 struct pcicfg_msi *msi = &dinfo->cfg.msi; 2296 2297 /* Disable MSI -> HT mapping. */ 2298 pci_ht_map_msi(child, 0); 2299 2300 /* Disable MSI in the control register. */ 2301 msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE; 2302 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, 2303 msi->msi_ctrl, 2); 2304 } 2305 2306 /* 2307 * Restore MSI registers during resume. If MSI is enabled then 2308 * restore the data and address registers in addition to the control 2309 * register. 2310 */ 2311 static void 2312 pci_resume_msi(device_t dev) 2313 { 2314 struct pci_devinfo *dinfo = device_get_ivars(dev); 2315 struct pcicfg_msi *msi = &dinfo->cfg.msi; 2316 uint64_t address; 2317 uint16_t data; 2318 2319 if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) { 2320 address = msi->msi_addr; 2321 data = msi->msi_data; 2322 pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR, 2323 address & 0xffffffff, 4); 2324 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) { 2325 pci_write_config(dev, msi->msi_location + 2326 PCIR_MSI_ADDR_HIGH, address >> 32, 4); 2327 pci_write_config(dev, msi->msi_location + 2328 PCIR_MSI_DATA_64BIT, data, 2); 2329 } else 2330 pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA, 2331 data, 2); 2332 } 2333 pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2334 2); 2335 } 2336 2337 static int 2338 pci_remap_intr_method(device_t bus, device_t dev, u_int irq) 2339 { 2340 struct pci_devinfo *dinfo = device_get_ivars(dev); 2341 pcicfgregs *cfg = &dinfo->cfg; 2342 struct resource_list_entry *rle; 2343 struct msix_table_entry *mte; 2344 struct msix_vector *mv; 2345 uint64_t addr; 2346 uint32_t data; 2347 int error, i, j; 2348 2349 /* 2350 * Handle MSI first. We try to find this IRQ among our list 2351 * of MSI IRQs. If we find it, we request updated address and 2352 * data registers and apply the results. 2353 */ 2354 if (cfg->msi.msi_alloc > 0) { 2355 2356 /* If we don't have any active handlers, nothing to do. */ 2357 if (cfg->msi.msi_handlers == 0) 2358 return (0); 2359 for (i = 0; i < cfg->msi.msi_alloc; i++) { 2360 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 2361 i + 1); 2362 if (rle->start == irq) { 2363 error = PCIB_MAP_MSI(device_get_parent(bus), 2364 dev, irq, &addr, &data); 2365 if (error) 2366 return (error); 2367 pci_disable_msi(dev); 2368 dinfo->cfg.msi.msi_addr = addr; 2369 dinfo->cfg.msi.msi_data = data; 2370 pci_enable_msi(dev, addr, data); 2371 return (0); 2372 } 2373 } 2374 return (ENOENT); 2375 } 2376 2377 /* 2378 * For MSI-X, we check to see if we have this IRQ. If we do, 2379 * we request the updated mapping info. If that works, we go 2380 * through all the slots that use this IRQ and update them. 2381 */ 2382 if (cfg->msix.msix_alloc > 0) { 2383 for (i = 0; i < cfg->msix.msix_alloc; i++) { 2384 mv = &cfg->msix.msix_vectors[i]; 2385 if (mv->mv_irq == irq) { 2386 error = PCIB_MAP_MSI(device_get_parent(bus), 2387 dev, irq, &addr, &data); 2388 if (error) 2389 return (error); 2390 mv->mv_address = addr; 2391 mv->mv_data = data; 2392 for (j = 0; j < cfg->msix.msix_table_len; j++) { 2393 mte = &cfg->msix.msix_table[j]; 2394 if (mte->mte_vector != i + 1) 2395 continue; 2396 if (mte->mte_handlers == 0) 2397 continue; 2398 pci_mask_msix(dev, j); 2399 pci_enable_msix(dev, j, addr, data); 2400 pci_unmask_msix(dev, j); 2401 } 2402 } 2403 } 2404 return (ENOENT); 2405 } 2406 2407 return (ENOENT); 2408 } 2409 2410 /* 2411 * Returns true if the specified device is blacklisted because MSI 2412 * doesn't work. 2413 */ 2414 int 2415 pci_msi_device_blacklisted(device_t dev) 2416 { 2417 2418 if (!pci_honor_msi_blacklist) 2419 return (0); 2420 2421 return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI)); 2422 } 2423 2424 /* 2425 * Determine if MSI is blacklisted globally on this system. Currently, 2426 * we just check for blacklisted chipsets as represented by the 2427 * host-PCI bridge at device 0:0:0. In the future, it may become 2428 * necessary to check other system attributes, such as the kenv values 2429 * that give the motherboard manufacturer and model number. 2430 */ 2431 static int 2432 pci_msi_blacklisted(void) 2433 { 2434 device_t dev; 2435 2436 if (!pci_honor_msi_blacklist) 2437 return (0); 2438 2439 /* Blacklist all non-PCI-express and non-PCI-X chipsets. */ 2440 if (!(pcie_chipset || pcix_chipset)) { 2441 if (vm_guest != VM_GUEST_NO) { 2442 /* 2443 * Whitelist older chipsets in virtual 2444 * machines known to support MSI. 2445 */ 2446 dev = pci_find_bsf(0, 0, 0); 2447 if (dev != NULL) 2448 return (!pci_has_quirk(pci_get_devid(dev), 2449 PCI_QUIRK_ENABLE_MSI_VM)); 2450 } 2451 return (1); 2452 } 2453 2454 dev = pci_find_bsf(0, 0, 0); 2455 if (dev != NULL) 2456 return (pci_msi_device_blacklisted(dev)); 2457 return (0); 2458 } 2459 2460 /* 2461 * Returns true if the specified device is blacklisted because MSI-X 2462 * doesn't work. Note that this assumes that if MSI doesn't work, 2463 * MSI-X doesn't either. 2464 */ 2465 int 2466 pci_msix_device_blacklisted(device_t dev) 2467 { 2468 2469 if (!pci_honor_msi_blacklist) 2470 return (0); 2471 2472 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX)) 2473 return (1); 2474 2475 return (pci_msi_device_blacklisted(dev)); 2476 } 2477 2478 /* 2479 * Determine if MSI-X is blacklisted globally on this system. If MSI 2480 * is blacklisted, assume that MSI-X is as well. Check for additional 2481 * chipsets where MSI works but MSI-X does not. 2482 */ 2483 static int 2484 pci_msix_blacklisted(void) 2485 { 2486 device_t dev; 2487 2488 if (!pci_honor_msi_blacklist) 2489 return (0); 2490 2491 dev = pci_find_bsf(0, 0, 0); 2492 if (dev != NULL && pci_has_quirk(pci_get_devid(dev), 2493 PCI_QUIRK_DISABLE_MSIX)) 2494 return (1); 2495 2496 return (pci_msi_blacklisted()); 2497 } 2498 2499 /* 2500 * Attempt to allocate *count MSI messages. The actual number allocated is 2501 * returned in *count. After this function returns, each message will be 2502 * available to the driver as SYS_RES_IRQ resources starting at a rid 1. 2503 */ 2504 int 2505 pci_alloc_msi_method(device_t dev, device_t child, int *count) 2506 { 2507 struct pci_devinfo *dinfo = device_get_ivars(child); 2508 pcicfgregs *cfg = &dinfo->cfg; 2509 struct resource_list_entry *rle; 2510 int actual, error, i, irqs[32]; 2511 uint16_t ctrl; 2512 2513 /* Don't let count == 0 get us into trouble. */ 2514 if (*count == 0) 2515 return (EINVAL); 2516 2517 /* If rid 0 is allocated, then fail. */ 2518 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); 2519 if (rle != NULL && rle->res != NULL) 2520 return (ENXIO); 2521 2522 /* Already have allocated messages? */ 2523 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) 2524 return (ENXIO); 2525 2526 /* If MSI is blacklisted for this system, fail. */ 2527 if (pci_msi_blacklisted()) 2528 return (ENXIO); 2529 2530 /* MSI capability present? */ 2531 if (cfg->msi.msi_location == 0 || !pci_do_msi) 2532 return (ENODEV); 2533 2534 if (bootverbose) 2535 device_printf(child, 2536 "attempting to allocate %d MSI vectors (%d supported)\n", 2537 *count, cfg->msi.msi_msgnum); 2538 2539 /* Don't ask for more than the device supports. */ 2540 actual = min(*count, cfg->msi.msi_msgnum); 2541 2542 /* Don't ask for more than 32 messages. */ 2543 actual = min(actual, 32); 2544 2545 /* MSI requires power of 2 number of messages. */ 2546 if (!powerof2(actual)) 2547 return (EINVAL); 2548 2549 for (;;) { 2550 /* Try to allocate N messages. */ 2551 error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual, 2552 actual, irqs); 2553 if (error == 0) 2554 break; 2555 if (actual == 1) 2556 return (error); 2557 2558 /* Try N / 2. */ 2559 actual >>= 1; 2560 } 2561 2562 /* 2563 * We now have N actual messages mapped onto SYS_RES_IRQ 2564 * resources in the irqs[] array, so add new resources 2565 * starting at rid 1. 2566 */ 2567 for (i = 0; i < actual; i++) 2568 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, 2569 irqs[i], irqs[i], 1); 2570 2571 if (bootverbose) { 2572 if (actual == 1) 2573 device_printf(child, "using IRQ %d for MSI\n", irqs[0]); 2574 else { 2575 int run; 2576 2577 /* 2578 * Be fancy and try to print contiguous runs 2579 * of IRQ values as ranges. 'run' is true if 2580 * we are in a range. 2581 */ 2582 device_printf(child, "using IRQs %d", irqs[0]); 2583 run = 0; 2584 for (i = 1; i < actual; i++) { 2585 2586 /* Still in a run? */ 2587 if (irqs[i] == irqs[i - 1] + 1) { 2588 run = 1; 2589 continue; 2590 } 2591 2592 /* Finish previous range. */ 2593 if (run) { 2594 printf("-%d", irqs[i - 1]); 2595 run = 0; 2596 } 2597 2598 /* Start new range. */ 2599 printf(",%d", irqs[i]); 2600 } 2601 2602 /* Unfinished range? */ 2603 if (run) 2604 printf("-%d", irqs[actual - 1]); 2605 printf(" for MSI\n"); 2606 } 2607 } 2608 2609 /* Update control register with actual count. */ 2610 ctrl = cfg->msi.msi_ctrl; 2611 ctrl &= ~PCIM_MSICTRL_MME_MASK; 2612 ctrl |= (ffs(actual) - 1) << 4; 2613 cfg->msi.msi_ctrl = ctrl; 2614 pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2); 2615 2616 /* Update counts of alloc'd messages. */ 2617 cfg->msi.msi_alloc = actual; 2618 cfg->msi.msi_handlers = 0; 2619 *count = actual; 2620 return (0); 2621 } 2622 2623 /* Release the MSI messages associated with this device. */ 2624 int 2625 pci_release_msi_method(device_t dev, device_t child) 2626 { 2627 struct pci_devinfo *dinfo = device_get_ivars(child); 2628 struct pcicfg_msi *msi = &dinfo->cfg.msi; 2629 struct resource_list_entry *rle; 2630 int error, i, irqs[32]; 2631 2632 /* Try MSI-X first. */ 2633 error = pci_release_msix(dev, child); 2634 if (error != ENODEV) 2635 return (error); 2636 2637 /* Do we have any messages to release? */ 2638 if (msi->msi_alloc == 0) 2639 return (ENODEV); 2640 KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages")); 2641 2642 /* Make sure none of the resources are allocated. */ 2643 if (msi->msi_handlers > 0) 2644 return (EBUSY); 2645 for (i = 0; i < msi->msi_alloc; i++) { 2646 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); 2647 KASSERT(rle != NULL, ("missing MSI resource")); 2648 if (rle->res != NULL) 2649 return (EBUSY); 2650 irqs[i] = rle->start; 2651 } 2652 2653 /* Update control register with 0 count. */ 2654 KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE), 2655 ("%s: MSI still enabled", __func__)); 2656 msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK; 2657 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, 2658 msi->msi_ctrl, 2); 2659 2660 /* Release the messages. */ 2661 PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs); 2662 for (i = 0; i < msi->msi_alloc; i++) 2663 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); 2664 2665 /* Update alloc count. */ 2666 msi->msi_alloc = 0; 2667 msi->msi_addr = 0; 2668 msi->msi_data = 0; 2669 return (0); 2670 } 2671 2672 /* 2673 * Return the max supported MSI messages this device supports. 2674 * Basically, assuming the MD code can alloc messages, this function 2675 * should return the maximum value that pci_alloc_msi() can return. 2676 * Thus, it is subject to the tunables, etc. 2677 */ 2678 int 2679 pci_msi_count_method(device_t dev, device_t child) 2680 { 2681 struct pci_devinfo *dinfo = device_get_ivars(child); 2682 struct pcicfg_msi *msi = &dinfo->cfg.msi; 2683 2684 if (pci_do_msi && msi->msi_location != 0) 2685 return (msi->msi_msgnum); 2686 return (0); 2687 } 2688 2689 /* free pcicfgregs structure and all depending data structures */ 2690 2691 int 2692 pci_freecfg(struct pci_devinfo *dinfo) 2693 { 2694 struct devlist *devlist_head; 2695 struct pci_map *pm, *next; 2696 int i; 2697 2698 devlist_head = &pci_devq; 2699 2700 if (dinfo->cfg.vpd.vpd_reg) { 2701 free(dinfo->cfg.vpd.vpd_ident, M_DEVBUF); 2702 for (i = 0; i < dinfo->cfg.vpd.vpd_rocnt; i++) 2703 free(dinfo->cfg.vpd.vpd_ros[i].value, M_DEVBUF); 2704 free(dinfo->cfg.vpd.vpd_ros, M_DEVBUF); 2705 for (i = 0; i < dinfo->cfg.vpd.vpd_wcnt; i++) 2706 free(dinfo->cfg.vpd.vpd_w[i].value, M_DEVBUF); 2707 free(dinfo->cfg.vpd.vpd_w, M_DEVBUF); 2708 } 2709 STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) { 2710 free(pm, M_DEVBUF); 2711 } 2712 STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links); 2713 free(dinfo, M_DEVBUF); 2714 2715 /* increment the generation count */ 2716 pci_generation++; 2717 2718 /* we're losing one device */ 2719 pci_numdevs--; 2720 return (0); 2721 } 2722 2723 /* 2724 * PCI power manangement 2725 */ 2726 int 2727 pci_set_powerstate_method(device_t dev, device_t child, int state) 2728 { 2729 struct pci_devinfo *dinfo = device_get_ivars(child); 2730 pcicfgregs *cfg = &dinfo->cfg; 2731 uint16_t status; 2732 int oldstate, highest, delay; 2733 2734 if (cfg->pp.pp_cap == 0) 2735 return (EOPNOTSUPP); 2736 2737 /* 2738 * Optimize a no state change request away. While it would be OK to 2739 * write to the hardware in theory, some devices have shown odd 2740 * behavior when going from D3 -> D3. 2741 */ 2742 oldstate = pci_get_powerstate(child); 2743 if (oldstate == state) 2744 return (0); 2745 2746 /* 2747 * The PCI power management specification states that after a state 2748 * transition between PCI power states, system software must 2749 * guarantee a minimal delay before the function accesses the device. 2750 * Compute the worst case delay that we need to guarantee before we 2751 * access the device. Many devices will be responsive much more 2752 * quickly than this delay, but there are some that don't respond 2753 * instantly to state changes. Transitions to/from D3 state require 2754 * 10ms, while D2 requires 200us, and D0/1 require none. The delay 2755 * is done below with DELAY rather than a sleeper function because 2756 * this function can be called from contexts where we cannot sleep. 2757 */ 2758 highest = (oldstate > state) ? oldstate : state; 2759 if (highest == PCI_POWERSTATE_D3) 2760 delay = 10000; 2761 else if (highest == PCI_POWERSTATE_D2) 2762 delay = 200; 2763 else 2764 delay = 0; 2765 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2) 2766 & ~PCIM_PSTAT_DMASK; 2767 switch (state) { 2768 case PCI_POWERSTATE_D0: 2769 status |= PCIM_PSTAT_D0; 2770 break; 2771 case PCI_POWERSTATE_D1: 2772 if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0) 2773 return (EOPNOTSUPP); 2774 status |= PCIM_PSTAT_D1; 2775 break; 2776 case PCI_POWERSTATE_D2: 2777 if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0) 2778 return (EOPNOTSUPP); 2779 status |= PCIM_PSTAT_D2; 2780 break; 2781 case PCI_POWERSTATE_D3: 2782 status |= PCIM_PSTAT_D3; 2783 break; 2784 default: 2785 return (EINVAL); 2786 } 2787 2788 if (bootverbose) 2789 pci_printf(cfg, "Transition from D%d to D%d\n", oldstate, 2790 state); 2791 2792 PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_status, status, 2); 2793 if (delay) 2794 DELAY(delay); 2795 return (0); 2796 } 2797 2798 int 2799 pci_get_powerstate_method(device_t dev, device_t child) 2800 { 2801 struct pci_devinfo *dinfo = device_get_ivars(child); 2802 pcicfgregs *cfg = &dinfo->cfg; 2803 uint16_t status; 2804 int result; 2805 2806 if (cfg->pp.pp_cap != 0) { 2807 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2); 2808 switch (status & PCIM_PSTAT_DMASK) { 2809 case PCIM_PSTAT_D0: 2810 result = PCI_POWERSTATE_D0; 2811 break; 2812 case PCIM_PSTAT_D1: 2813 result = PCI_POWERSTATE_D1; 2814 break; 2815 case PCIM_PSTAT_D2: 2816 result = PCI_POWERSTATE_D2; 2817 break; 2818 case PCIM_PSTAT_D3: 2819 result = PCI_POWERSTATE_D3; 2820 break; 2821 default: 2822 result = PCI_POWERSTATE_UNKNOWN; 2823 break; 2824 } 2825 } else { 2826 /* No support, device is always at D0 */ 2827 result = PCI_POWERSTATE_D0; 2828 } 2829 return (result); 2830 } 2831 2832 /* 2833 * Some convenience functions for PCI device drivers. 2834 */ 2835 2836 static __inline void 2837 pci_set_command_bit(device_t dev, device_t child, uint16_t bit) 2838 { 2839 uint16_t command; 2840 2841 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2); 2842 command |= bit; 2843 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2); 2844 } 2845 2846 static __inline void 2847 pci_clear_command_bit(device_t dev, device_t child, uint16_t bit) 2848 { 2849 uint16_t command; 2850 2851 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2); 2852 command &= ~bit; 2853 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2); 2854 } 2855 2856 int 2857 pci_enable_busmaster_method(device_t dev, device_t child) 2858 { 2859 pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN); 2860 return (0); 2861 } 2862 2863 int 2864 pci_disable_busmaster_method(device_t dev, device_t child) 2865 { 2866 pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN); 2867 return (0); 2868 } 2869 2870 int 2871 pci_enable_io_method(device_t dev, device_t child, int space) 2872 { 2873 uint16_t bit; 2874 2875 switch(space) { 2876 case SYS_RES_IOPORT: 2877 bit = PCIM_CMD_PORTEN; 2878 break; 2879 case SYS_RES_MEMORY: 2880 bit = PCIM_CMD_MEMEN; 2881 break; 2882 default: 2883 return (EINVAL); 2884 } 2885 pci_set_command_bit(dev, child, bit); 2886 return (0); 2887 } 2888 2889 int 2890 pci_disable_io_method(device_t dev, device_t child, int space) 2891 { 2892 uint16_t bit; 2893 2894 switch(space) { 2895 case SYS_RES_IOPORT: 2896 bit = PCIM_CMD_PORTEN; 2897 break; 2898 case SYS_RES_MEMORY: 2899 bit = PCIM_CMD_MEMEN; 2900 break; 2901 default: 2902 return (EINVAL); 2903 } 2904 pci_clear_command_bit(dev, child, bit); 2905 return (0); 2906 } 2907 2908 /* 2909 * New style pci driver. Parent device is either a pci-host-bridge or a 2910 * pci-pci-bridge. Both kinds are represented by instances of pcib. 2911 */ 2912 2913 void 2914 pci_print_verbose(struct pci_devinfo *dinfo) 2915 { 2916 2917 if (bootverbose) { 2918 pcicfgregs *cfg = &dinfo->cfg; 2919 2920 printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n", 2921 cfg->vendor, cfg->device, cfg->revid); 2922 printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n", 2923 cfg->domain, cfg->bus, cfg->slot, cfg->func); 2924 printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n", 2925 cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype, 2926 cfg->mfdev); 2927 printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n", 2928 cfg->cmdreg, cfg->statreg, cfg->cachelnsz); 2929 printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n", 2930 cfg->lattimer, cfg->lattimer * 30, cfg->mingnt, 2931 cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250); 2932 if (cfg->intpin > 0) 2933 printf("\tintpin=%c, irq=%d\n", 2934 cfg->intpin +'a' -1, cfg->intline); 2935 if (cfg->pp.pp_cap) { 2936 uint16_t status; 2937 2938 status = pci_read_config(cfg->dev, cfg->pp.pp_status, 2); 2939 printf("\tpowerspec %d supports D0%s%s D3 current D%d\n", 2940 cfg->pp.pp_cap & PCIM_PCAP_SPEC, 2941 cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "", 2942 cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "", 2943 status & PCIM_PSTAT_DMASK); 2944 } 2945 if (cfg->msi.msi_location) { 2946 int ctrl; 2947 2948 ctrl = cfg->msi.msi_ctrl; 2949 printf("\tMSI supports %d message%s%s%s\n", 2950 cfg->msi.msi_msgnum, 2951 (cfg->msi.msi_msgnum == 1) ? "" : "s", 2952 (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "", 2953 (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":""); 2954 } 2955 if (cfg->msix.msix_location) { 2956 printf("\tMSI-X supports %d message%s ", 2957 cfg->msix.msix_msgnum, 2958 (cfg->msix.msix_msgnum == 1) ? "" : "s"); 2959 if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar) 2960 printf("in map 0x%x\n", 2961 cfg->msix.msix_table_bar); 2962 else 2963 printf("in maps 0x%x and 0x%x\n", 2964 cfg->msix.msix_table_bar, 2965 cfg->msix.msix_pba_bar); 2966 } 2967 } 2968 } 2969 2970 static int 2971 pci_porten(device_t dev) 2972 { 2973 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0; 2974 } 2975 2976 static int 2977 pci_memen(device_t dev) 2978 { 2979 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0; 2980 } 2981 2982 void 2983 pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp, 2984 int *bar64) 2985 { 2986 struct pci_devinfo *dinfo; 2987 pci_addr_t map, testval; 2988 int ln2range; 2989 uint16_t cmd; 2990 2991 /* 2992 * The device ROM BAR is special. It is always a 32-bit 2993 * memory BAR. Bit 0 is special and should not be set when 2994 * sizing the BAR. 2995 */ 2996 dinfo = device_get_ivars(dev); 2997 if (PCIR_IS_BIOS(&dinfo->cfg, reg)) { 2998 map = pci_read_config(dev, reg, 4); 2999 pci_write_config(dev, reg, 0xfffffffe, 4); 3000 testval = pci_read_config(dev, reg, 4); 3001 pci_write_config(dev, reg, map, 4); 3002 *mapp = map; 3003 *testvalp = testval; 3004 if (bar64 != NULL) 3005 *bar64 = 0; 3006 return; 3007 } 3008 3009 map = pci_read_config(dev, reg, 4); 3010 ln2range = pci_maprange(map); 3011 if (ln2range == 64) 3012 map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32; 3013 3014 /* 3015 * Disable decoding via the command register before 3016 * determining the BAR's length since we will be placing it in 3017 * a weird state. 3018 */ 3019 cmd = pci_read_config(dev, PCIR_COMMAND, 2); 3020 pci_write_config(dev, PCIR_COMMAND, 3021 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2); 3022 3023 /* 3024 * Determine the BAR's length by writing all 1's. The bottom 3025 * log_2(size) bits of the BAR will stick as 0 when we read 3026 * the value back. 3027 * 3028 * NB: according to the PCI Local Bus Specification, rev. 3.0: 3029 * "Software writes 0FFFFFFFFh to both registers, reads them back, 3030 * and combines the result into a 64-bit value." (section 6.2.5.1) 3031 * 3032 * Writes to both registers must be performed before attempting to 3033 * read back the size value. 3034 */ 3035 testval = 0; 3036 pci_write_config(dev, reg, 0xffffffff, 4); 3037 if (ln2range == 64) { 3038 pci_write_config(dev, reg + 4, 0xffffffff, 4); 3039 testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32; 3040 } 3041 testval |= pci_read_config(dev, reg, 4); 3042 3043 /* 3044 * Restore the original value of the BAR. We may have reprogrammed 3045 * the BAR of the low-level console device and when booting verbose, 3046 * we need the console device addressable. 3047 */ 3048 pci_write_config(dev, reg, map, 4); 3049 if (ln2range == 64) 3050 pci_write_config(dev, reg + 4, map >> 32, 4); 3051 pci_write_config(dev, PCIR_COMMAND, cmd, 2); 3052 3053 *mapp = map; 3054 *testvalp = testval; 3055 if (bar64 != NULL) 3056 *bar64 = (ln2range == 64); 3057 } 3058 3059 static void 3060 pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base) 3061 { 3062 struct pci_devinfo *dinfo; 3063 int ln2range; 3064 3065 /* The device ROM BAR is always a 32-bit memory BAR. */ 3066 dinfo = device_get_ivars(dev); 3067 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg)) 3068 ln2range = 32; 3069 else 3070 ln2range = pci_maprange(pm->pm_value); 3071 pci_write_config(dev, pm->pm_reg, base, 4); 3072 if (ln2range == 64) 3073 pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4); 3074 pm->pm_value = pci_read_config(dev, pm->pm_reg, 4); 3075 if (ln2range == 64) 3076 pm->pm_value |= (pci_addr_t)pci_read_config(dev, 3077 pm->pm_reg + 4, 4) << 32; 3078 } 3079 3080 struct pci_map * 3081 pci_find_bar(device_t dev, int reg) 3082 { 3083 struct pci_devinfo *dinfo; 3084 struct pci_map *pm; 3085 3086 dinfo = device_get_ivars(dev); 3087 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) { 3088 if (pm->pm_reg == reg) 3089 return (pm); 3090 } 3091 return (NULL); 3092 } 3093 3094 int 3095 pci_bar_enabled(device_t dev, struct pci_map *pm) 3096 { 3097 struct pci_devinfo *dinfo; 3098 uint16_t cmd; 3099 3100 dinfo = device_get_ivars(dev); 3101 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) && 3102 !(pm->pm_value & PCIM_BIOS_ENABLE)) 3103 return (0); 3104 cmd = pci_read_config(dev, PCIR_COMMAND, 2); 3105 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value)) 3106 return ((cmd & PCIM_CMD_MEMEN) != 0); 3107 else 3108 return ((cmd & PCIM_CMD_PORTEN) != 0); 3109 } 3110 3111 struct pci_map * 3112 pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size) 3113 { 3114 struct pci_devinfo *dinfo; 3115 struct pci_map *pm, *prev; 3116 3117 dinfo = device_get_ivars(dev); 3118 pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO); 3119 pm->pm_reg = reg; 3120 pm->pm_value = value; 3121 pm->pm_size = size; 3122 STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) { 3123 KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x", 3124 reg)); 3125 if (STAILQ_NEXT(prev, pm_link) == NULL || 3126 STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg) 3127 break; 3128 } 3129 if (prev != NULL) 3130 STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link); 3131 else 3132 STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link); 3133 return (pm); 3134 } 3135 3136 static void 3137 pci_restore_bars(device_t dev) 3138 { 3139 struct pci_devinfo *dinfo; 3140 struct pci_map *pm; 3141 int ln2range; 3142 3143 dinfo = device_get_ivars(dev); 3144 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) { 3145 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg)) 3146 ln2range = 32; 3147 else 3148 ln2range = pci_maprange(pm->pm_value); 3149 pci_write_config(dev, pm->pm_reg, pm->pm_value, 4); 3150 if (ln2range == 64) 3151 pci_write_config(dev, pm->pm_reg + 4, 3152 pm->pm_value >> 32, 4); 3153 } 3154 } 3155 3156 /* 3157 * Add a resource based on a pci map register. Return 1 if the map 3158 * register is a 32bit map register or 2 if it is a 64bit register. 3159 */ 3160 static int 3161 pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl, 3162 int force, int prefetch) 3163 { 3164 struct pci_map *pm; 3165 pci_addr_t base, map, testval; 3166 pci_addr_t start, end, count; 3167 int barlen, basezero, flags, maprange, mapsize, type; 3168 uint16_t cmd; 3169 struct resource *res; 3170 3171 /* 3172 * The BAR may already exist if the device is a CardBus card 3173 * whose CIS is stored in this BAR. 3174 */ 3175 pm = pci_find_bar(dev, reg); 3176 if (pm != NULL) { 3177 maprange = pci_maprange(pm->pm_value); 3178 barlen = maprange == 64 ? 2 : 1; 3179 return (barlen); 3180 } 3181 3182 pci_read_bar(dev, reg, &map, &testval, NULL); 3183 if (PCI_BAR_MEM(map)) { 3184 type = SYS_RES_MEMORY; 3185 if (map & PCIM_BAR_MEM_PREFETCH) 3186 prefetch = 1; 3187 } else 3188 type = SYS_RES_IOPORT; 3189 mapsize = pci_mapsize(testval); 3190 base = pci_mapbase(map); 3191 #ifdef __PCI_BAR_ZERO_VALID 3192 basezero = 0; 3193 #else 3194 basezero = base == 0; 3195 #endif 3196 maprange = pci_maprange(map); 3197 barlen = maprange == 64 ? 2 : 1; 3198 3199 /* 3200 * For I/O registers, if bottom bit is set, and the next bit up 3201 * isn't clear, we know we have a BAR that doesn't conform to the 3202 * spec, so ignore it. Also, sanity check the size of the data 3203 * areas to the type of memory involved. Memory must be at least 3204 * 16 bytes in size, while I/O ranges must be at least 4. 3205 */ 3206 if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0) 3207 return (barlen); 3208 if ((type == SYS_RES_MEMORY && mapsize < 4) || 3209 (type == SYS_RES_IOPORT && mapsize < 2)) 3210 return (barlen); 3211 3212 /* Save a record of this BAR. */ 3213 pm = pci_add_bar(dev, reg, map, mapsize); 3214 if (bootverbose) { 3215 printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d", 3216 reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize); 3217 if (type == SYS_RES_IOPORT && !pci_porten(dev)) 3218 printf(", port disabled\n"); 3219 else if (type == SYS_RES_MEMORY && !pci_memen(dev)) 3220 printf(", memory disabled\n"); 3221 else 3222 printf(", enabled\n"); 3223 } 3224 3225 /* 3226 * If base is 0, then we have problems if this architecture does 3227 * not allow that. It is best to ignore such entries for the 3228 * moment. These will be allocated later if the driver specifically 3229 * requests them. However, some removable buses look better when 3230 * all resources are allocated, so allow '0' to be overriden. 3231 * 3232 * Similarly treat maps whose values is the same as the test value 3233 * read back. These maps have had all f's written to them by the 3234 * BIOS in an attempt to disable the resources. 3235 */ 3236 if (!force && (basezero || map == testval)) 3237 return (barlen); 3238 if ((u_long)base != base) { 3239 device_printf(bus, 3240 "pci%d:%d:%d:%d bar %#x too many address bits", 3241 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), 3242 pci_get_function(dev), reg); 3243 return (barlen); 3244 } 3245 3246 /* 3247 * This code theoretically does the right thing, but has 3248 * undesirable side effects in some cases where peripherals 3249 * respond oddly to having these bits enabled. Let the user 3250 * be able to turn them off (since pci_enable_io_modes is 1 by 3251 * default). 3252 */ 3253 if (pci_enable_io_modes) { 3254 /* Turn on resources that have been left off by a lazy BIOS */ 3255 if (type == SYS_RES_IOPORT && !pci_porten(dev)) { 3256 cmd = pci_read_config(dev, PCIR_COMMAND, 2); 3257 cmd |= PCIM_CMD_PORTEN; 3258 pci_write_config(dev, PCIR_COMMAND, cmd, 2); 3259 } 3260 if (type == SYS_RES_MEMORY && !pci_memen(dev)) { 3261 cmd = pci_read_config(dev, PCIR_COMMAND, 2); 3262 cmd |= PCIM_CMD_MEMEN; 3263 pci_write_config(dev, PCIR_COMMAND, cmd, 2); 3264 } 3265 } else { 3266 if (type == SYS_RES_IOPORT && !pci_porten(dev)) 3267 return (barlen); 3268 if (type == SYS_RES_MEMORY && !pci_memen(dev)) 3269 return (barlen); 3270 } 3271 3272 count = (pci_addr_t)1 << mapsize; 3273 flags = RF_ALIGNMENT_LOG2(mapsize); 3274 if (prefetch) 3275 flags |= RF_PREFETCHABLE; 3276 if (basezero || base == pci_mapbase(testval) || pci_clear_bars) { 3277 start = 0; /* Let the parent decide. */ 3278 end = ~0; 3279 } else { 3280 start = base; 3281 end = base + count - 1; 3282 } 3283 resource_list_add(rl, type, reg, start, end, count); 3284 3285 /* 3286 * Try to allocate the resource for this BAR from our parent 3287 * so that this resource range is already reserved. The 3288 * driver for this device will later inherit this resource in 3289 * pci_alloc_resource(). 3290 */ 3291 res = resource_list_reserve(rl, bus, dev, type, ®, start, end, count, 3292 flags); 3293 if ((pci_do_realloc_bars 3294 || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR)) 3295 && res == NULL && (start != 0 || end != ~0)) { 3296 /* 3297 * If the allocation fails, try to allocate a resource for 3298 * this BAR using any available range. The firmware felt 3299 * it was important enough to assign a resource, so don't 3300 * disable decoding if we can help it. 3301 */ 3302 resource_list_delete(rl, type, reg); 3303 resource_list_add(rl, type, reg, 0, ~0, count); 3304 res = resource_list_reserve(rl, bus, dev, type, ®, 0, ~0, 3305 count, flags); 3306 } 3307 if (res == NULL) { 3308 /* 3309 * If the allocation fails, delete the resource list entry 3310 * and disable decoding for this device. 3311 * 3312 * If the driver requests this resource in the future, 3313 * pci_reserve_map() will try to allocate a fresh 3314 * resource range. 3315 */ 3316 resource_list_delete(rl, type, reg); 3317 pci_disable_io(dev, type); 3318 if (bootverbose) 3319 device_printf(bus, 3320 "pci%d:%d:%d:%d bar %#x failed to allocate\n", 3321 pci_get_domain(dev), pci_get_bus(dev), 3322 pci_get_slot(dev), pci_get_function(dev), reg); 3323 } else { 3324 start = rman_get_start(res); 3325 pci_write_bar(dev, pm, start); 3326 } 3327 return (barlen); 3328 } 3329 3330 /* 3331 * For ATA devices we need to decide early what addressing mode to use. 3332 * Legacy demands that the primary and secondary ATA ports sits on the 3333 * same addresses that old ISA hardware did. This dictates that we use 3334 * those addresses and ignore the BAR's if we cannot set PCI native 3335 * addressing mode. 3336 */ 3337 static void 3338 pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force, 3339 uint32_t prefetchmask) 3340 { 3341 int rid, type, progif; 3342 #if 0 3343 /* if this device supports PCI native addressing use it */ 3344 progif = pci_read_config(dev, PCIR_PROGIF, 1); 3345 if ((progif & 0x8a) == 0x8a) { 3346 if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) && 3347 pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) { 3348 printf("Trying ATA native PCI addressing mode\n"); 3349 pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1); 3350 } 3351 } 3352 #endif 3353 progif = pci_read_config(dev, PCIR_PROGIF, 1); 3354 type = SYS_RES_IOPORT; 3355 if (progif & PCIP_STORAGE_IDE_MODEPRIM) { 3356 pci_add_map(bus, dev, PCIR_BAR(0), rl, force, 3357 prefetchmask & (1 << 0)); 3358 pci_add_map(bus, dev, PCIR_BAR(1), rl, force, 3359 prefetchmask & (1 << 1)); 3360 } else { 3361 rid = PCIR_BAR(0); 3362 resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8); 3363 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0, 3364 0x1f7, 8, 0); 3365 rid = PCIR_BAR(1); 3366 resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1); 3367 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6, 3368 0x3f6, 1, 0); 3369 } 3370 if (progif & PCIP_STORAGE_IDE_MODESEC) { 3371 pci_add_map(bus, dev, PCIR_BAR(2), rl, force, 3372 prefetchmask & (1 << 2)); 3373 pci_add_map(bus, dev, PCIR_BAR(3), rl, force, 3374 prefetchmask & (1 << 3)); 3375 } else { 3376 rid = PCIR_BAR(2); 3377 resource_list_add(rl, type, rid, 0x170, 0x177, 8); 3378 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x170, 3379 0x177, 8, 0); 3380 rid = PCIR_BAR(3); 3381 resource_list_add(rl, type, rid, 0x376, 0x376, 1); 3382 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x376, 3383 0x376, 1, 0); 3384 } 3385 pci_add_map(bus, dev, PCIR_BAR(4), rl, force, 3386 prefetchmask & (1 << 4)); 3387 pci_add_map(bus, dev, PCIR_BAR(5), rl, force, 3388 prefetchmask & (1 << 5)); 3389 } 3390 3391 static void 3392 pci_assign_interrupt(device_t bus, device_t dev, int force_route) 3393 { 3394 struct pci_devinfo *dinfo = device_get_ivars(dev); 3395 pcicfgregs *cfg = &dinfo->cfg; 3396 char tunable_name[64]; 3397 int irq; 3398 3399 /* Has to have an intpin to have an interrupt. */ 3400 if (cfg->intpin == 0) 3401 return; 3402 3403 /* Let the user override the IRQ with a tunable. */ 3404 irq = PCI_INVALID_IRQ; 3405 snprintf(tunable_name, sizeof(tunable_name), 3406 "hw.pci%d.%d.%d.INT%c.irq", 3407 cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1); 3408 if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0)) 3409 irq = PCI_INVALID_IRQ; 3410 3411 /* 3412 * If we didn't get an IRQ via the tunable, then we either use the 3413 * IRQ value in the intline register or we ask the bus to route an 3414 * interrupt for us. If force_route is true, then we only use the 3415 * value in the intline register if the bus was unable to assign an 3416 * IRQ. 3417 */ 3418 if (!PCI_INTERRUPT_VALID(irq)) { 3419 if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route) 3420 irq = PCI_ASSIGN_INTERRUPT(bus, dev); 3421 if (!PCI_INTERRUPT_VALID(irq)) 3422 irq = cfg->intline; 3423 } 3424 3425 /* If after all that we don't have an IRQ, just bail. */ 3426 if (!PCI_INTERRUPT_VALID(irq)) 3427 return; 3428 3429 /* Update the config register if it changed. */ 3430 if (irq != cfg->intline) { 3431 cfg->intline = irq; 3432 pci_write_config(dev, PCIR_INTLINE, irq, 1); 3433 } 3434 3435 /* Add this IRQ as rid 0 interrupt resource. */ 3436 resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1); 3437 } 3438 3439 /* Perform early OHCI takeover from SMM. */ 3440 static void 3441 ohci_early_takeover(device_t self) 3442 { 3443 struct resource *res; 3444 uint32_t ctl; 3445 int rid; 3446 int i; 3447 3448 rid = PCIR_BAR(0); 3449 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); 3450 if (res == NULL) 3451 return; 3452 3453 ctl = bus_read_4(res, OHCI_CONTROL); 3454 if (ctl & OHCI_IR) { 3455 if (bootverbose) 3456 printf("ohci early: " 3457 "SMM active, request owner change\n"); 3458 bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR); 3459 for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) { 3460 DELAY(1000); 3461 ctl = bus_read_4(res, OHCI_CONTROL); 3462 } 3463 if (ctl & OHCI_IR) { 3464 if (bootverbose) 3465 printf("ohci early: " 3466 "SMM does not respond, resetting\n"); 3467 bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET); 3468 } 3469 /* Disable interrupts */ 3470 bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS); 3471 } 3472 3473 bus_release_resource(self, SYS_RES_MEMORY, rid, res); 3474 } 3475 3476 /* Perform early UHCI takeover from SMM. */ 3477 static void 3478 uhci_early_takeover(device_t self) 3479 { 3480 struct resource *res; 3481 int rid; 3482 3483 /* 3484 * Set the PIRQD enable bit and switch off all the others. We don't 3485 * want legacy support to interfere with us XXX Does this also mean 3486 * that the BIOS won't touch the keyboard anymore if it is connected 3487 * to the ports of the root hub? 3488 */ 3489 pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2); 3490 3491 /* Disable interrupts */ 3492 rid = PCI_UHCI_BASE_REG; 3493 res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE); 3494 if (res != NULL) { 3495 bus_write_2(res, UHCI_INTR, 0); 3496 bus_release_resource(self, SYS_RES_IOPORT, rid, res); 3497 } 3498 } 3499 3500 /* Perform early EHCI takeover from SMM. */ 3501 static void 3502 ehci_early_takeover(device_t self) 3503 { 3504 struct resource *res; 3505 uint32_t cparams; 3506 uint32_t eec; 3507 uint8_t eecp; 3508 uint8_t bios_sem; 3509 uint8_t offs; 3510 int rid; 3511 int i; 3512 3513 rid = PCIR_BAR(0); 3514 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); 3515 if (res == NULL) 3516 return; 3517 3518 cparams = bus_read_4(res, EHCI_HCCPARAMS); 3519 3520 /* Synchronise with the BIOS if it owns the controller. */ 3521 for (eecp = EHCI_HCC_EECP(cparams); eecp != 0; 3522 eecp = EHCI_EECP_NEXT(eec)) { 3523 eec = pci_read_config(self, eecp, 4); 3524 if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) { 3525 continue; 3526 } 3527 bios_sem = pci_read_config(self, eecp + 3528 EHCI_LEGSUP_BIOS_SEM, 1); 3529 if (bios_sem == 0) { 3530 continue; 3531 } 3532 if (bootverbose) 3533 printf("ehci early: " 3534 "SMM active, request owner change\n"); 3535 3536 pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1); 3537 3538 for (i = 0; (i < 100) && (bios_sem != 0); i++) { 3539 DELAY(1000); 3540 bios_sem = pci_read_config(self, eecp + 3541 EHCI_LEGSUP_BIOS_SEM, 1); 3542 } 3543 3544 if (bios_sem != 0) { 3545 if (bootverbose) 3546 printf("ehci early: " 3547 "SMM does not respond\n"); 3548 } 3549 /* Disable interrupts */ 3550 offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION)); 3551 bus_write_4(res, offs + EHCI_USBINTR, 0); 3552 } 3553 bus_release_resource(self, SYS_RES_MEMORY, rid, res); 3554 } 3555 3556 /* Perform early XHCI takeover from SMM. */ 3557 static void 3558 xhci_early_takeover(device_t self) 3559 { 3560 struct resource *res; 3561 uint32_t cparams; 3562 uint32_t eec; 3563 uint8_t eecp; 3564 uint8_t bios_sem; 3565 uint8_t offs; 3566 int rid; 3567 int i; 3568 3569 rid = PCIR_BAR(0); 3570 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); 3571 if (res == NULL) 3572 return; 3573 3574 cparams = bus_read_4(res, XHCI_HCSPARAMS0); 3575 3576 eec = -1; 3577 3578 /* Synchronise with the BIOS if it owns the controller. */ 3579 for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec); 3580 eecp += XHCI_XECP_NEXT(eec) << 2) { 3581 eec = bus_read_4(res, eecp); 3582 3583 if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY) 3584 continue; 3585 3586 bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM); 3587 if (bios_sem == 0) 3588 continue; 3589 3590 if (bootverbose) 3591 printf("xhci early: " 3592 "SMM active, request owner change\n"); 3593 3594 bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1); 3595 3596 /* wait a maximum of 5 second */ 3597 3598 for (i = 0; (i < 5000) && (bios_sem != 0); i++) { 3599 DELAY(1000); 3600 bios_sem = bus_read_1(res, eecp + 3601 XHCI_XECP_BIOS_SEM); 3602 } 3603 3604 if (bios_sem != 0) { 3605 if (bootverbose) 3606 printf("xhci early: " 3607 "SMM does not respond\n"); 3608 } 3609 3610 /* Disable interrupts */ 3611 offs = bus_read_1(res, XHCI_CAPLENGTH); 3612 bus_write_4(res, offs + XHCI_USBCMD, 0); 3613 bus_read_4(res, offs + XHCI_USBSTS); 3614 } 3615 bus_release_resource(self, SYS_RES_MEMORY, rid, res); 3616 } 3617 3618 #if defined(NEW_PCIB) && defined(PCI_RES_BUS) 3619 static void 3620 pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg, 3621 struct resource_list *rl) 3622 { 3623 struct resource *res; 3624 char *cp; 3625 rman_res_t start, end, count; 3626 int rid, sec_bus, sec_reg, sub_bus, sub_reg, sup_bus; 3627 3628 switch (cfg->hdrtype & PCIM_HDRTYPE) { 3629 case PCIM_HDRTYPE_BRIDGE: 3630 sec_reg = PCIR_SECBUS_1; 3631 sub_reg = PCIR_SUBBUS_1; 3632 break; 3633 case PCIM_HDRTYPE_CARDBUS: 3634 sec_reg = PCIR_SECBUS_2; 3635 sub_reg = PCIR_SUBBUS_2; 3636 break; 3637 default: 3638 return; 3639 } 3640 3641 /* 3642 * If the existing bus range is valid, attempt to reserve it 3643 * from our parent. If this fails for any reason, clear the 3644 * secbus and subbus registers. 3645 * 3646 * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus? 3647 * This would at least preserve the existing sec_bus if it is 3648 * valid. 3649 */ 3650 sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1); 3651 sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1); 3652 3653 /* Quirk handling. */ 3654 switch (pci_get_devid(dev)) { 3655 case 0x12258086: /* Intel 82454KX/GX (Orion) */ 3656 sup_bus = pci_read_config(dev, 0x41, 1); 3657 if (sup_bus != 0xff) { 3658 sec_bus = sup_bus + 1; 3659 sub_bus = sup_bus + 1; 3660 PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1); 3661 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1); 3662 } 3663 break; 3664 3665 case 0x00dd10de: 3666 /* Compaq R3000 BIOS sets wrong subordinate bus number. */ 3667 if ((cp = kern_getenv("smbios.planar.maker")) == NULL) 3668 break; 3669 if (strncmp(cp, "Compal", 6) != 0) { 3670 freeenv(cp); 3671 break; 3672 } 3673 freeenv(cp); 3674 if ((cp = kern_getenv("smbios.planar.product")) == NULL) 3675 break; 3676 if (strncmp(cp, "08A0", 4) != 0) { 3677 freeenv(cp); 3678 break; 3679 } 3680 freeenv(cp); 3681 if (sub_bus < 0xa) { 3682 sub_bus = 0xa; 3683 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1); 3684 } 3685 break; 3686 } 3687 3688 if (bootverbose) 3689 printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus); 3690 if (sec_bus > 0 && sub_bus >= sec_bus) { 3691 start = sec_bus; 3692 end = sub_bus; 3693 count = end - start + 1; 3694 3695 resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count); 3696 3697 /* 3698 * If requested, clear secondary bus registers in 3699 * bridge devices to force a complete renumbering 3700 * rather than reserving the existing range. However, 3701 * preserve the existing size. 3702 */ 3703 if (pci_clear_buses) 3704 goto clear; 3705 3706 rid = 0; 3707 res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, &rid, 3708 start, end, count, 0); 3709 if (res != NULL) 3710 return; 3711 3712 if (bootverbose) 3713 device_printf(bus, 3714 "pci%d:%d:%d:%d secbus failed to allocate\n", 3715 pci_get_domain(dev), pci_get_bus(dev), 3716 pci_get_slot(dev), pci_get_function(dev)); 3717 } 3718 3719 clear: 3720 PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1); 3721 PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1); 3722 } 3723 3724 static struct resource * 3725 pci_alloc_secbus(device_t dev, device_t child, int *rid, rman_res_t start, 3726 rman_res_t end, rman_res_t count, u_int flags) 3727 { 3728 struct pci_devinfo *dinfo; 3729 pcicfgregs *cfg; 3730 struct resource_list *rl; 3731 struct resource *res; 3732 int sec_reg, sub_reg; 3733 3734 dinfo = device_get_ivars(child); 3735 cfg = &dinfo->cfg; 3736 rl = &dinfo->resources; 3737 switch (cfg->hdrtype & PCIM_HDRTYPE) { 3738 case PCIM_HDRTYPE_BRIDGE: 3739 sec_reg = PCIR_SECBUS_1; 3740 sub_reg = PCIR_SUBBUS_1; 3741 break; 3742 case PCIM_HDRTYPE_CARDBUS: 3743 sec_reg = PCIR_SECBUS_2; 3744 sub_reg = PCIR_SUBBUS_2; 3745 break; 3746 default: 3747 return (NULL); 3748 } 3749 3750 if (*rid != 0) 3751 return (NULL); 3752 3753 if (resource_list_find(rl, PCI_RES_BUS, *rid) == NULL) 3754 resource_list_add(rl, PCI_RES_BUS, *rid, start, end, count); 3755 if (!resource_list_reserved(rl, PCI_RES_BUS, *rid)) { 3756 res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid, 3757 start, end, count, flags & ~RF_ACTIVE); 3758 if (res == NULL) { 3759 resource_list_delete(rl, PCI_RES_BUS, *rid); 3760 device_printf(child, "allocating %ju bus%s failed\n", 3761 count, count == 1 ? "" : "es"); 3762 return (NULL); 3763 } 3764 if (bootverbose) 3765 device_printf(child, 3766 "Lazy allocation of %ju bus%s at %ju\n", count, 3767 count == 1 ? "" : "es", rman_get_start(res)); 3768 PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1); 3769 PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1); 3770 } 3771 return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start, 3772 end, count, flags)); 3773 } 3774 #endif 3775 3776 static int 3777 pci_ea_bei_to_rid(device_t dev, int bei) 3778 { 3779 #ifdef PCI_IOV 3780 struct pci_devinfo *dinfo; 3781 int iov_pos; 3782 struct pcicfg_iov *iov; 3783 3784 dinfo = device_get_ivars(dev); 3785 iov = dinfo->cfg.iov; 3786 if (iov != NULL) 3787 iov_pos = iov->iov_pos; 3788 else 3789 iov_pos = 0; 3790 #endif 3791 3792 /* Check if matches BAR */ 3793 if ((bei >= PCIM_EA_BEI_BAR_0) && 3794 (bei <= PCIM_EA_BEI_BAR_5)) 3795 return (PCIR_BAR(bei)); 3796 3797 /* Check ROM */ 3798 if (bei == PCIM_EA_BEI_ROM) 3799 return (PCIR_BIOS); 3800 3801 #ifdef PCI_IOV 3802 /* Check if matches VF_BAR */ 3803 if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) && 3804 (bei <= PCIM_EA_BEI_VF_BAR_5)) 3805 return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) + 3806 iov_pos); 3807 #endif 3808 3809 return (-1); 3810 } 3811 3812 int 3813 pci_ea_is_enabled(device_t dev, int rid) 3814 { 3815 struct pci_ea_entry *ea; 3816 struct pci_devinfo *dinfo; 3817 3818 dinfo = device_get_ivars(dev); 3819 3820 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) { 3821 if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid) 3822 return ((ea->eae_flags & PCIM_EA_ENABLE) > 0); 3823 } 3824 3825 return (0); 3826 } 3827 3828 void 3829 pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov) 3830 { 3831 struct pci_ea_entry *ea; 3832 struct pci_devinfo *dinfo; 3833 pci_addr_t start, end, count; 3834 struct resource_list *rl; 3835 int type, flags, rid; 3836 struct resource *res; 3837 uint32_t tmp; 3838 #ifdef PCI_IOV 3839 struct pcicfg_iov *iov; 3840 #endif 3841 3842 dinfo = device_get_ivars(dev); 3843 rl = &dinfo->resources; 3844 flags = 0; 3845 3846 #ifdef PCI_IOV 3847 iov = dinfo->cfg.iov; 3848 #endif 3849 3850 if (dinfo->cfg.ea.ea_location == 0) 3851 return; 3852 3853 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) { 3854 3855 /* 3856 * TODO: Ignore EA-BAR if is not enabled. 3857 * Currently the EA implementation supports 3858 * only situation, where EA structure contains 3859 * predefined entries. In case they are not enabled 3860 * leave them unallocated and proceed with 3861 * a legacy-BAR mechanism. 3862 */ 3863 if ((ea->eae_flags & PCIM_EA_ENABLE) == 0) 3864 continue; 3865 3866 switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) { 3867 case PCIM_EA_P_MEM_PREFETCH: 3868 case PCIM_EA_P_VF_MEM_PREFETCH: 3869 flags = RF_PREFETCHABLE; 3870 /* FALLTHROUGH */ 3871 case PCIM_EA_P_VF_MEM: 3872 case PCIM_EA_P_MEM: 3873 type = SYS_RES_MEMORY; 3874 break; 3875 case PCIM_EA_P_IO: 3876 type = SYS_RES_IOPORT; 3877 break; 3878 default: 3879 continue; 3880 } 3881 3882 if (alloc_iov != 0) { 3883 #ifdef PCI_IOV 3884 /* Allocating IOV, confirm BEI matches */ 3885 if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) || 3886 (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5)) 3887 continue; 3888 #else 3889 continue; 3890 #endif 3891 } else { 3892 /* Allocating BAR, confirm BEI matches */ 3893 if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) || 3894 (ea->eae_bei > PCIM_EA_BEI_BAR_5)) && 3895 (ea->eae_bei != PCIM_EA_BEI_ROM)) 3896 continue; 3897 } 3898 3899 rid = pci_ea_bei_to_rid(dev, ea->eae_bei); 3900 if (rid < 0) 3901 continue; 3902 3903 /* Skip resources already allocated by EA */ 3904 if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) || 3905 (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL)) 3906 continue; 3907 3908 start = ea->eae_base; 3909 count = ea->eae_max_offset + 1; 3910 #ifdef PCI_IOV 3911 if (iov != NULL) 3912 count = count * iov->iov_num_vfs; 3913 #endif 3914 end = start + count - 1; 3915 if (count == 0) 3916 continue; 3917 3918 resource_list_add(rl, type, rid, start, end, count); 3919 res = resource_list_reserve(rl, bus, dev, type, &rid, start, end, count, 3920 flags); 3921 if (res == NULL) { 3922 resource_list_delete(rl, type, rid); 3923 3924 /* 3925 * Failed to allocate using EA, disable entry. 3926 * Another attempt to allocation will be performed 3927 * further, but this time using legacy BAR registers 3928 */ 3929 tmp = pci_read_config(dev, ea->eae_cfg_offset, 4); 3930 tmp &= ~PCIM_EA_ENABLE; 3931 pci_write_config(dev, ea->eae_cfg_offset, tmp, 4); 3932 3933 /* 3934 * Disabling entry might fail in case it is hardwired. 3935 * Read flags again to match current status. 3936 */ 3937 ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4); 3938 3939 continue; 3940 } 3941 3942 /* As per specification, fill BAR with zeros */ 3943 pci_write_config(dev, rid, 0, 4); 3944 } 3945 } 3946 3947 void 3948 pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask) 3949 { 3950 struct pci_devinfo *dinfo; 3951 pcicfgregs *cfg; 3952 struct resource_list *rl; 3953 const struct pci_quirk *q; 3954 uint32_t devid; 3955 int i; 3956 3957 dinfo = device_get_ivars(dev); 3958 cfg = &dinfo->cfg; 3959 rl = &dinfo->resources; 3960 devid = (cfg->device << 16) | cfg->vendor; 3961 3962 /* Allocate resources using Enhanced Allocation */ 3963 pci_add_resources_ea(bus, dev, 0); 3964 3965 /* ATA devices needs special map treatment */ 3966 if ((pci_get_class(dev) == PCIC_STORAGE) && 3967 (pci_get_subclass(dev) == PCIS_STORAGE_IDE) && 3968 ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) || 3969 (!pci_read_config(dev, PCIR_BAR(0), 4) && 3970 !pci_read_config(dev, PCIR_BAR(2), 4))) ) 3971 pci_ata_maps(bus, dev, rl, force, prefetchmask); 3972 else 3973 for (i = 0; i < cfg->nummaps;) { 3974 /* Skip resources already managed by EA */ 3975 if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) || 3976 (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) || 3977 pci_ea_is_enabled(dev, PCIR_BAR(i))) { 3978 i++; 3979 continue; 3980 } 3981 3982 /* 3983 * Skip quirked resources. 3984 */ 3985 for (q = &pci_quirks[0]; q->devid != 0; q++) 3986 if (q->devid == devid && 3987 q->type == PCI_QUIRK_UNMAP_REG && 3988 q->arg1 == PCIR_BAR(i)) 3989 break; 3990 if (q->devid != 0) { 3991 i++; 3992 continue; 3993 } 3994 i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force, 3995 prefetchmask & (1 << i)); 3996 } 3997 3998 /* 3999 * Add additional, quirked resources. 4000 */ 4001 for (q = &pci_quirks[0]; q->devid != 0; q++) 4002 if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG) 4003 pci_add_map(bus, dev, q->arg1, rl, force, 0); 4004 4005 if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) { 4006 #ifdef __PCI_REROUTE_INTERRUPT 4007 /* 4008 * Try to re-route interrupts. Sometimes the BIOS or 4009 * firmware may leave bogus values in these registers. 4010 * If the re-route fails, then just stick with what we 4011 * have. 4012 */ 4013 pci_assign_interrupt(bus, dev, 1); 4014 #else 4015 pci_assign_interrupt(bus, dev, 0); 4016 #endif 4017 } 4018 4019 if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS && 4020 pci_get_subclass(dev) == PCIS_SERIALBUS_USB) { 4021 if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI) 4022 xhci_early_takeover(dev); 4023 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI) 4024 ehci_early_takeover(dev); 4025 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI) 4026 ohci_early_takeover(dev); 4027 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI) 4028 uhci_early_takeover(dev); 4029 } 4030 4031 #if defined(NEW_PCIB) && defined(PCI_RES_BUS) 4032 /* 4033 * Reserve resources for secondary bus ranges behind bridge 4034 * devices. 4035 */ 4036 pci_reserve_secbus(bus, dev, cfg, rl); 4037 #endif 4038 } 4039 4040 static struct pci_devinfo * 4041 pci_identify_function(device_t pcib, device_t dev, int domain, int busno, 4042 int slot, int func) 4043 { 4044 struct pci_devinfo *dinfo; 4045 4046 dinfo = pci_read_device(pcib, dev, domain, busno, slot, func); 4047 if (dinfo != NULL) 4048 pci_add_child(dev, dinfo); 4049 4050 return (dinfo); 4051 } 4052 4053 void 4054 pci_add_children(device_t dev, int domain, int busno) 4055 { 4056 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w) 4057 device_t pcib = device_get_parent(dev); 4058 struct pci_devinfo *dinfo; 4059 int maxslots; 4060 int s, f, pcifunchigh; 4061 uint8_t hdrtype; 4062 int first_func; 4063 4064 /* 4065 * Try to detect a device at slot 0, function 0. If it exists, try to 4066 * enable ARI. We must enable ARI before detecting the rest of the 4067 * functions on this bus as ARI changes the set of slots and functions 4068 * that are legal on this bus. 4069 */ 4070 dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0); 4071 if (dinfo != NULL && pci_enable_ari) 4072 PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev); 4073 4074 /* 4075 * Start looking for new devices on slot 0 at function 1 because we 4076 * just identified the device at slot 0, function 0. 4077 */ 4078 first_func = 1; 4079 4080 maxslots = PCIB_MAXSLOTS(pcib); 4081 for (s = 0; s <= maxslots; s++, first_func = 0) { 4082 pcifunchigh = 0; 4083 f = 0; 4084 DELAY(1); 4085 hdrtype = REG(PCIR_HDRTYPE, 1); 4086 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE) 4087 continue; 4088 if (hdrtype & PCIM_MFDEV) 4089 pcifunchigh = PCIB_MAXFUNCS(pcib); 4090 for (f = first_func; f <= pcifunchigh; f++) 4091 pci_identify_function(pcib, dev, domain, busno, s, f); 4092 } 4093 #undef REG 4094 } 4095 4096 int 4097 pci_rescan_method(device_t dev) 4098 { 4099 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w) 4100 device_t pcib = device_get_parent(dev); 4101 device_t child, *devlist, *unchanged; 4102 int devcount, error, i, j, maxslots, oldcount; 4103 int busno, domain, s, f, pcifunchigh; 4104 uint8_t hdrtype; 4105 4106 /* No need to check for ARI on a rescan. */ 4107 error = device_get_children(dev, &devlist, &devcount); 4108 if (error) 4109 return (error); 4110 if (devcount != 0) { 4111 unchanged = malloc(devcount * sizeof(device_t), M_TEMP, 4112 M_NOWAIT | M_ZERO); 4113 if (unchanged == NULL) { 4114 free(devlist, M_TEMP); 4115 return (ENOMEM); 4116 } 4117 } else 4118 unchanged = NULL; 4119 4120 domain = pcib_get_domain(dev); 4121 busno = pcib_get_bus(dev); 4122 maxslots = PCIB_MAXSLOTS(pcib); 4123 for (s = 0; s <= maxslots; s++) { 4124 /* If function 0 is not present, skip to the next slot. */ 4125 f = 0; 4126 if (REG(PCIR_VENDOR, 2) == 0xffff) 4127 continue; 4128 pcifunchigh = 0; 4129 hdrtype = REG(PCIR_HDRTYPE, 1); 4130 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE) 4131 continue; 4132 if (hdrtype & PCIM_MFDEV) 4133 pcifunchigh = PCIB_MAXFUNCS(pcib); 4134 for (f = 0; f <= pcifunchigh; f++) { 4135 if (REG(PCIR_VENDOR, 2) == 0xffff) 4136 continue; 4137 4138 /* 4139 * Found a valid function. Check if a 4140 * device_t for this device already exists. 4141 */ 4142 for (i = 0; i < devcount; i++) { 4143 child = devlist[i]; 4144 if (child == NULL) 4145 continue; 4146 if (pci_get_slot(child) == s && 4147 pci_get_function(child) == f) { 4148 unchanged[i] = child; 4149 goto next_func; 4150 } 4151 } 4152 4153 pci_identify_function(pcib, dev, domain, busno, s, f); 4154 next_func:; 4155 } 4156 } 4157 4158 /* Remove devices that are no longer present. */ 4159 for (i = 0; i < devcount; i++) { 4160 if (unchanged[i] != NULL) 4161 continue; 4162 device_delete_child(dev, devlist[i]); 4163 } 4164 4165 free(devlist, M_TEMP); 4166 oldcount = devcount; 4167 4168 /* Try to attach the devices just added. */ 4169 error = device_get_children(dev, &devlist, &devcount); 4170 if (error) { 4171 free(unchanged, M_TEMP); 4172 return (error); 4173 } 4174 4175 for (i = 0; i < devcount; i++) { 4176 for (j = 0; j < oldcount; j++) { 4177 if (devlist[i] == unchanged[j]) 4178 goto next_device; 4179 } 4180 4181 device_probe_and_attach(devlist[i]); 4182 next_device:; 4183 } 4184 4185 free(unchanged, M_TEMP); 4186 free(devlist, M_TEMP); 4187 return (0); 4188 #undef REG 4189 } 4190 4191 #ifdef PCI_IOV 4192 device_t 4193 pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid, 4194 uint16_t did) 4195 { 4196 struct pci_devinfo *vf_dinfo; 4197 device_t pcib; 4198 int busno, slot, func; 4199 4200 pcib = device_get_parent(bus); 4201 4202 PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func); 4203 4204 vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno, 4205 slot, func, vid, did); 4206 4207 vf_dinfo->cfg.flags |= PCICFG_VF; 4208 pci_add_child(bus, vf_dinfo); 4209 4210 return (vf_dinfo->cfg.dev); 4211 } 4212 4213 device_t 4214 pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid, 4215 uint16_t vid, uint16_t did) 4216 { 4217 4218 return (pci_add_iov_child(bus, pf, rid, vid, did)); 4219 } 4220 #endif 4221 4222 static void 4223 pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo) 4224 { 4225 int aer; 4226 uint32_t r; 4227 uint16_t r2; 4228 4229 if (dinfo->cfg.pcie.pcie_location != 0 && 4230 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) { 4231 r2 = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + 4232 PCIER_ROOT_CTL, 2); 4233 r2 &= ~(PCIEM_ROOT_CTL_SERR_CORR | 4234 PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL); 4235 pci_write_config(dev, dinfo->cfg.pcie.pcie_location + 4236 PCIER_ROOT_CTL, r2, 2); 4237 } 4238 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) { 4239 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); 4240 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4); 4241 if (r != 0 && bootverbose) { 4242 pci_printf(&dinfo->cfg, 4243 "clearing AER UC 0x%08x -> 0x%08x\n", 4244 r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4245 4)); 4246 } 4247 4248 r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4); 4249 r &= ~(PCIM_AER_UC_TRAINING_ERROR | 4250 PCIM_AER_UC_DL_PROTOCOL_ERROR | 4251 PCIM_AER_UC_SURPRISE_LINK_DOWN | 4252 PCIM_AER_UC_POISONED_TLP | 4253 PCIM_AER_UC_FC_PROTOCOL_ERROR | 4254 PCIM_AER_UC_COMPLETION_TIMEOUT | 4255 PCIM_AER_UC_COMPLETER_ABORT | 4256 PCIM_AER_UC_UNEXPECTED_COMPLETION | 4257 PCIM_AER_UC_RECEIVER_OVERFLOW | 4258 PCIM_AER_UC_MALFORMED_TLP | 4259 PCIM_AER_UC_ECRC_ERROR | 4260 PCIM_AER_UC_UNSUPPORTED_REQUEST | 4261 PCIM_AER_UC_ACS_VIOLATION | 4262 PCIM_AER_UC_INTERNAL_ERROR | 4263 PCIM_AER_UC_MC_BLOCKED_TLP | 4264 PCIM_AER_UC_ATOMIC_EGRESS_BLK | 4265 PCIM_AER_UC_TLP_PREFIX_BLOCKED); 4266 pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4); 4267 4268 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); 4269 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4); 4270 if (r != 0 && bootverbose) { 4271 pci_printf(&dinfo->cfg, 4272 "clearing AER COR 0x%08x -> 0x%08x\n", 4273 r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4274 4)); 4275 } 4276 4277 r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4); 4278 r &= ~(PCIM_AER_COR_RECEIVER_ERROR | 4279 PCIM_AER_COR_BAD_TLP | 4280 PCIM_AER_COR_BAD_DLLP | 4281 PCIM_AER_COR_REPLAY_ROLLOVER | 4282 PCIM_AER_COR_REPLAY_TIMEOUT | 4283 PCIM_AER_COR_ADVISORY_NF_ERROR | 4284 PCIM_AER_COR_INTERNAL_ERROR | 4285 PCIM_AER_COR_HEADER_LOG_OVFLOW); 4286 pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4); 4287 4288 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + 4289 PCIER_DEVICE_CTL, 2); 4290 r |= PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE | 4291 PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE; 4292 pci_write_config(dev, dinfo->cfg.pcie.pcie_location + 4293 PCIER_DEVICE_CTL, r, 2); 4294 } 4295 } 4296 4297 void 4298 pci_add_child(device_t bus, struct pci_devinfo *dinfo) 4299 { 4300 device_t dev; 4301 4302 dinfo->cfg.dev = dev = device_add_child(bus, NULL, -1); 4303 device_set_ivars(dev, dinfo); 4304 resource_list_init(&dinfo->resources); 4305 pci_cfg_save(dev, dinfo, 0); 4306 pci_cfg_restore(dev, dinfo); 4307 pci_print_verbose(dinfo); 4308 pci_add_resources(bus, dev, 0, 0); 4309 pci_child_added(dinfo->cfg.dev); 4310 4311 if (pci_clear_aer_on_attach) 4312 pci_add_child_clear_aer(dev, dinfo); 4313 4314 EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev); 4315 } 4316 4317 void 4318 pci_child_added_method(device_t dev, device_t child) 4319 { 4320 4321 } 4322 4323 static int 4324 pci_probe(device_t dev) 4325 { 4326 4327 device_set_desc(dev, "PCI bus"); 4328 4329 /* Allow other subclasses to override this driver. */ 4330 return (BUS_PROBE_GENERIC); 4331 } 4332 4333 int 4334 pci_attach_common(device_t dev) 4335 { 4336 struct pci_softc *sc; 4337 int busno, domain; 4338 #ifdef PCI_DMA_BOUNDARY 4339 int error, tag_valid; 4340 #endif 4341 #ifdef PCI_RES_BUS 4342 int rid; 4343 #endif 4344 4345 sc = device_get_softc(dev); 4346 domain = pcib_get_domain(dev); 4347 busno = pcib_get_bus(dev); 4348 #ifdef PCI_RES_BUS 4349 rid = 0; 4350 sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno, 4351 1, 0); 4352 if (sc->sc_bus == NULL) { 4353 device_printf(dev, "failed to allocate bus number\n"); 4354 return (ENXIO); 4355 } 4356 #endif 4357 if (bootverbose) 4358 device_printf(dev, "domain=%d, physical bus=%d\n", 4359 domain, busno); 4360 #ifdef PCI_DMA_BOUNDARY 4361 tag_valid = 0; 4362 if (device_get_devclass(device_get_parent(device_get_parent(dev))) != 4363 devclass_find("pci")) { 4364 error = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 4365 PCI_DMA_BOUNDARY, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, 4366 NULL, NULL, BUS_SPACE_MAXSIZE, BUS_SPACE_UNRESTRICTED, 4367 BUS_SPACE_MAXSIZE, 0, NULL, NULL, &sc->sc_dma_tag); 4368 if (error) 4369 device_printf(dev, "Failed to create DMA tag: %d\n", 4370 error); 4371 else 4372 tag_valid = 1; 4373 } 4374 if (!tag_valid) 4375 #endif 4376 sc->sc_dma_tag = bus_get_dma_tag(dev); 4377 return (0); 4378 } 4379 4380 static int 4381 pci_attach(device_t dev) 4382 { 4383 int busno, domain, error; 4384 4385 error = pci_attach_common(dev); 4386 if (error) 4387 return (error); 4388 4389 /* 4390 * Since there can be multiple independently numbered PCI 4391 * buses on systems with multiple PCI domains, we can't use 4392 * the unit number to decide which bus we are probing. We ask 4393 * the parent pcib what our domain and bus numbers are. 4394 */ 4395 domain = pcib_get_domain(dev); 4396 busno = pcib_get_bus(dev); 4397 pci_add_children(dev, domain, busno); 4398 return (bus_generic_attach(dev)); 4399 } 4400 4401 static int 4402 pci_detach(device_t dev) 4403 { 4404 #ifdef PCI_RES_BUS 4405 struct pci_softc *sc; 4406 #endif 4407 int error; 4408 4409 error = bus_generic_detach(dev); 4410 if (error) 4411 return (error); 4412 #ifdef PCI_RES_BUS 4413 sc = device_get_softc(dev); 4414 error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus); 4415 if (error) 4416 return (error); 4417 #endif 4418 return (device_delete_children(dev)); 4419 } 4420 4421 static void 4422 pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp) 4423 { 4424 int line, unit; 4425 const char *at; 4426 char me1[24], me2[32]; 4427 uint8_t b, s, f; 4428 uint32_t d; 4429 4430 d = pci_get_domain(child); 4431 b = pci_get_bus(child); 4432 s = pci_get_slot(child); 4433 f = pci_get_function(child); 4434 snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f); 4435 snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f); 4436 line = 0; 4437 while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) { 4438 resource_string_value(name, unit, "at", &at); 4439 if (strcmp(at, me1) != 0 && strcmp(at, me2) != 0) 4440 continue; /* No match, try next candidate */ 4441 *unitp = unit; 4442 return; 4443 } 4444 } 4445 4446 static void 4447 pci_set_power_child(device_t dev, device_t child, int state) 4448 { 4449 device_t pcib; 4450 int dstate; 4451 4452 /* 4453 * Set the device to the given state. If the firmware suggests 4454 * a different power state, use it instead. If power management 4455 * is not present, the firmware is responsible for managing 4456 * device power. Skip children who aren't attached since they 4457 * are handled separately. 4458 */ 4459 pcib = device_get_parent(dev); 4460 dstate = state; 4461 if (device_is_attached(child) && 4462 PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0) 4463 pci_set_powerstate(child, dstate); 4464 } 4465 4466 int 4467 pci_suspend_child(device_t dev, device_t child) 4468 { 4469 struct pci_devinfo *dinfo; 4470 struct resource_list_entry *rle; 4471 int error; 4472 4473 dinfo = device_get_ivars(child); 4474 4475 /* 4476 * Save the PCI configuration space for the child and set the 4477 * device in the appropriate power state for this sleep state. 4478 */ 4479 pci_cfg_save(child, dinfo, 0); 4480 4481 /* Suspend devices before potentially powering them down. */ 4482 error = bus_generic_suspend_child(dev, child); 4483 4484 if (error) 4485 return (error); 4486 4487 if (pci_do_power_suspend) { 4488 /* 4489 * Make sure this device's interrupt handler is not invoked 4490 * in the case the device uses a shared interrupt that can 4491 * be raised by some other device. 4492 * This is applicable only to regular (legacy) PCI interrupts 4493 * as MSI/MSI-X interrupts are never shared. 4494 */ 4495 rle = resource_list_find(&dinfo->resources, 4496 SYS_RES_IRQ, 0); 4497 if (rle != NULL && rle->res != NULL) 4498 (void)bus_suspend_intr(child, rle->res); 4499 pci_set_power_child(dev, child, PCI_POWERSTATE_D3); 4500 } 4501 4502 return (0); 4503 } 4504 4505 int 4506 pci_resume_child(device_t dev, device_t child) 4507 { 4508 struct pci_devinfo *dinfo; 4509 struct resource_list_entry *rle; 4510 4511 if (pci_do_power_resume) 4512 pci_set_power_child(dev, child, PCI_POWERSTATE_D0); 4513 4514 dinfo = device_get_ivars(child); 4515 pci_cfg_restore(child, dinfo); 4516 if (!device_is_attached(child)) 4517 pci_cfg_save(child, dinfo, 1); 4518 4519 bus_generic_resume_child(dev, child); 4520 4521 /* 4522 * Allow interrupts only after fully resuming the driver and hardware. 4523 */ 4524 if (pci_do_power_suspend) { 4525 /* See pci_suspend_child for details. */ 4526 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); 4527 if (rle != NULL && rle->res != NULL) 4528 (void)bus_resume_intr(child, rle->res); 4529 } 4530 4531 return (0); 4532 } 4533 4534 int 4535 pci_resume(device_t dev) 4536 { 4537 device_t child, *devlist; 4538 int error, i, numdevs; 4539 4540 if ((error = device_get_children(dev, &devlist, &numdevs)) != 0) 4541 return (error); 4542 4543 /* 4544 * Resume critical devices first, then everything else later. 4545 */ 4546 for (i = 0; i < numdevs; i++) { 4547 child = devlist[i]; 4548 switch (pci_get_class(child)) { 4549 case PCIC_DISPLAY: 4550 case PCIC_MEMORY: 4551 case PCIC_BRIDGE: 4552 case PCIC_BASEPERIPH: 4553 BUS_RESUME_CHILD(dev, child); 4554 break; 4555 } 4556 } 4557 for (i = 0; i < numdevs; i++) { 4558 child = devlist[i]; 4559 switch (pci_get_class(child)) { 4560 case PCIC_DISPLAY: 4561 case PCIC_MEMORY: 4562 case PCIC_BRIDGE: 4563 case PCIC_BASEPERIPH: 4564 break; 4565 default: 4566 BUS_RESUME_CHILD(dev, child); 4567 } 4568 } 4569 free(devlist, M_TEMP); 4570 return (0); 4571 } 4572 4573 static void 4574 pci_load_vendor_data(void) 4575 { 4576 caddr_t data; 4577 void *ptr; 4578 size_t sz; 4579 4580 data = preload_search_by_type("pci_vendor_data"); 4581 if (data != NULL) { 4582 ptr = preload_fetch_addr(data); 4583 sz = preload_fetch_size(data); 4584 if (ptr != NULL && sz != 0) { 4585 pci_vendordata = ptr; 4586 pci_vendordata_size = sz; 4587 /* terminate the database */ 4588 pci_vendordata[pci_vendordata_size] = '\n'; 4589 } 4590 } 4591 } 4592 4593 void 4594 pci_driver_added(device_t dev, driver_t *driver) 4595 { 4596 int numdevs; 4597 device_t *devlist; 4598 device_t child; 4599 struct pci_devinfo *dinfo; 4600 int i; 4601 4602 if (bootverbose) 4603 device_printf(dev, "driver added\n"); 4604 DEVICE_IDENTIFY(driver, dev); 4605 if (device_get_children(dev, &devlist, &numdevs) != 0) 4606 return; 4607 for (i = 0; i < numdevs; i++) { 4608 child = devlist[i]; 4609 if (device_get_state(child) != DS_NOTPRESENT) 4610 continue; 4611 dinfo = device_get_ivars(child); 4612 pci_print_verbose(dinfo); 4613 if (bootverbose) 4614 pci_printf(&dinfo->cfg, "reprobing on driver added\n"); 4615 pci_cfg_restore(child, dinfo); 4616 if (device_probe_and_attach(child) != 0) 4617 pci_child_detached(dev, child); 4618 } 4619 free(devlist, M_TEMP); 4620 } 4621 4622 int 4623 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, 4624 driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep) 4625 { 4626 struct pci_devinfo *dinfo; 4627 struct msix_table_entry *mte; 4628 struct msix_vector *mv; 4629 uint64_t addr; 4630 uint32_t data; 4631 void *cookie; 4632 int error, rid; 4633 4634 error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr, 4635 arg, &cookie); 4636 if (error) 4637 return (error); 4638 4639 /* If this is not a direct child, just bail out. */ 4640 if (device_get_parent(child) != dev) { 4641 *cookiep = cookie; 4642 return(0); 4643 } 4644 4645 rid = rman_get_rid(irq); 4646 if (rid == 0) { 4647 /* Make sure that INTx is enabled */ 4648 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS); 4649 } else { 4650 /* 4651 * Check to see if the interrupt is MSI or MSI-X. 4652 * Ask our parent to map the MSI and give 4653 * us the address and data register values. 4654 * If we fail for some reason, teardown the 4655 * interrupt handler. 4656 */ 4657 dinfo = device_get_ivars(child); 4658 if (dinfo->cfg.msi.msi_alloc > 0) { 4659 if (dinfo->cfg.msi.msi_addr == 0) { 4660 KASSERT(dinfo->cfg.msi.msi_handlers == 0, 4661 ("MSI has handlers, but vectors not mapped")); 4662 error = PCIB_MAP_MSI(device_get_parent(dev), 4663 child, rman_get_start(irq), &addr, &data); 4664 if (error) 4665 goto bad; 4666 dinfo->cfg.msi.msi_addr = addr; 4667 dinfo->cfg.msi.msi_data = data; 4668 } 4669 if (dinfo->cfg.msi.msi_handlers == 0) 4670 pci_enable_msi(child, dinfo->cfg.msi.msi_addr, 4671 dinfo->cfg.msi.msi_data); 4672 dinfo->cfg.msi.msi_handlers++; 4673 } else { 4674 KASSERT(dinfo->cfg.msix.msix_alloc > 0, 4675 ("No MSI or MSI-X interrupts allocated")); 4676 KASSERT(rid <= dinfo->cfg.msix.msix_table_len, 4677 ("MSI-X index too high")); 4678 mte = &dinfo->cfg.msix.msix_table[rid - 1]; 4679 KASSERT(mte->mte_vector != 0, ("no message vector")); 4680 mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1]; 4681 KASSERT(mv->mv_irq == rman_get_start(irq), 4682 ("IRQ mismatch")); 4683 if (mv->mv_address == 0) { 4684 KASSERT(mte->mte_handlers == 0, 4685 ("MSI-X table entry has handlers, but vector not mapped")); 4686 error = PCIB_MAP_MSI(device_get_parent(dev), 4687 child, rman_get_start(irq), &addr, &data); 4688 if (error) 4689 goto bad; 4690 mv->mv_address = addr; 4691 mv->mv_data = data; 4692 } 4693 4694 /* 4695 * The MSIX table entry must be made valid by 4696 * incrementing the mte_handlers before 4697 * calling pci_enable_msix() and 4698 * pci_resume_msix(). Else the MSIX rewrite 4699 * table quirk will not work as expected. 4700 */ 4701 mte->mte_handlers++; 4702 if (mte->mte_handlers == 1) { 4703 pci_enable_msix(child, rid - 1, mv->mv_address, 4704 mv->mv_data); 4705 pci_unmask_msix(child, rid - 1); 4706 } 4707 } 4708 4709 /* 4710 * Make sure that INTx is disabled if we are using MSI/MSI-X, 4711 * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG, 4712 * in which case we "enable" INTx so MSI/MSI-X actually works. 4713 */ 4714 if (!pci_has_quirk(pci_get_devid(child), 4715 PCI_QUIRK_MSI_INTX_BUG)) 4716 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS); 4717 else 4718 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS); 4719 bad: 4720 if (error) { 4721 (void)bus_generic_teardown_intr(dev, child, irq, 4722 cookie); 4723 return (error); 4724 } 4725 } 4726 *cookiep = cookie; 4727 return (0); 4728 } 4729 4730 int 4731 pci_teardown_intr(device_t dev, device_t child, struct resource *irq, 4732 void *cookie) 4733 { 4734 struct msix_table_entry *mte; 4735 struct resource_list_entry *rle; 4736 struct pci_devinfo *dinfo; 4737 int error, rid; 4738 4739 if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE)) 4740 return (EINVAL); 4741 4742 /* If this isn't a direct child, just bail out */ 4743 if (device_get_parent(child) != dev) 4744 return(bus_generic_teardown_intr(dev, child, irq, cookie)); 4745 4746 rid = rman_get_rid(irq); 4747 if (rid == 0) { 4748 /* Mask INTx */ 4749 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS); 4750 } else { 4751 /* 4752 * Check to see if the interrupt is MSI or MSI-X. If so, 4753 * decrement the appropriate handlers count and mask the 4754 * MSI-X message, or disable MSI messages if the count 4755 * drops to 0. 4756 */ 4757 dinfo = device_get_ivars(child); 4758 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid); 4759 if (rle->res != irq) 4760 return (EINVAL); 4761 if (dinfo->cfg.msi.msi_alloc > 0) { 4762 KASSERT(rid <= dinfo->cfg.msi.msi_alloc, 4763 ("MSI-X index too high")); 4764 if (dinfo->cfg.msi.msi_handlers == 0) 4765 return (EINVAL); 4766 dinfo->cfg.msi.msi_handlers--; 4767 if (dinfo->cfg.msi.msi_handlers == 0) 4768 pci_disable_msi(child); 4769 } else { 4770 KASSERT(dinfo->cfg.msix.msix_alloc > 0, 4771 ("No MSI or MSI-X interrupts allocated")); 4772 KASSERT(rid <= dinfo->cfg.msix.msix_table_len, 4773 ("MSI-X index too high")); 4774 mte = &dinfo->cfg.msix.msix_table[rid - 1]; 4775 if (mte->mte_handlers == 0) 4776 return (EINVAL); 4777 mte->mte_handlers--; 4778 if (mte->mte_handlers == 0) 4779 pci_mask_msix(child, rid - 1); 4780 } 4781 } 4782 error = bus_generic_teardown_intr(dev, child, irq, cookie); 4783 if (rid > 0) 4784 KASSERT(error == 0, 4785 ("%s: generic teardown failed for MSI/MSI-X", __func__)); 4786 return (error); 4787 } 4788 4789 int 4790 pci_print_child(device_t dev, device_t child) 4791 { 4792 struct pci_devinfo *dinfo; 4793 struct resource_list *rl; 4794 int retval = 0; 4795 4796 dinfo = device_get_ivars(child); 4797 rl = &dinfo->resources; 4798 4799 retval += bus_print_child_header(dev, child); 4800 4801 retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx"); 4802 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx"); 4803 retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd"); 4804 if (device_get_flags(dev)) 4805 retval += printf(" flags %#x", device_get_flags(dev)); 4806 4807 retval += printf(" at device %d.%d", pci_get_slot(child), 4808 pci_get_function(child)); 4809 4810 retval += bus_print_child_domain(dev, child); 4811 retval += bus_print_child_footer(dev, child); 4812 4813 return (retval); 4814 } 4815 4816 static const struct 4817 { 4818 int class; 4819 int subclass; 4820 int report; /* 0 = bootverbose, 1 = always */ 4821 const char *desc; 4822 } pci_nomatch_tab[] = { 4823 {PCIC_OLD, -1, 1, "old"}, 4824 {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"}, 4825 {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"}, 4826 {PCIC_STORAGE, -1, 1, "mass storage"}, 4827 {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"}, 4828 {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"}, 4829 {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"}, 4830 {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"}, 4831 {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"}, 4832 {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"}, 4833 {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"}, 4834 {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"}, 4835 {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"}, 4836 {PCIC_NETWORK, -1, 1, "network"}, 4837 {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"}, 4838 {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"}, 4839 {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"}, 4840 {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"}, 4841 {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"}, 4842 {PCIC_DISPLAY, -1, 1, "display"}, 4843 {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"}, 4844 {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"}, 4845 {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"}, 4846 {PCIC_MULTIMEDIA, -1, 1, "multimedia"}, 4847 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"}, 4848 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"}, 4849 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"}, 4850 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"}, 4851 {PCIC_MEMORY, -1, 1, "memory"}, 4852 {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"}, 4853 {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"}, 4854 {PCIC_BRIDGE, -1, 1, "bridge"}, 4855 {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"}, 4856 {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"}, 4857 {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"}, 4858 {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"}, 4859 {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"}, 4860 {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"}, 4861 {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"}, 4862 {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"}, 4863 {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"}, 4864 {PCIC_SIMPLECOMM, -1, 1, "simple comms"}, 4865 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */ 4866 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"}, 4867 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"}, 4868 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"}, 4869 {PCIC_BASEPERIPH, -1, 0, "base peripheral"}, 4870 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"}, 4871 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"}, 4872 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"}, 4873 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"}, 4874 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"}, 4875 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"}, 4876 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"}, 4877 {PCIC_INPUTDEV, -1, 1, "input device"}, 4878 {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"}, 4879 {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"}, 4880 {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"}, 4881 {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"}, 4882 {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"}, 4883 {PCIC_DOCKING, -1, 1, "docking station"}, 4884 {PCIC_PROCESSOR, -1, 1, "processor"}, 4885 {PCIC_SERIALBUS, -1, 1, "serial bus"}, 4886 {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"}, 4887 {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"}, 4888 {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"}, 4889 {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"}, 4890 {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"}, 4891 {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"}, 4892 {PCIC_WIRELESS, -1, 1, "wireless controller"}, 4893 {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"}, 4894 {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"}, 4895 {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"}, 4896 {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"}, 4897 {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"}, 4898 {PCIC_SATCOM, -1, 1, "satellite communication"}, 4899 {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"}, 4900 {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"}, 4901 {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"}, 4902 {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"}, 4903 {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"}, 4904 {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"}, 4905 {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"}, 4906 {PCIC_DASP, -1, 0, "dasp"}, 4907 {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"}, 4908 {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"}, 4909 {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"}, 4910 {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"}, 4911 {0, 0, 0, NULL} 4912 }; 4913 4914 void 4915 pci_probe_nomatch(device_t dev, device_t child) 4916 { 4917 int i, report; 4918 const char *cp, *scp; 4919 char *device; 4920 4921 /* 4922 * Look for a listing for this device in a loaded device database. 4923 */ 4924 report = 1; 4925 if ((device = pci_describe_device(child)) != NULL) { 4926 device_printf(dev, "<%s>", device); 4927 free(device, M_DEVBUF); 4928 } else { 4929 /* 4930 * Scan the class/subclass descriptions for a general 4931 * description. 4932 */ 4933 cp = "unknown"; 4934 scp = NULL; 4935 for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) { 4936 if (pci_nomatch_tab[i].class == pci_get_class(child)) { 4937 if (pci_nomatch_tab[i].subclass == -1) { 4938 cp = pci_nomatch_tab[i].desc; 4939 report = pci_nomatch_tab[i].report; 4940 } else if (pci_nomatch_tab[i].subclass == 4941 pci_get_subclass(child)) { 4942 scp = pci_nomatch_tab[i].desc; 4943 report = pci_nomatch_tab[i].report; 4944 } 4945 } 4946 } 4947 if (report || bootverbose) { 4948 device_printf(dev, "<%s%s%s>", 4949 cp ? cp : "", 4950 ((cp != NULL) && (scp != NULL)) ? ", " : "", 4951 scp ? scp : ""); 4952 } 4953 } 4954 if (report || bootverbose) { 4955 printf(" at device %d.%d (no driver attached)\n", 4956 pci_get_slot(child), pci_get_function(child)); 4957 } 4958 pci_cfg_save(child, device_get_ivars(child), 1); 4959 } 4960 4961 void 4962 pci_child_detached(device_t dev, device_t child) 4963 { 4964 struct pci_devinfo *dinfo; 4965 struct resource_list *rl; 4966 4967 dinfo = device_get_ivars(child); 4968 rl = &dinfo->resources; 4969 4970 /* 4971 * Have to deallocate IRQs before releasing any MSI messages and 4972 * have to release MSI messages before deallocating any memory 4973 * BARs. 4974 */ 4975 if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0) 4976 pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n"); 4977 if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) { 4978 pci_printf(&dinfo->cfg, "Device leaked MSI vectors\n"); 4979 (void)pci_release_msi(child); 4980 } 4981 if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0) 4982 pci_printf(&dinfo->cfg, "Device leaked memory resources\n"); 4983 if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0) 4984 pci_printf(&dinfo->cfg, "Device leaked I/O resources\n"); 4985 #ifdef PCI_RES_BUS 4986 if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0) 4987 pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n"); 4988 #endif 4989 4990 pci_cfg_save(child, dinfo, 1); 4991 } 4992 4993 /* 4994 * Parse the PCI device database, if loaded, and return a pointer to a 4995 * description of the device. 4996 * 4997 * The database is flat text formatted as follows: 4998 * 4999 * Any line not in a valid format is ignored. 5000 * Lines are terminated with newline '\n' characters. 5001 * 5002 * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then 5003 * the vendor name. 5004 * 5005 * A DEVICE line is entered immediately below the corresponding VENDOR ID. 5006 * - devices cannot be listed without a corresponding VENDOR line. 5007 * A DEVICE line consists of a TAB, the 4 digit (hex) device code, 5008 * another TAB, then the device name. 5009 */ 5010 5011 /* 5012 * Assuming (ptr) points to the beginning of a line in the database, 5013 * return the vendor or device and description of the next entry. 5014 * The value of (vendor) or (device) inappropriate for the entry type 5015 * is set to -1. Returns nonzero at the end of the database. 5016 * 5017 * Note that this is slightly unrobust in the face of corrupt data; 5018 * we attempt to safeguard against this by spamming the end of the 5019 * database with a newline when we initialise. 5020 */ 5021 static int 5022 pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc) 5023 { 5024 char *cp = *ptr; 5025 int left; 5026 5027 *device = -1; 5028 *vendor = -1; 5029 **desc = '\0'; 5030 for (;;) { 5031 left = pci_vendordata_size - (cp - pci_vendordata); 5032 if (left <= 0) { 5033 *ptr = cp; 5034 return(1); 5035 } 5036 5037 /* vendor entry? */ 5038 if (*cp != '\t' && 5039 sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2) 5040 break; 5041 /* device entry? */ 5042 if (*cp == '\t' && 5043 sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2) 5044 break; 5045 5046 /* skip to next line */ 5047 while (*cp != '\n' && left > 0) { 5048 cp++; 5049 left--; 5050 } 5051 if (*cp == '\n') { 5052 cp++; 5053 left--; 5054 } 5055 } 5056 /* skip to next line */ 5057 while (*cp != '\n' && left > 0) { 5058 cp++; 5059 left--; 5060 } 5061 if (*cp == '\n' && left > 0) 5062 cp++; 5063 *ptr = cp; 5064 return(0); 5065 } 5066 5067 static char * 5068 pci_describe_device(device_t dev) 5069 { 5070 int vendor, device; 5071 char *desc, *vp, *dp, *line; 5072 5073 desc = vp = dp = NULL; 5074 5075 /* 5076 * If we have no vendor data, we can't do anything. 5077 */ 5078 if (pci_vendordata == NULL) 5079 goto out; 5080 5081 /* 5082 * Scan the vendor data looking for this device 5083 */ 5084 line = pci_vendordata; 5085 if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL) 5086 goto out; 5087 for (;;) { 5088 if (pci_describe_parse_line(&line, &vendor, &device, &vp)) 5089 goto out; 5090 if (vendor == pci_get_vendor(dev)) 5091 break; 5092 } 5093 if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL) 5094 goto out; 5095 for (;;) { 5096 if (pci_describe_parse_line(&line, &vendor, &device, &dp)) { 5097 *dp = 0; 5098 break; 5099 } 5100 if (vendor != -1) { 5101 *dp = 0; 5102 break; 5103 } 5104 if (device == pci_get_device(dev)) 5105 break; 5106 } 5107 if (dp[0] == '\0') 5108 snprintf(dp, 80, "0x%x", pci_get_device(dev)); 5109 if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) != 5110 NULL) 5111 sprintf(desc, "%s, %s", vp, dp); 5112 out: 5113 if (vp != NULL) 5114 free(vp, M_DEVBUF); 5115 if (dp != NULL) 5116 free(dp, M_DEVBUF); 5117 return(desc); 5118 } 5119 5120 int 5121 pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) 5122 { 5123 struct pci_devinfo *dinfo; 5124 pcicfgregs *cfg; 5125 5126 dinfo = device_get_ivars(child); 5127 cfg = &dinfo->cfg; 5128 5129 switch (which) { 5130 case PCI_IVAR_ETHADDR: 5131 /* 5132 * The generic accessor doesn't deal with failure, so 5133 * we set the return value, then return an error. 5134 */ 5135 *((uint8_t **) result) = NULL; 5136 return (EINVAL); 5137 case PCI_IVAR_SUBVENDOR: 5138 *result = cfg->subvendor; 5139 break; 5140 case PCI_IVAR_SUBDEVICE: 5141 *result = cfg->subdevice; 5142 break; 5143 case PCI_IVAR_VENDOR: 5144 *result = cfg->vendor; 5145 break; 5146 case PCI_IVAR_DEVICE: 5147 *result = cfg->device; 5148 break; 5149 case PCI_IVAR_DEVID: 5150 *result = (cfg->device << 16) | cfg->vendor; 5151 break; 5152 case PCI_IVAR_CLASS: 5153 *result = cfg->baseclass; 5154 break; 5155 case PCI_IVAR_SUBCLASS: 5156 *result = cfg->subclass; 5157 break; 5158 case PCI_IVAR_PROGIF: 5159 *result = cfg->progif; 5160 break; 5161 case PCI_IVAR_REVID: 5162 *result = cfg->revid; 5163 break; 5164 case PCI_IVAR_INTPIN: 5165 *result = cfg->intpin; 5166 break; 5167 case PCI_IVAR_IRQ: 5168 *result = cfg->intline; 5169 break; 5170 case PCI_IVAR_DOMAIN: 5171 *result = cfg->domain; 5172 break; 5173 case PCI_IVAR_BUS: 5174 *result = cfg->bus; 5175 break; 5176 case PCI_IVAR_SLOT: 5177 *result = cfg->slot; 5178 break; 5179 case PCI_IVAR_FUNCTION: 5180 *result = cfg->func; 5181 break; 5182 case PCI_IVAR_CMDREG: 5183 *result = cfg->cmdreg; 5184 break; 5185 case PCI_IVAR_CACHELNSZ: 5186 *result = cfg->cachelnsz; 5187 break; 5188 case PCI_IVAR_MINGNT: 5189 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) { 5190 *result = -1; 5191 return (EINVAL); 5192 } 5193 *result = cfg->mingnt; 5194 break; 5195 case PCI_IVAR_MAXLAT: 5196 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) { 5197 *result = -1; 5198 return (EINVAL); 5199 } 5200 *result = cfg->maxlat; 5201 break; 5202 case PCI_IVAR_LATTIMER: 5203 *result = cfg->lattimer; 5204 break; 5205 default: 5206 return (ENOENT); 5207 } 5208 return (0); 5209 } 5210 5211 int 5212 pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value) 5213 { 5214 struct pci_devinfo *dinfo; 5215 5216 dinfo = device_get_ivars(child); 5217 5218 switch (which) { 5219 case PCI_IVAR_INTPIN: 5220 dinfo->cfg.intpin = value; 5221 return (0); 5222 case PCI_IVAR_ETHADDR: 5223 case PCI_IVAR_SUBVENDOR: 5224 case PCI_IVAR_SUBDEVICE: 5225 case PCI_IVAR_VENDOR: 5226 case PCI_IVAR_DEVICE: 5227 case PCI_IVAR_DEVID: 5228 case PCI_IVAR_CLASS: 5229 case PCI_IVAR_SUBCLASS: 5230 case PCI_IVAR_PROGIF: 5231 case PCI_IVAR_REVID: 5232 case PCI_IVAR_IRQ: 5233 case PCI_IVAR_DOMAIN: 5234 case PCI_IVAR_BUS: 5235 case PCI_IVAR_SLOT: 5236 case PCI_IVAR_FUNCTION: 5237 return (EINVAL); /* disallow for now */ 5238 5239 default: 5240 return (ENOENT); 5241 } 5242 } 5243 5244 #include "opt_ddb.h" 5245 #ifdef DDB 5246 #include <ddb/ddb.h> 5247 #include <sys/cons.h> 5248 5249 /* 5250 * List resources based on pci map registers, used for within ddb 5251 */ 5252 5253 DB_SHOW_COMMAND(pciregs, db_pci_dump) 5254 { 5255 struct pci_devinfo *dinfo; 5256 struct devlist *devlist_head; 5257 struct pci_conf *p; 5258 const char *name; 5259 int i, error, none_count; 5260 5261 none_count = 0; 5262 /* get the head of the device queue */ 5263 devlist_head = &pci_devq; 5264 5265 /* 5266 * Go through the list of devices and print out devices 5267 */ 5268 for (error = 0, i = 0, 5269 dinfo = STAILQ_FIRST(devlist_head); 5270 (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit; 5271 dinfo = STAILQ_NEXT(dinfo, pci_links), i++) { 5272 5273 /* Populate pd_name and pd_unit */ 5274 name = NULL; 5275 if (dinfo->cfg.dev) 5276 name = device_get_name(dinfo->cfg.dev); 5277 5278 p = &dinfo->conf; 5279 db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x " 5280 "chip=0x%08x rev=0x%02x hdr=0x%02x\n", 5281 (name && *name) ? name : "none", 5282 (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) : 5283 none_count++, 5284 p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev, 5285 p->pc_sel.pc_func, (p->pc_class << 16) | 5286 (p->pc_subclass << 8) | p->pc_progif, 5287 (p->pc_subdevice << 16) | p->pc_subvendor, 5288 (p->pc_device << 16) | p->pc_vendor, 5289 p->pc_revid, p->pc_hdr); 5290 } 5291 } 5292 #endif /* DDB */ 5293 5294 static struct resource * 5295 pci_reserve_map(device_t dev, device_t child, int type, int *rid, 5296 rman_res_t start, rman_res_t end, rman_res_t count, u_int num, 5297 u_int flags) 5298 { 5299 struct pci_devinfo *dinfo = device_get_ivars(child); 5300 struct resource_list *rl = &dinfo->resources; 5301 struct resource *res; 5302 struct pci_map *pm; 5303 uint16_t cmd; 5304 pci_addr_t map, testval; 5305 int mapsize; 5306 5307 res = NULL; 5308 5309 /* If rid is managed by EA, ignore it */ 5310 if (pci_ea_is_enabled(child, *rid)) 5311 goto out; 5312 5313 pm = pci_find_bar(child, *rid); 5314 if (pm != NULL) { 5315 /* This is a BAR that we failed to allocate earlier. */ 5316 mapsize = pm->pm_size; 5317 map = pm->pm_value; 5318 } else { 5319 /* 5320 * Weed out the bogons, and figure out how large the 5321 * BAR/map is. BARs that read back 0 here are bogus 5322 * and unimplemented. Note: atapci in legacy mode are 5323 * special and handled elsewhere in the code. If you 5324 * have a atapci device in legacy mode and it fails 5325 * here, that other code is broken. 5326 */ 5327 pci_read_bar(child, *rid, &map, &testval, NULL); 5328 5329 /* 5330 * Determine the size of the BAR and ignore BARs with a size 5331 * of 0. Device ROM BARs use a different mask value. 5332 */ 5333 if (PCIR_IS_BIOS(&dinfo->cfg, *rid)) 5334 mapsize = pci_romsize(testval); 5335 else 5336 mapsize = pci_mapsize(testval); 5337 if (mapsize == 0) 5338 goto out; 5339 pm = pci_add_bar(child, *rid, map, mapsize); 5340 } 5341 5342 if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, *rid)) { 5343 if (type != SYS_RES_MEMORY) { 5344 if (bootverbose) 5345 device_printf(dev, 5346 "child %s requested type %d for rid %#x," 5347 " but the BAR says it is an memio\n", 5348 device_get_nameunit(child), type, *rid); 5349 goto out; 5350 } 5351 } else { 5352 if (type != SYS_RES_IOPORT) { 5353 if (bootverbose) 5354 device_printf(dev, 5355 "child %s requested type %d for rid %#x," 5356 " but the BAR says it is an ioport\n", 5357 device_get_nameunit(child), type, *rid); 5358 goto out; 5359 } 5360 } 5361 5362 /* 5363 * For real BARs, we need to override the size that 5364 * the driver requests, because that's what the BAR 5365 * actually uses and we would otherwise have a 5366 * situation where we might allocate the excess to 5367 * another driver, which won't work. 5368 */ 5369 count = ((pci_addr_t)1 << mapsize) * num; 5370 if (RF_ALIGNMENT(flags) < mapsize) 5371 flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize); 5372 if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH)) 5373 flags |= RF_PREFETCHABLE; 5374 5375 /* 5376 * Allocate enough resource, and then write back the 5377 * appropriate BAR for that resource. 5378 */ 5379 resource_list_add(rl, type, *rid, start, end, count); 5380 res = resource_list_reserve(rl, dev, child, type, rid, start, end, 5381 count, flags & ~RF_ACTIVE); 5382 if (res == NULL) { 5383 resource_list_delete(rl, type, *rid); 5384 device_printf(child, 5385 "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n", 5386 count, *rid, type, start, end); 5387 goto out; 5388 } 5389 if (bootverbose) 5390 device_printf(child, 5391 "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n", 5392 count, *rid, type, rman_get_start(res)); 5393 5394 /* Disable decoding via the CMD register before updating the BAR */ 5395 cmd = pci_read_config(child, PCIR_COMMAND, 2); 5396 pci_write_config(child, PCIR_COMMAND, 5397 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2); 5398 5399 map = rman_get_start(res); 5400 pci_write_bar(child, pm, map); 5401 5402 /* Restore the original value of the CMD register */ 5403 pci_write_config(child, PCIR_COMMAND, cmd, 2); 5404 out: 5405 return (res); 5406 } 5407 5408 struct resource * 5409 pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid, 5410 rman_res_t start, rman_res_t end, rman_res_t count, u_long num, 5411 u_int flags) 5412 { 5413 struct pci_devinfo *dinfo; 5414 struct resource_list *rl; 5415 struct resource_list_entry *rle; 5416 struct resource *res; 5417 pcicfgregs *cfg; 5418 5419 /* 5420 * Perform lazy resource allocation 5421 */ 5422 dinfo = device_get_ivars(child); 5423 rl = &dinfo->resources; 5424 cfg = &dinfo->cfg; 5425 switch (type) { 5426 #if defined(NEW_PCIB) && defined(PCI_RES_BUS) 5427 case PCI_RES_BUS: 5428 return (pci_alloc_secbus(dev, child, rid, start, end, count, 5429 flags)); 5430 #endif 5431 case SYS_RES_IRQ: 5432 /* 5433 * Can't alloc legacy interrupt once MSI messages have 5434 * been allocated. 5435 */ 5436 if (*rid == 0 && (cfg->msi.msi_alloc > 0 || 5437 cfg->msix.msix_alloc > 0)) 5438 return (NULL); 5439 5440 /* 5441 * If the child device doesn't have an interrupt 5442 * routed and is deserving of an interrupt, try to 5443 * assign it one. 5444 */ 5445 if (*rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) && 5446 (cfg->intpin != 0)) 5447 pci_assign_interrupt(dev, child, 0); 5448 break; 5449 case SYS_RES_IOPORT: 5450 case SYS_RES_MEMORY: 5451 #ifdef NEW_PCIB 5452 /* 5453 * PCI-PCI bridge I/O window resources are not BARs. 5454 * For those allocations just pass the request up the 5455 * tree. 5456 */ 5457 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) { 5458 switch (*rid) { 5459 case PCIR_IOBASEL_1: 5460 case PCIR_MEMBASE_1: 5461 case PCIR_PMBASEL_1: 5462 /* 5463 * XXX: Should we bother creating a resource 5464 * list entry? 5465 */ 5466 return (bus_generic_alloc_resource(dev, child, 5467 type, rid, start, end, count, flags)); 5468 } 5469 } 5470 #endif 5471 /* Reserve resources for this BAR if needed. */ 5472 rle = resource_list_find(rl, type, *rid); 5473 if (rle == NULL) { 5474 res = pci_reserve_map(dev, child, type, rid, start, end, 5475 count, num, flags); 5476 if (res == NULL) 5477 return (NULL); 5478 } 5479 } 5480 return (resource_list_alloc(rl, dev, child, type, rid, 5481 start, end, count, flags)); 5482 } 5483 5484 struct resource * 5485 pci_alloc_resource(device_t dev, device_t child, int type, int *rid, 5486 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) 5487 { 5488 #ifdef PCI_IOV 5489 struct pci_devinfo *dinfo; 5490 #endif 5491 5492 if (device_get_parent(child) != dev) 5493 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child, 5494 type, rid, start, end, count, flags)); 5495 5496 #ifdef PCI_IOV 5497 dinfo = device_get_ivars(child); 5498 if (dinfo->cfg.flags & PCICFG_VF) { 5499 switch (type) { 5500 /* VFs can't have I/O BARs. */ 5501 case SYS_RES_IOPORT: 5502 return (NULL); 5503 case SYS_RES_MEMORY: 5504 return (pci_vf_alloc_mem_resource(dev, child, rid, 5505 start, end, count, flags)); 5506 } 5507 5508 /* Fall through for other types of resource allocations. */ 5509 } 5510 #endif 5511 5512 return (pci_alloc_multi_resource(dev, child, type, rid, start, end, 5513 count, 1, flags)); 5514 } 5515 5516 int 5517 pci_release_resource(device_t dev, device_t child, int type, int rid, 5518 struct resource *r) 5519 { 5520 struct pci_devinfo *dinfo; 5521 struct resource_list *rl; 5522 pcicfgregs *cfg; 5523 5524 if (device_get_parent(child) != dev) 5525 return (BUS_RELEASE_RESOURCE(device_get_parent(dev), child, 5526 type, rid, r)); 5527 5528 dinfo = device_get_ivars(child); 5529 cfg = &dinfo->cfg; 5530 5531 #ifdef PCI_IOV 5532 if (dinfo->cfg.flags & PCICFG_VF) { 5533 switch (type) { 5534 /* VFs can't have I/O BARs. */ 5535 case SYS_RES_IOPORT: 5536 return (EDOOFUS); 5537 case SYS_RES_MEMORY: 5538 return (pci_vf_release_mem_resource(dev, child, rid, 5539 r)); 5540 } 5541 5542 /* Fall through for other types of resource allocations. */ 5543 } 5544 #endif 5545 5546 #ifdef NEW_PCIB 5547 /* 5548 * PCI-PCI bridge I/O window resources are not BARs. For 5549 * those allocations just pass the request up the tree. 5550 */ 5551 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE && 5552 (type == SYS_RES_IOPORT || type == SYS_RES_MEMORY)) { 5553 switch (rid) { 5554 case PCIR_IOBASEL_1: 5555 case PCIR_MEMBASE_1: 5556 case PCIR_PMBASEL_1: 5557 return (bus_generic_release_resource(dev, child, type, 5558 rid, r)); 5559 } 5560 } 5561 #endif 5562 5563 rl = &dinfo->resources; 5564 return (resource_list_release(rl, dev, child, type, rid, r)); 5565 } 5566 5567 int 5568 pci_activate_resource(device_t dev, device_t child, int type, int rid, 5569 struct resource *r) 5570 { 5571 struct pci_devinfo *dinfo; 5572 int error; 5573 5574 error = bus_generic_activate_resource(dev, child, type, rid, r); 5575 if (error) 5576 return (error); 5577 5578 /* Enable decoding in the command register when activating BARs. */ 5579 if (device_get_parent(child) == dev) { 5580 /* Device ROMs need their decoding explicitly enabled. */ 5581 dinfo = device_get_ivars(child); 5582 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid)) 5583 pci_write_bar(child, pci_find_bar(child, rid), 5584 rman_get_start(r) | PCIM_BIOS_ENABLE); 5585 switch (type) { 5586 case SYS_RES_IOPORT: 5587 case SYS_RES_MEMORY: 5588 error = PCI_ENABLE_IO(dev, child, type); 5589 break; 5590 } 5591 } 5592 return (error); 5593 } 5594 5595 int 5596 pci_deactivate_resource(device_t dev, device_t child, int type, 5597 int rid, struct resource *r) 5598 { 5599 struct pci_devinfo *dinfo; 5600 int error; 5601 5602 error = bus_generic_deactivate_resource(dev, child, type, rid, r); 5603 if (error) 5604 return (error); 5605 5606 /* Disable decoding for device ROMs. */ 5607 if (device_get_parent(child) == dev) { 5608 dinfo = device_get_ivars(child); 5609 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid)) 5610 pci_write_bar(child, pci_find_bar(child, rid), 5611 rman_get_start(r)); 5612 } 5613 return (0); 5614 } 5615 5616 void 5617 pci_child_deleted(device_t dev, device_t child) 5618 { 5619 struct resource_list_entry *rle; 5620 struct resource_list *rl; 5621 struct pci_devinfo *dinfo; 5622 5623 dinfo = device_get_ivars(child); 5624 rl = &dinfo->resources; 5625 5626 EVENTHANDLER_INVOKE(pci_delete_device, child); 5627 5628 /* Turn off access to resources we're about to free */ 5629 if (bus_child_present(child) != 0) { 5630 pci_write_config(child, PCIR_COMMAND, pci_read_config(child, 5631 PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2); 5632 5633 pci_disable_busmaster(child); 5634 } 5635 5636 /* Free all allocated resources */ 5637 STAILQ_FOREACH(rle, rl, link) { 5638 if (rle->res) { 5639 if (rman_get_flags(rle->res) & RF_ACTIVE || 5640 resource_list_busy(rl, rle->type, rle->rid)) { 5641 pci_printf(&dinfo->cfg, 5642 "Resource still owned, oops. " 5643 "(type=%d, rid=%d, addr=%lx)\n", 5644 rle->type, rle->rid, 5645 rman_get_start(rle->res)); 5646 bus_release_resource(child, rle->type, rle->rid, 5647 rle->res); 5648 } 5649 resource_list_unreserve(rl, dev, child, rle->type, 5650 rle->rid); 5651 } 5652 } 5653 resource_list_free(rl); 5654 5655 pci_freecfg(dinfo); 5656 } 5657 5658 void 5659 pci_delete_resource(device_t dev, device_t child, int type, int rid) 5660 { 5661 struct pci_devinfo *dinfo; 5662 struct resource_list *rl; 5663 struct resource_list_entry *rle; 5664 5665 if (device_get_parent(child) != dev) 5666 return; 5667 5668 dinfo = device_get_ivars(child); 5669 rl = &dinfo->resources; 5670 rle = resource_list_find(rl, type, rid); 5671 if (rle == NULL) 5672 return; 5673 5674 if (rle->res) { 5675 if (rman_get_flags(rle->res) & RF_ACTIVE || 5676 resource_list_busy(rl, type, rid)) { 5677 device_printf(dev, "delete_resource: " 5678 "Resource still owned by child, oops. " 5679 "(type=%d, rid=%d, addr=%jx)\n", 5680 type, rid, rman_get_start(rle->res)); 5681 return; 5682 } 5683 resource_list_unreserve(rl, dev, child, type, rid); 5684 } 5685 resource_list_delete(rl, type, rid); 5686 } 5687 5688 struct resource_list * 5689 pci_get_resource_list (device_t dev, device_t child) 5690 { 5691 struct pci_devinfo *dinfo = device_get_ivars(child); 5692 5693 return (&dinfo->resources); 5694 } 5695 5696 bus_dma_tag_t 5697 pci_get_dma_tag(device_t bus, device_t dev) 5698 { 5699 struct pci_softc *sc = device_get_softc(bus); 5700 5701 return (sc->sc_dma_tag); 5702 } 5703 5704 uint32_t 5705 pci_read_config_method(device_t dev, device_t child, int reg, int width) 5706 { 5707 struct pci_devinfo *dinfo = device_get_ivars(child); 5708 pcicfgregs *cfg = &dinfo->cfg; 5709 5710 #ifdef PCI_IOV 5711 /* 5712 * SR-IOV VFs don't implement the VID or DID registers, so we have to 5713 * emulate them here. 5714 */ 5715 if (cfg->flags & PCICFG_VF) { 5716 if (reg == PCIR_VENDOR) { 5717 switch (width) { 5718 case 4: 5719 return (cfg->device << 16 | cfg->vendor); 5720 case 2: 5721 return (cfg->vendor); 5722 case 1: 5723 return (cfg->vendor & 0xff); 5724 default: 5725 return (0xffffffff); 5726 } 5727 } else if (reg == PCIR_DEVICE) { 5728 switch (width) { 5729 /* Note that an unaligned 4-byte read is an error. */ 5730 case 2: 5731 return (cfg->device); 5732 case 1: 5733 return (cfg->device & 0xff); 5734 default: 5735 return (0xffffffff); 5736 } 5737 } 5738 } 5739 #endif 5740 5741 return (PCIB_READ_CONFIG(device_get_parent(dev), 5742 cfg->bus, cfg->slot, cfg->func, reg, width)); 5743 } 5744 5745 void 5746 pci_write_config_method(device_t dev, device_t child, int reg, 5747 uint32_t val, int width) 5748 { 5749 struct pci_devinfo *dinfo = device_get_ivars(child); 5750 pcicfgregs *cfg = &dinfo->cfg; 5751 5752 PCIB_WRITE_CONFIG(device_get_parent(dev), 5753 cfg->bus, cfg->slot, cfg->func, reg, val, width); 5754 } 5755 5756 int 5757 pci_child_location_str_method(device_t dev, device_t child, char *buf, 5758 size_t buflen) 5759 { 5760 5761 snprintf(buf, buflen, "slot=%d function=%d dbsf=pci%d:%d:%d:%d", 5762 pci_get_slot(child), pci_get_function(child), pci_get_domain(child), 5763 pci_get_bus(child), pci_get_slot(child), pci_get_function(child)); 5764 return (0); 5765 } 5766 5767 int 5768 pci_child_pnpinfo_str_method(device_t dev, device_t child, char *buf, 5769 size_t buflen) 5770 { 5771 struct pci_devinfo *dinfo; 5772 pcicfgregs *cfg; 5773 5774 dinfo = device_get_ivars(child); 5775 cfg = &dinfo->cfg; 5776 snprintf(buf, buflen, "vendor=0x%04x device=0x%04x subvendor=0x%04x " 5777 "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device, 5778 cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass, 5779 cfg->progif); 5780 return (0); 5781 } 5782 5783 int 5784 pci_assign_interrupt_method(device_t dev, device_t child) 5785 { 5786 struct pci_devinfo *dinfo = device_get_ivars(child); 5787 pcicfgregs *cfg = &dinfo->cfg; 5788 5789 return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child, 5790 cfg->intpin)); 5791 } 5792 5793 static void 5794 pci_lookup(void *arg, const char *name, device_t *dev) 5795 { 5796 long val; 5797 char *end; 5798 int domain, bus, slot, func; 5799 5800 if (*dev != NULL) 5801 return; 5802 5803 /* 5804 * Accept pciconf-style selectors of either pciD:B:S:F or 5805 * pciB:S:F. In the latter case, the domain is assumed to 5806 * be zero. 5807 */ 5808 if (strncmp(name, "pci", 3) != 0) 5809 return; 5810 val = strtol(name + 3, &end, 10); 5811 if (val < 0 || val > INT_MAX || *end != ':') 5812 return; 5813 domain = val; 5814 val = strtol(end + 1, &end, 10); 5815 if (val < 0 || val > INT_MAX || *end != ':') 5816 return; 5817 bus = val; 5818 val = strtol(end + 1, &end, 10); 5819 if (val < 0 || val > INT_MAX) 5820 return; 5821 slot = val; 5822 if (*end == ':') { 5823 val = strtol(end + 1, &end, 10); 5824 if (val < 0 || val > INT_MAX || *end != '\0') 5825 return; 5826 func = val; 5827 } else if (*end == '\0') { 5828 func = slot; 5829 slot = bus; 5830 bus = domain; 5831 domain = 0; 5832 } else 5833 return; 5834 5835 if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX || 5836 func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX)) 5837 return; 5838 5839 *dev = pci_find_dbsf(domain, bus, slot, func); 5840 } 5841 5842 static int 5843 pci_modevent(module_t mod, int what, void *arg) 5844 { 5845 static struct cdev *pci_cdev; 5846 static eventhandler_tag tag; 5847 5848 switch (what) { 5849 case MOD_LOAD: 5850 STAILQ_INIT(&pci_devq); 5851 pci_generation = 0; 5852 pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644, 5853 "pci"); 5854 pci_load_vendor_data(); 5855 tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL, 5856 1000); 5857 break; 5858 5859 case MOD_UNLOAD: 5860 if (tag != NULL) 5861 EVENTHANDLER_DEREGISTER(dev_lookup, tag); 5862 destroy_dev(pci_cdev); 5863 break; 5864 } 5865 5866 return (0); 5867 } 5868 5869 static void 5870 pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo) 5871 { 5872 #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2) 5873 struct pcicfg_pcie *cfg; 5874 int version, pos; 5875 5876 cfg = &dinfo->cfg.pcie; 5877 pos = cfg->pcie_location; 5878 5879 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION; 5880 5881 WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl); 5882 5883 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || 5884 cfg->pcie_type == PCIEM_TYPE_ENDPOINT || 5885 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT) 5886 WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl); 5887 5888 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || 5889 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT && 5890 (cfg->pcie_flags & PCIEM_FLAGS_SLOT)))) 5891 WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl); 5892 5893 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || 5894 cfg->pcie_type == PCIEM_TYPE_ROOT_EC) 5895 WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl); 5896 5897 if (version > 1) { 5898 WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2); 5899 WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2); 5900 WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2); 5901 } 5902 #undef WREG 5903 } 5904 5905 static void 5906 pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo) 5907 { 5908 pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 5909 dinfo->cfg.pcix.pcix_command, 2); 5910 } 5911 5912 void 5913 pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo) 5914 { 5915 5916 /* 5917 * Restore the device to full power mode. We must do this 5918 * before we restore the registers because moving from D3 to 5919 * D0 will cause the chip's BARs and some other registers to 5920 * be reset to some unknown power on reset values. Cut down 5921 * the noise on boot by doing nothing if we are already in 5922 * state D0. 5923 */ 5924 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) 5925 pci_set_powerstate(dev, PCI_POWERSTATE_D0); 5926 pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2); 5927 pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1); 5928 pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1); 5929 pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1); 5930 pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1); 5931 pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1); 5932 pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1); 5933 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) { 5934 case PCIM_HDRTYPE_NORMAL: 5935 pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1); 5936 pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1); 5937 break; 5938 case PCIM_HDRTYPE_BRIDGE: 5939 pci_write_config(dev, PCIR_SECLAT_1, 5940 dinfo->cfg.bridge.br_seclat, 1); 5941 pci_write_config(dev, PCIR_SUBBUS_1, 5942 dinfo->cfg.bridge.br_subbus, 1); 5943 pci_write_config(dev, PCIR_SECBUS_1, 5944 dinfo->cfg.bridge.br_secbus, 1); 5945 pci_write_config(dev, PCIR_PRIBUS_1, 5946 dinfo->cfg.bridge.br_pribus, 1); 5947 pci_write_config(dev, PCIR_BRIDGECTL_1, 5948 dinfo->cfg.bridge.br_control, 2); 5949 break; 5950 case PCIM_HDRTYPE_CARDBUS: 5951 pci_write_config(dev, PCIR_SECLAT_2, 5952 dinfo->cfg.bridge.br_seclat, 1); 5953 pci_write_config(dev, PCIR_SUBBUS_2, 5954 dinfo->cfg.bridge.br_subbus, 1); 5955 pci_write_config(dev, PCIR_SECBUS_2, 5956 dinfo->cfg.bridge.br_secbus, 1); 5957 pci_write_config(dev, PCIR_PRIBUS_2, 5958 dinfo->cfg.bridge.br_pribus, 1); 5959 pci_write_config(dev, PCIR_BRIDGECTL_2, 5960 dinfo->cfg.bridge.br_control, 2); 5961 break; 5962 } 5963 pci_restore_bars(dev); 5964 5965 /* 5966 * Restore extended capabilities for PCI-Express and PCI-X 5967 */ 5968 if (dinfo->cfg.pcie.pcie_location != 0) 5969 pci_cfg_restore_pcie(dev, dinfo); 5970 if (dinfo->cfg.pcix.pcix_location != 0) 5971 pci_cfg_restore_pcix(dev, dinfo); 5972 5973 /* Restore MSI and MSI-X configurations if they are present. */ 5974 if (dinfo->cfg.msi.msi_location != 0) 5975 pci_resume_msi(dev); 5976 if (dinfo->cfg.msix.msix_location != 0) 5977 pci_resume_msix(dev); 5978 5979 #ifdef PCI_IOV 5980 if (dinfo->cfg.iov != NULL) 5981 pci_iov_cfg_restore(dev, dinfo); 5982 #endif 5983 } 5984 5985 static void 5986 pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo) 5987 { 5988 #define RREG(n) pci_read_config(dev, pos + (n), 2) 5989 struct pcicfg_pcie *cfg; 5990 int version, pos; 5991 5992 cfg = &dinfo->cfg.pcie; 5993 pos = cfg->pcie_location; 5994 5995 cfg->pcie_flags = RREG(PCIER_FLAGS); 5996 5997 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION; 5998 5999 cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL); 6000 6001 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || 6002 cfg->pcie_type == PCIEM_TYPE_ENDPOINT || 6003 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT) 6004 cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL); 6005 6006 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || 6007 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT && 6008 (cfg->pcie_flags & PCIEM_FLAGS_SLOT)))) 6009 cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL); 6010 6011 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || 6012 cfg->pcie_type == PCIEM_TYPE_ROOT_EC) 6013 cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL); 6014 6015 if (version > 1) { 6016 cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2); 6017 cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2); 6018 cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2); 6019 } 6020 #undef RREG 6021 } 6022 6023 static void 6024 pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo) 6025 { 6026 dinfo->cfg.pcix.pcix_command = pci_read_config(dev, 6027 dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2); 6028 } 6029 6030 void 6031 pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate) 6032 { 6033 uint32_t cls; 6034 int ps; 6035 6036 /* 6037 * Some drivers apparently write to these registers w/o updating our 6038 * cached copy. No harm happens if we update the copy, so do so here 6039 * so we can restore them. The COMMAND register is modified by the 6040 * bus w/o updating the cache. This should represent the normally 6041 * writable portion of the 'defined' part of type 0/1/2 headers. 6042 */ 6043 dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2); 6044 dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2); 6045 dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2); 6046 dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1); 6047 dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1); 6048 dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1); 6049 dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1); 6050 dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1); 6051 dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1); 6052 dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1); 6053 dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1); 6054 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) { 6055 case PCIM_HDRTYPE_NORMAL: 6056 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2); 6057 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2); 6058 dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1); 6059 dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1); 6060 break; 6061 case PCIM_HDRTYPE_BRIDGE: 6062 dinfo->cfg.bridge.br_seclat = pci_read_config(dev, 6063 PCIR_SECLAT_1, 1); 6064 dinfo->cfg.bridge.br_subbus = pci_read_config(dev, 6065 PCIR_SUBBUS_1, 1); 6066 dinfo->cfg.bridge.br_secbus = pci_read_config(dev, 6067 PCIR_SECBUS_1, 1); 6068 dinfo->cfg.bridge.br_pribus = pci_read_config(dev, 6069 PCIR_PRIBUS_1, 1); 6070 dinfo->cfg.bridge.br_control = pci_read_config(dev, 6071 PCIR_BRIDGECTL_1, 2); 6072 break; 6073 case PCIM_HDRTYPE_CARDBUS: 6074 dinfo->cfg.bridge.br_seclat = pci_read_config(dev, 6075 PCIR_SECLAT_2, 1); 6076 dinfo->cfg.bridge.br_subbus = pci_read_config(dev, 6077 PCIR_SUBBUS_2, 1); 6078 dinfo->cfg.bridge.br_secbus = pci_read_config(dev, 6079 PCIR_SECBUS_2, 1); 6080 dinfo->cfg.bridge.br_pribus = pci_read_config(dev, 6081 PCIR_PRIBUS_2, 1); 6082 dinfo->cfg.bridge.br_control = pci_read_config(dev, 6083 PCIR_BRIDGECTL_2, 2); 6084 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2); 6085 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2); 6086 break; 6087 } 6088 6089 if (dinfo->cfg.pcie.pcie_location != 0) 6090 pci_cfg_save_pcie(dev, dinfo); 6091 6092 if (dinfo->cfg.pcix.pcix_location != 0) 6093 pci_cfg_save_pcix(dev, dinfo); 6094 6095 #ifdef PCI_IOV 6096 if (dinfo->cfg.iov != NULL) 6097 pci_iov_cfg_save(dev, dinfo); 6098 #endif 6099 6100 /* 6101 * don't set the state for display devices, base peripherals and 6102 * memory devices since bad things happen when they are powered down. 6103 * We should (a) have drivers that can easily detach and (b) use 6104 * generic drivers for these devices so that some device actually 6105 * attaches. We need to make sure that when we implement (a) we don't 6106 * power the device down on a reattach. 6107 */ 6108 cls = pci_get_class(dev); 6109 if (!setstate) 6110 return; 6111 switch (pci_do_power_nodriver) 6112 { 6113 case 0: /* NO powerdown at all */ 6114 return; 6115 case 1: /* Conservative about what to power down */ 6116 if (cls == PCIC_STORAGE) 6117 return; 6118 /*FALLTHROUGH*/ 6119 case 2: /* Aggressive about what to power down */ 6120 if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY || 6121 cls == PCIC_BASEPERIPH) 6122 return; 6123 /*FALLTHROUGH*/ 6124 case 3: /* Power down everything */ 6125 break; 6126 } 6127 /* 6128 * PCI spec says we can only go into D3 state from D0 state. 6129 * Transition from D[12] into D0 before going to D3 state. 6130 */ 6131 ps = pci_get_powerstate(dev); 6132 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3) 6133 pci_set_powerstate(dev, PCI_POWERSTATE_D0); 6134 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3) 6135 pci_set_powerstate(dev, PCI_POWERSTATE_D3); 6136 } 6137 6138 /* Wrapper APIs suitable for device driver use. */ 6139 void 6140 pci_save_state(device_t dev) 6141 { 6142 struct pci_devinfo *dinfo; 6143 6144 dinfo = device_get_ivars(dev); 6145 pci_cfg_save(dev, dinfo, 0); 6146 } 6147 6148 void 6149 pci_restore_state(device_t dev) 6150 { 6151 struct pci_devinfo *dinfo; 6152 6153 dinfo = device_get_ivars(dev); 6154 pci_cfg_restore(dev, dinfo); 6155 } 6156 6157 static int 6158 pci_get_id_method(device_t dev, device_t child, enum pci_id_type type, 6159 uintptr_t *id) 6160 { 6161 6162 return (PCIB_GET_ID(device_get_parent(dev), child, type, id)); 6163 } 6164 6165 /* Find the upstream port of a given PCI device in a root complex. */ 6166 device_t 6167 pci_find_pcie_root_port(device_t dev) 6168 { 6169 struct pci_devinfo *dinfo; 6170 devclass_t pci_class; 6171 device_t pcib, bus; 6172 6173 pci_class = devclass_find("pci"); 6174 KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class, 6175 ("%s: non-pci device %s", __func__, device_get_nameunit(dev))); 6176 6177 /* 6178 * Walk the bridge hierarchy until we find a PCI-e root 6179 * port or a non-PCI device. 6180 */ 6181 for (;;) { 6182 bus = device_get_parent(dev); 6183 KASSERT(bus != NULL, ("%s: null parent of %s", __func__, 6184 device_get_nameunit(dev))); 6185 6186 pcib = device_get_parent(bus); 6187 KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__, 6188 device_get_nameunit(bus))); 6189 6190 /* 6191 * pcib's parent must be a PCI bus for this to be a 6192 * PCI-PCI bridge. 6193 */ 6194 if (device_get_devclass(device_get_parent(pcib)) != pci_class) 6195 return (NULL); 6196 6197 dinfo = device_get_ivars(pcib); 6198 if (dinfo->cfg.pcie.pcie_location != 0 && 6199 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) 6200 return (pcib); 6201 6202 dev = pcib; 6203 } 6204 } 6205 6206 /* 6207 * Wait for pending transactions to complete on a PCI-express function. 6208 * 6209 * The maximum delay is specified in milliseconds in max_delay. Note 6210 * that this function may sleep. 6211 * 6212 * Returns true if the function is idle and false if the timeout is 6213 * exceeded. If dev is not a PCI-express function, this returns true. 6214 */ 6215 bool 6216 pcie_wait_for_pending_transactions(device_t dev, u_int max_delay) 6217 { 6218 struct pci_devinfo *dinfo = device_get_ivars(dev); 6219 uint16_t sta; 6220 int cap; 6221 6222 cap = dinfo->cfg.pcie.pcie_location; 6223 if (cap == 0) 6224 return (true); 6225 6226 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2); 6227 while (sta & PCIEM_STA_TRANSACTION_PND) { 6228 if (max_delay == 0) 6229 return (false); 6230 6231 /* Poll once every 100 milliseconds up to the timeout. */ 6232 if (max_delay > 100) { 6233 pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK); 6234 max_delay -= 100; 6235 } else { 6236 pause_sbt("pcietp", max_delay * SBT_1MS, 0, 6237 C_HARDCLOCK); 6238 max_delay = 0; 6239 } 6240 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2); 6241 } 6242 6243 return (true); 6244 } 6245 6246 /* 6247 * Determine the maximum Completion Timeout in microseconds. 6248 * 6249 * For non-PCI-express functions this returns 0. 6250 */ 6251 int 6252 pcie_get_max_completion_timeout(device_t dev) 6253 { 6254 struct pci_devinfo *dinfo = device_get_ivars(dev); 6255 int cap; 6256 6257 cap = dinfo->cfg.pcie.pcie_location; 6258 if (cap == 0) 6259 return (0); 6260 6261 /* 6262 * Functions using the 1.x spec use the default timeout range of 6263 * 50 microseconds to 50 milliseconds. Functions that do not 6264 * support programmable timeouts also use this range. 6265 */ 6266 if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 || 6267 (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) & 6268 PCIEM_CAP2_COMP_TIMO_RANGES) == 0) 6269 return (50 * 1000); 6270 6271 switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) & 6272 PCIEM_CTL2_COMP_TIMO_VAL) { 6273 case PCIEM_CTL2_COMP_TIMO_100US: 6274 return (100); 6275 case PCIEM_CTL2_COMP_TIMO_10MS: 6276 return (10 * 1000); 6277 case PCIEM_CTL2_COMP_TIMO_55MS: 6278 return (55 * 1000); 6279 case PCIEM_CTL2_COMP_TIMO_210MS: 6280 return (210 * 1000); 6281 case PCIEM_CTL2_COMP_TIMO_900MS: 6282 return (900 * 1000); 6283 case PCIEM_CTL2_COMP_TIMO_3500MS: 6284 return (3500 * 1000); 6285 case PCIEM_CTL2_COMP_TIMO_13S: 6286 return (13 * 1000 * 1000); 6287 case PCIEM_CTL2_COMP_TIMO_64S: 6288 return (64 * 1000 * 1000); 6289 default: 6290 return (50 * 1000); 6291 } 6292 } 6293 6294 /* 6295 * Perform a Function Level Reset (FLR) on a device. 6296 * 6297 * This function first waits for any pending transactions to complete 6298 * within the timeout specified by max_delay. If transactions are 6299 * still pending, the function will return false without attempting a 6300 * reset. 6301 * 6302 * If dev is not a PCI-express function or does not support FLR, this 6303 * function returns false. 6304 * 6305 * Note that no registers are saved or restored. The caller is 6306 * responsible for saving and restoring any registers including 6307 * PCI-standard registers via pci_save_state() and 6308 * pci_restore_state(). 6309 */ 6310 bool 6311 pcie_flr(device_t dev, u_int max_delay, bool force) 6312 { 6313 struct pci_devinfo *dinfo = device_get_ivars(dev); 6314 uint16_t cmd, ctl; 6315 int compl_delay; 6316 int cap; 6317 6318 cap = dinfo->cfg.pcie.pcie_location; 6319 if (cap == 0) 6320 return (false); 6321 6322 if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR)) 6323 return (false); 6324 6325 /* 6326 * Disable busmastering to prevent generation of new 6327 * transactions while waiting for the device to go idle. If 6328 * the idle timeout fails, the command register is restored 6329 * which will re-enable busmastering. 6330 */ 6331 cmd = pci_read_config(dev, PCIR_COMMAND, 2); 6332 pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2); 6333 if (!pcie_wait_for_pending_transactions(dev, max_delay)) { 6334 if (!force) { 6335 pci_write_config(dev, PCIR_COMMAND, cmd, 2); 6336 return (false); 6337 } 6338 pci_printf(&dinfo->cfg, 6339 "Resetting with transactions pending after %d ms\n", 6340 max_delay); 6341 6342 /* 6343 * Extend the post-FLR delay to cover the maximum 6344 * Completion Timeout delay of anything in flight 6345 * during the FLR delay. Enforce a minimum delay of 6346 * at least 10ms. 6347 */ 6348 compl_delay = pcie_get_max_completion_timeout(dev) / 1000; 6349 if (compl_delay < 10) 6350 compl_delay = 10; 6351 } else 6352 compl_delay = 0; 6353 6354 /* Initiate the reset. */ 6355 ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); 6356 pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl | 6357 PCIEM_CTL_INITIATE_FLR, 2); 6358 6359 /* Wait for 100ms. */ 6360 pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK); 6361 6362 if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) & 6363 PCIEM_STA_TRANSACTION_PND) 6364 pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n"); 6365 return (true); 6366 } 6367 6368 const struct pci_device_table * 6369 pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt) 6370 { 6371 bool match; 6372 uint16_t vendor, device, subvendor, subdevice, class, subclass, revid; 6373 6374 vendor = pci_get_vendor(child); 6375 device = pci_get_device(child); 6376 subvendor = pci_get_subvendor(child); 6377 subdevice = pci_get_subdevice(child); 6378 class = pci_get_class(child); 6379 subclass = pci_get_subclass(child); 6380 revid = pci_get_revid(child); 6381 while (nelt-- > 0) { 6382 match = true; 6383 if (id->match_flag_vendor) 6384 match &= vendor == id->vendor; 6385 if (id->match_flag_device) 6386 match &= device == id->device; 6387 if (id->match_flag_subvendor) 6388 match &= subvendor == id->subvendor; 6389 if (id->match_flag_subdevice) 6390 match &= subdevice == id->subdevice; 6391 if (id->match_flag_class) 6392 match &= class == id->class_id; 6393 if (id->match_flag_subclass) 6394 match &= subclass == id->subclass; 6395 if (id->match_flag_revid) 6396 match &= revid == id->revid; 6397 if (match) 6398 return (id); 6399 id++; 6400 } 6401 return (NULL); 6402 } 6403 6404 static void 6405 pci_print_faulted_dev_name(const struct pci_devinfo *dinfo) 6406 { 6407 const char *dev_name; 6408 device_t dev; 6409 6410 dev = dinfo->cfg.dev; 6411 printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus, 6412 dinfo->cfg.slot, dinfo->cfg.func); 6413 dev_name = device_get_name(dev); 6414 if (dev_name != NULL) 6415 printf(" (%s%d)", dev_name, device_get_unit(dev)); 6416 } 6417 6418 void 6419 pci_print_faulted_dev(void) 6420 { 6421 struct pci_devinfo *dinfo; 6422 device_t dev; 6423 int aer, i; 6424 uint32_t r1, r2; 6425 uint16_t status; 6426 6427 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { 6428 dev = dinfo->cfg.dev; 6429 status = pci_read_config(dev, PCIR_STATUS, 2); 6430 status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT | 6431 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT | 6432 PCIM_STATUS_SERR | PCIM_STATUS_PERR; 6433 if (status != 0) { 6434 pci_print_faulted_dev_name(dinfo); 6435 printf(" error 0x%04x\n", status); 6436 } 6437 if (dinfo->cfg.pcie.pcie_location != 0) { 6438 status = pci_read_config(dev, 6439 dinfo->cfg.pcie.pcie_location + 6440 PCIER_DEVICE_STA, 2); 6441 if ((status & (PCIEM_STA_CORRECTABLE_ERROR | 6442 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR | 6443 PCIEM_STA_UNSUPPORTED_REQ)) != 0) { 6444 pci_print_faulted_dev_name(dinfo); 6445 printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n", 6446 pci_read_config(dev, 6447 dinfo->cfg.pcie.pcie_location + 6448 PCIER_DEVICE_CTL, 2), 6449 status); 6450 } 6451 } 6452 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) { 6453 r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); 6454 r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); 6455 if (r1 != 0 || r2 != 0) { 6456 pci_print_faulted_dev_name(dinfo); 6457 printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n" 6458 " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n", 6459 r1, pci_read_config(dev, aer + 6460 PCIR_AER_UC_MASK, 4), 6461 pci_read_config(dev, aer + 6462 PCIR_AER_UC_SEVERITY, 4), 6463 r2, pci_read_config(dev, aer + 6464 PCIR_AER_COR_MASK, 4), 6465 pci_read_config(dev, aer + 6466 PCIR_AER_CAP_CONTROL, 4)); 6467 for (i = 0; i < 4; i++) { 6468 r1 = pci_read_config(dev, aer + 6469 PCIR_AER_HEADER_LOG + i * 4, 4); 6470 printf(" HL%d: 0x%08x\n", i, r1); 6471 } 6472 } 6473 } 6474 } 6475 } 6476 6477 #ifdef DDB 6478 DB_SHOW_COMMAND(pcierr, pci_print_faulted_dev_db) 6479 { 6480 6481 pci_print_faulted_dev(); 6482 } 6483 6484 static void 6485 db_clear_pcie_errors(const struct pci_devinfo *dinfo) 6486 { 6487 device_t dev; 6488 int aer; 6489 uint32_t r; 6490 6491 dev = dinfo->cfg.dev; 6492 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + 6493 PCIER_DEVICE_STA, 2); 6494 pci_write_config(dev, dinfo->cfg.pcie.pcie_location + 6495 PCIER_DEVICE_STA, r, 2); 6496 6497 if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0) 6498 return; 6499 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); 6500 if (r != 0) 6501 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4); 6502 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); 6503 if (r != 0) 6504 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4); 6505 } 6506 6507 DB_COMMAND(pci_clearerr, db_pci_clearerr) 6508 { 6509 struct pci_devinfo *dinfo; 6510 device_t dev; 6511 uint16_t status, status1; 6512 6513 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { 6514 dev = dinfo->cfg.dev; 6515 status1 = status = pci_read_config(dev, PCIR_STATUS, 2); 6516 status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT | 6517 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT | 6518 PCIM_STATUS_SERR | PCIM_STATUS_PERR; 6519 if (status1 != 0) { 6520 status &= ~status1; 6521 pci_write_config(dev, PCIR_STATUS, status, 2); 6522 } 6523 if (dinfo->cfg.pcie.pcie_location != 0) 6524 db_clear_pcie_errors(dinfo); 6525 } 6526 } 6527 #endif 6528