1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org>
5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org>
6 * Copyright (c) 2000, BSDi
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice unmodified, this list of conditions, and the following
14 * disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include "opt_acpi.h"
33 #include "opt_iommu.h"
34 #include "opt_bus.h"
35
36 #include <sys/param.h>
37 #include <sys/conf.h>
38 #include <sys/endian.h>
39 #include <sys/eventhandler.h>
40 #include <sys/fcntl.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/linker.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/queue.h>
47 #include <sys/sbuf.h>
48 #include <sys/sysctl.h>
49 #include <sys/systm.h>
50 #include <sys/taskqueue.h>
51 #include <sys/tree.h>
52
53 #include <vm/vm.h>
54 #include <vm/pmap.h>
55 #include <vm/vm_extern.h>
56
57 #include <sys/bus.h>
58 #include <machine/bus.h>
59 #include <sys/rman.h>
60 #include <machine/resource.h>
61 #include <machine/stdarg.h>
62
63 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
64 #include <machine/intr_machdep.h>
65 #endif
66
67 #include <sys/pciio.h>
68 #include <dev/pci/pcireg.h>
69 #include <dev/pci/pcivar.h>
70 #include <dev/pci/pci_private.h>
71
72 #ifdef PCI_IOV
73 #include <sys/nv.h>
74 #include <dev/pci/pci_iov_private.h>
75 #endif
76
77 #include <dev/usb/controller/xhcireg.h>
78 #include <dev/usb/controller/ehcireg.h>
79 #include <dev/usb/controller/ohcireg.h>
80 #include <dev/usb/controller/uhcireg.h>
81
82 #include <dev/iommu/iommu.h>
83
84 #include "pcib_if.h"
85 #include "pci_if.h"
86
87 #define PCIR_IS_BIOS(cfg, reg) \
88 (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \
89 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))
90
91 static device_probe_t pci_probe;
92
93 static bus_reset_post_t pci_reset_post;
94 static bus_reset_prepare_t pci_reset_prepare;
95 static bus_reset_child_t pci_reset_child;
96 static bus_hint_device_unit_t pci_hint_device_unit;
97 static bus_remap_intr_t pci_remap_intr_method;
98
99 static pci_get_id_t pci_get_id_method;
100
101 static int pci_has_quirk(uint32_t devid, int quirk);
102 static pci_addr_t pci_mapbase(uint64_t mapreg);
103 static const char *pci_maptype(uint64_t mapreg);
104 static int pci_maprange(uint64_t mapreg);
105 static pci_addr_t pci_rombase(uint64_t mapreg);
106 static int pci_romsize(uint64_t testval);
107 static void pci_fixancient(pcicfgregs *cfg);
108 static int pci_printf(pcicfgregs *cfg, const char *fmt, ...);
109
110 static int pci_porten(device_t dev);
111 static int pci_memen(device_t dev);
112 static void pci_assign_interrupt(device_t bus, device_t dev,
113 int force_route);
114 static int pci_add_map(device_t bus, device_t dev, int reg,
115 struct resource_list *rl, int force, int prefetch);
116 static void pci_load_vendor_data(void);
117 static int pci_describe_parse_line(char **ptr, int *vendor,
118 int *device, char **desc);
119 static char *pci_describe_device(device_t dev);
120 static int pci_modevent(module_t mod, int what, void *arg);
121 static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
122 pcicfgregs *cfg);
123 static void pci_read_cap(device_t pcib, pcicfgregs *cfg);
124 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg,
125 int reg, uint32_t *data);
126 #if 0
127 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg,
128 int reg, uint32_t data);
129 #endif
130 static void pci_read_vpd(device_t pcib, pcicfgregs *cfg);
131 static void pci_mask_msix(device_t dev, u_int index);
132 static void pci_unmask_msix(device_t dev, u_int index);
133 static int pci_msi_blacklisted(void);
134 static int pci_msix_blacklisted(void);
135 static void pci_resume_msi(device_t dev);
136 static void pci_resume_msix(device_t dev);
137 static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d,
138 int b, int s, int f, uint16_t vid, uint16_t did);
139
140 static device_method_t pci_methods[] = {
141 /* Device interface */
142 DEVMETHOD(device_probe, pci_probe),
143 DEVMETHOD(device_attach, pci_attach),
144 DEVMETHOD(device_detach, pci_detach),
145 DEVMETHOD(device_shutdown, bus_generic_shutdown),
146 DEVMETHOD(device_suspend, bus_generic_suspend),
147 DEVMETHOD(device_resume, pci_resume),
148
149 /* Bus interface */
150 DEVMETHOD(bus_print_child, pci_print_child),
151 DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch),
152 DEVMETHOD(bus_read_ivar, pci_read_ivar),
153 DEVMETHOD(bus_write_ivar, pci_write_ivar),
154 DEVMETHOD(bus_driver_added, pci_driver_added),
155 DEVMETHOD(bus_setup_intr, pci_setup_intr),
156 DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
157 DEVMETHOD(bus_reset_prepare, pci_reset_prepare),
158 DEVMETHOD(bus_reset_post, pci_reset_post),
159 DEVMETHOD(bus_reset_child, pci_reset_child),
160
161 DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag),
162 DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
163 DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
164 DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource),
165 DEVMETHOD(bus_delete_resource, pci_delete_resource),
166 DEVMETHOD(bus_alloc_resource, pci_alloc_resource),
167 #ifdef PCI_IOV
168 DEVMETHOD(bus_adjust_resource, pci_adjust_resource),
169 #else
170 DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource),
171 #endif
172 DEVMETHOD(bus_release_resource, pci_release_resource),
173 DEVMETHOD(bus_activate_resource, pci_activate_resource),
174 DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource),
175 #ifdef PCI_IOV
176 DEVMETHOD(bus_map_resource, pci_map_resource),
177 DEVMETHOD(bus_unmap_resource, pci_unmap_resource),
178 #endif
179 DEVMETHOD(bus_child_deleted, pci_child_deleted),
180 DEVMETHOD(bus_child_detached, pci_child_detached),
181 DEVMETHOD(bus_child_pnpinfo, pci_child_pnpinfo_method),
182 DEVMETHOD(bus_child_location, pci_child_location_method),
183 DEVMETHOD(bus_get_device_path, pci_get_device_path_method),
184 DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit),
185 DEVMETHOD(bus_remap_intr, pci_remap_intr_method),
186 DEVMETHOD(bus_suspend_child, pci_suspend_child),
187 DEVMETHOD(bus_resume_child, pci_resume_child),
188 DEVMETHOD(bus_rescan, pci_rescan_method),
189
190 /* PCI interface */
191 DEVMETHOD(pci_read_config, pci_read_config_method),
192 DEVMETHOD(pci_write_config, pci_write_config_method),
193 DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method),
194 DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method),
195 DEVMETHOD(pci_enable_io, pci_enable_io_method),
196 DEVMETHOD(pci_disable_io, pci_disable_io_method),
197 DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method),
198 DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method),
199 DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method),
200 DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method),
201 DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method),
202 DEVMETHOD(pci_find_cap, pci_find_cap_method),
203 DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method),
204 DEVMETHOD(pci_find_extcap, pci_find_extcap_method),
205 DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method),
206 DEVMETHOD(pci_find_htcap, pci_find_htcap_method),
207 DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method),
208 DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method),
209 DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method),
210 DEVMETHOD(pci_enable_msi, pci_enable_msi_method),
211 DEVMETHOD(pci_enable_msix, pci_enable_msix_method),
212 DEVMETHOD(pci_disable_msi, pci_disable_msi_method),
213 DEVMETHOD(pci_remap_msix, pci_remap_msix_method),
214 DEVMETHOD(pci_release_msi, pci_release_msi_method),
215 DEVMETHOD(pci_msi_count, pci_msi_count_method),
216 DEVMETHOD(pci_msix_count, pci_msix_count_method),
217 DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method),
218 DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method),
219 DEVMETHOD(pci_get_id, pci_get_id_method),
220 DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method),
221 DEVMETHOD(pci_child_added, pci_child_added_method),
222 #ifdef PCI_IOV
223 DEVMETHOD(pci_iov_attach, pci_iov_attach_method),
224 DEVMETHOD(pci_iov_detach, pci_iov_detach_method),
225 DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method),
226 #endif
227
228 DEVMETHOD_END
229 };
230
231 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));
232
233 EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_modevent, NULL, BUS_PASS_BUS);
234 MODULE_VERSION(pci, 1);
235
236 static char *pci_vendordata;
237 static size_t pci_vendordata_size;
238
239 struct pci_quirk {
240 uint32_t devid; /* Vendor/device of the card */
241 int type;
242 #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */
243 #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */
244 #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */
245 #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */
246 #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */
247 #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */
248 #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */
249 int arg1;
250 int arg2;
251 };
252
253 static const struct pci_quirk pci_quirks[] = {
254 /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
255 { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 },
256 { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 },
257 /* As does the Serverworks OSB4 (the SMBus mapping register) */
258 { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 },
259
260 /*
261 * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge
262 * or the CMIC-SL (AKA ServerWorks GC_LE).
263 */
264 { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
265 { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 },
266
267 /*
268 * MSI doesn't work on earlier Intel chipsets including
269 * E7500, E7501, E7505, 845, 865, 875/E7210, and 855.
270 */
271 { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
272 { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
273 { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
274 { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
275 { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
276 { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
277 { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 },
278
279 /*
280 * MSI doesn't work with devices behind the AMD 8131 HT-PCIX
281 * bridge.
282 */
283 { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 },
284
285 /*
286 * Some virtualization environments emulate an older chipset
287 * but support MSI just fine. QEMU uses the Intel 82440.
288 */
289 { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 },
290
291 /*
292 * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus
293 * controller depending on SoftPciRst register (PM_IO 0x55 [7]).
294 * It prevents us from attaching hpet(4) when the bit is unset.
295 * Note this quirk only affects SB600 revision A13 and earlier.
296 * For SB600 A21 and later, firmware must set the bit to hide it.
297 * For SB700 and later, it is unused and hardcoded to zero.
298 */
299 { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 },
300
301 /*
302 * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have
303 * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit
304 * of the command register is set.
305 */
306 { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
307 { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
308 { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
309 { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
310 { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 },
311
312 /*
313 * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't
314 * issue MSI interrupts with PCIM_CMD_INTxDIS set either.
315 */
316 { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */
317 { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */
318 { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */
319 { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */
320 { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */
321 { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */
322
323 /*
324 * HPE Gen 10 VGA has a memory range that can't be allocated in the
325 * expected place.
326 */
327 { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 },
328 { 0 }
329 };
330
331 /* map register information */
332 #define PCI_MAPMEM 0x01 /* memory map */
333 #define PCI_MAPMEMP 0x02 /* prefetchable memory map */
334 #define PCI_MAPPORT 0x04 /* port map */
335
336 struct devlist pci_devq;
337 uint32_t pci_generation;
338 uint32_t pci_numdevs = 0;
339 static int pcie_chipset, pcix_chipset;
340
341 /* sysctl vars */
342 SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
343 "PCI bus tuning parameters");
344
345 static int pci_enable_io_modes = 1;
346 SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN,
347 &pci_enable_io_modes, 1,
348 "Enable I/O and memory bits in the config register. Some BIOSes do not"
349 " enable these bits correctly. We'd like to do this all the time, but"
350 " there are some peripherals that this causes problems with.");
351
352 static int pci_do_realloc_bars = 1;
353 SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN,
354 &pci_do_realloc_bars, 0,
355 "Attempt to allocate a new range for any BARs whose original "
356 "firmware-assigned ranges fail to allocate during the initial device scan.");
357
358 static int pci_do_power_nodriver = 0;
359 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN,
360 &pci_do_power_nodriver, 0,
361 "Place a function into D3 state when no driver attaches to it. 0 means"
362 " disable. 1 means conservatively place devices into D3 state. 2 means"
363 " aggressively place devices into D3 state. 3 means put absolutely"
364 " everything in D3 state.");
365
366 int pci_do_power_resume = 1;
367 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN,
368 &pci_do_power_resume, 1,
369 "Transition from D3 -> D0 on resume.");
370
371 int pci_do_power_suspend = 1;
372 SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN,
373 &pci_do_power_suspend, 1,
374 "Transition from D0 -> D3 on suspend.");
375
376 static int pci_do_msi = 1;
377 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1,
378 "Enable support for MSI interrupts");
379
380 static int pci_do_msix = 1;
381 SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1,
382 "Enable support for MSI-X interrupts");
383
384 static int pci_msix_rewrite_table = 0;
385 SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
386 &pci_msix_rewrite_table, 0,
387 "Rewrite entire MSI-X table when updating MSI-X entries");
388
389 static int pci_honor_msi_blacklist = 1;
390 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN,
391 &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");
392
393 #if defined(__i386__) || defined(__amd64__)
394 static int pci_usb_takeover = 1;
395 #else
396 static int pci_usb_takeover = 0;
397 #endif
398 SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN,
399 &pci_usb_takeover, 1,
400 "Enable early takeover of USB controllers. Disable this if you depend on"
401 " BIOS emulation of USB devices, that is you use USB devices (like"
402 " keyboard or mouse) but do not load USB drivers");
403
404 static int pci_clear_bars;
405 SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0,
406 "Ignore firmware-assigned resources for BARs.");
407
408 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
409 static int pci_clear_buses;
410 SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0,
411 "Ignore firmware-assigned bus numbers.");
412 #endif
413
414 static int pci_enable_ari = 1;
415 SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari,
416 0, "Enable support for PCIe Alternative RID Interpretation");
417
418 int pci_enable_aspm = 1;
419 SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm,
420 0, "Enable support for PCIe Active State Power Management");
421
422 static int pci_clear_aer_on_attach = 0;
423 SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN,
424 &pci_clear_aer_on_attach, 0,
425 "Clear port and device AER state on driver attach");
426
427 static bool pci_enable_mps_tune = true;
428 SYSCTL_BOOL(_hw_pci, OID_AUTO, enable_mps_tune, CTLFLAG_RWTUN,
429 &pci_enable_mps_tune, 1,
430 "Enable tuning of MPS(maximum payload size)." );
431
432 static int
pci_has_quirk(uint32_t devid,int quirk)433 pci_has_quirk(uint32_t devid, int quirk)
434 {
435 const struct pci_quirk *q;
436
437 for (q = &pci_quirks[0]; q->devid; q++) {
438 if (q->devid == devid && q->type == quirk)
439 return (1);
440 }
441 return (0);
442 }
443
444 /* Find a device_t by bus/slot/function in domain 0 */
445
446 device_t
pci_find_bsf(uint8_t bus,uint8_t slot,uint8_t func)447 pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func)
448 {
449
450 return (pci_find_dbsf(0, bus, slot, func));
451 }
452
453 /* Find a device_t by domain/bus/slot/function */
454
455 device_t
pci_find_dbsf(uint32_t domain,uint8_t bus,uint8_t slot,uint8_t func)456 pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func)
457 {
458 struct pci_devinfo *dinfo = NULL;
459
460 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
461 if ((dinfo->cfg.domain == domain) &&
462 (dinfo->cfg.bus == bus) &&
463 (dinfo->cfg.slot == slot) &&
464 (dinfo->cfg.func == func)) {
465 break;
466 }
467 }
468
469 return (dinfo != NULL ? dinfo->cfg.dev : NULL);
470 }
471
472 /* Find a device_t by vendor/device ID */
473
474 device_t
pci_find_device(uint16_t vendor,uint16_t device)475 pci_find_device(uint16_t vendor, uint16_t device)
476 {
477 struct pci_devinfo *dinfo;
478
479 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
480 if ((dinfo->cfg.vendor == vendor) &&
481 (dinfo->cfg.device == device)) {
482 return (dinfo->cfg.dev);
483 }
484 }
485
486 return (NULL);
487 }
488
489 device_t
pci_find_class(uint8_t class,uint8_t subclass)490 pci_find_class(uint8_t class, uint8_t subclass)
491 {
492 struct pci_devinfo *dinfo;
493
494 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
495 if (dinfo->cfg.baseclass == class &&
496 dinfo->cfg.subclass == subclass) {
497 return (dinfo->cfg.dev);
498 }
499 }
500
501 return (NULL);
502 }
503
504 device_t
pci_find_class_from(uint8_t class,uint8_t subclass,device_t from)505 pci_find_class_from(uint8_t class, uint8_t subclass, device_t from)
506 {
507 struct pci_devinfo *dinfo;
508 bool found = false;
509
510 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
511 if (from != NULL && found == false) {
512 if (from != dinfo->cfg.dev)
513 continue;
514 found = true;
515 continue;
516 }
517 if (dinfo->cfg.baseclass == class &&
518 dinfo->cfg.subclass == subclass) {
519 return (dinfo->cfg.dev);
520 }
521 }
522
523 return (NULL);
524 }
525
526 static int
pci_printf(pcicfgregs * cfg,const char * fmt,...)527 pci_printf(pcicfgregs *cfg, const char *fmt, ...)
528 {
529 va_list ap;
530 int retval;
531
532 retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot,
533 cfg->func);
534 va_start(ap, fmt);
535 retval += vprintf(fmt, ap);
536 va_end(ap);
537 return (retval);
538 }
539
540 /* return base address of memory or port map */
541
542 static pci_addr_t
pci_mapbase(uint64_t mapreg)543 pci_mapbase(uint64_t mapreg)
544 {
545
546 if (PCI_BAR_MEM(mapreg))
547 return (mapreg & PCIM_BAR_MEM_BASE);
548 else
549 return (mapreg & PCIM_BAR_IO_BASE);
550 }
551
552 /* return map type of memory or port map */
553
554 static const char *
pci_maptype(uint64_t mapreg)555 pci_maptype(uint64_t mapreg)
556 {
557
558 if (PCI_BAR_IO(mapreg))
559 return ("I/O Port");
560 if (mapreg & PCIM_BAR_MEM_PREFETCH)
561 return ("Prefetchable Memory");
562 return ("Memory");
563 }
564
565 /* return log2 of map size decoded for memory or port map */
566
567 int
pci_mapsize(uint64_t testval)568 pci_mapsize(uint64_t testval)
569 {
570 int ln2size;
571
572 testval = pci_mapbase(testval);
573 ln2size = 0;
574 if (testval != 0) {
575 while ((testval & 1) == 0)
576 {
577 ln2size++;
578 testval >>= 1;
579 }
580 }
581 return (ln2size);
582 }
583
584 /* return base address of device ROM */
585
586 static pci_addr_t
pci_rombase(uint64_t mapreg)587 pci_rombase(uint64_t mapreg)
588 {
589
590 return (mapreg & PCIM_BIOS_ADDR_MASK);
591 }
592
593 /* return log2 of map size decided for device ROM */
594
595 static int
pci_romsize(uint64_t testval)596 pci_romsize(uint64_t testval)
597 {
598 int ln2size;
599
600 testval = pci_rombase(testval);
601 ln2size = 0;
602 if (testval != 0) {
603 while ((testval & 1) == 0)
604 {
605 ln2size++;
606 testval >>= 1;
607 }
608 }
609 return (ln2size);
610 }
611
612 /* return log2 of address range supported by map register */
613
614 static int
pci_maprange(uint64_t mapreg)615 pci_maprange(uint64_t mapreg)
616 {
617 int ln2range = 0;
618
619 if (PCI_BAR_IO(mapreg))
620 ln2range = 32;
621 else
622 switch (mapreg & PCIM_BAR_MEM_TYPE) {
623 case PCIM_BAR_MEM_32:
624 ln2range = 32;
625 break;
626 case PCIM_BAR_MEM_1MB:
627 ln2range = 20;
628 break;
629 case PCIM_BAR_MEM_64:
630 ln2range = 64;
631 break;
632 }
633 return (ln2range);
634 }
635
636 /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */
637
638 static void
pci_fixancient(pcicfgregs * cfg)639 pci_fixancient(pcicfgregs *cfg)
640 {
641 if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
642 return;
643
644 /* PCI to PCI bridges use header type 1 */
645 if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI)
646 cfg->hdrtype = PCIM_HDRTYPE_BRIDGE;
647 }
648
649 /* extract header type specific config data */
650
651 static void
pci_hdrtypedata(device_t pcib,int b,int s,int f,pcicfgregs * cfg)652 pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg)
653 {
654 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
655 switch (cfg->hdrtype & PCIM_HDRTYPE) {
656 case PCIM_HDRTYPE_NORMAL:
657 cfg->subvendor = REG(PCIR_SUBVEND_0, 2);
658 cfg->subdevice = REG(PCIR_SUBDEV_0, 2);
659 cfg->mingnt = REG(PCIR_MINGNT, 1);
660 cfg->maxlat = REG(PCIR_MAXLAT, 1);
661 cfg->nummaps = PCI_MAXMAPS_0;
662 break;
663 case PCIM_HDRTYPE_BRIDGE:
664 cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1);
665 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1);
666 cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1);
667 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1);
668 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2);
669 cfg->nummaps = PCI_MAXMAPS_1;
670 break;
671 case PCIM_HDRTYPE_CARDBUS:
672 cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1);
673 cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1);
674 cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1);
675 cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1);
676 cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2);
677 cfg->subvendor = REG(PCIR_SUBVEND_2, 2);
678 cfg->subdevice = REG(PCIR_SUBDEV_2, 2);
679 cfg->nummaps = PCI_MAXMAPS_2;
680 break;
681 }
682 #undef REG
683 }
684
685 /* read configuration header into pcicfgregs structure */
686 struct pci_devinfo *
pci_read_device(device_t pcib,device_t bus,int d,int b,int s,int f)687 pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f)
688 {
689 #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w)
690 uint16_t vid, did;
691
692 vid = REG(PCIR_VENDOR, 2);
693 if (vid == PCIV_INVALID)
694 return (NULL);
695
696 did = REG(PCIR_DEVICE, 2);
697
698 return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did));
699 }
700
701 struct pci_devinfo *
pci_alloc_devinfo_method(device_t dev)702 pci_alloc_devinfo_method(device_t dev)
703 {
704
705 return (malloc(sizeof(struct pci_devinfo), M_DEVBUF,
706 M_WAITOK | M_ZERO));
707 }
708
709 static struct pci_devinfo *
pci_fill_devinfo(device_t pcib,device_t bus,int d,int b,int s,int f,uint16_t vid,uint16_t did)710 pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f,
711 uint16_t vid, uint16_t did)
712 {
713 struct pci_devinfo *devlist_entry;
714 pcicfgregs *cfg;
715
716 devlist_entry = PCI_ALLOC_DEVINFO(bus);
717
718 cfg = &devlist_entry->cfg;
719
720 cfg->domain = d;
721 cfg->bus = b;
722 cfg->slot = s;
723 cfg->func = f;
724 cfg->vendor = vid;
725 cfg->device = did;
726 cfg->cmdreg = REG(PCIR_COMMAND, 2);
727 cfg->statreg = REG(PCIR_STATUS, 2);
728 cfg->baseclass = REG(PCIR_CLASS, 1);
729 cfg->subclass = REG(PCIR_SUBCLASS, 1);
730 cfg->progif = REG(PCIR_PROGIF, 1);
731 cfg->revid = REG(PCIR_REVID, 1);
732 cfg->hdrtype = REG(PCIR_HDRTYPE, 1);
733 cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1);
734 cfg->lattimer = REG(PCIR_LATTIMER, 1);
735 cfg->intpin = REG(PCIR_INTPIN, 1);
736 cfg->intline = REG(PCIR_INTLINE, 1);
737
738 cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0;
739 cfg->hdrtype &= ~PCIM_MFDEV;
740 STAILQ_INIT(&cfg->maps);
741
742 cfg->iov = NULL;
743
744 pci_fixancient(cfg);
745 pci_hdrtypedata(pcib, b, s, f, cfg);
746
747 if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)
748 pci_read_cap(pcib, cfg);
749
750 STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links);
751
752 devlist_entry->conf.pc_sel.pc_domain = cfg->domain;
753 devlist_entry->conf.pc_sel.pc_bus = cfg->bus;
754 devlist_entry->conf.pc_sel.pc_dev = cfg->slot;
755 devlist_entry->conf.pc_sel.pc_func = cfg->func;
756 devlist_entry->conf.pc_hdr = cfg->hdrtype;
757
758 devlist_entry->conf.pc_subvendor = cfg->subvendor;
759 devlist_entry->conf.pc_subdevice = cfg->subdevice;
760 devlist_entry->conf.pc_vendor = cfg->vendor;
761 devlist_entry->conf.pc_device = cfg->device;
762
763 devlist_entry->conf.pc_class = cfg->baseclass;
764 devlist_entry->conf.pc_subclass = cfg->subclass;
765 devlist_entry->conf.pc_progif = cfg->progif;
766 devlist_entry->conf.pc_revid = cfg->revid;
767
768 pci_numdevs++;
769 pci_generation++;
770
771 return (devlist_entry);
772 }
773 #undef REG
774
775 static void
pci_ea_fill_info(device_t pcib,pcicfgregs * cfg)776 pci_ea_fill_info(device_t pcib, pcicfgregs *cfg)
777 {
778 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \
779 cfg->ea.ea_location + (n), w)
780 int num_ent;
781 int ptr;
782 int a, b;
783 uint32_t val;
784 int ent_size;
785 uint32_t dw[4];
786 uint64_t base, max_offset;
787 struct pci_ea_entry *eae;
788
789 if (cfg->ea.ea_location == 0)
790 return;
791
792 STAILQ_INIT(&cfg->ea.ea_entries);
793
794 /* Determine the number of entries */
795 num_ent = REG(PCIR_EA_NUM_ENT, 2);
796 num_ent &= PCIM_EA_NUM_ENT_MASK;
797
798 /* Find the first entry to care of */
799 ptr = PCIR_EA_FIRST_ENT;
800
801 /* Skip DWORD 2 for type 1 functions */
802 if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE)
803 ptr += 4;
804
805 for (a = 0; a < num_ent; a++) {
806 eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO);
807 eae->eae_cfg_offset = cfg->ea.ea_location + ptr;
808
809 /* Read a number of dwords in the entry */
810 val = REG(ptr, 4);
811 ptr += 4;
812 ent_size = (val & PCIM_EA_ES);
813
814 for (b = 0; b < ent_size; b++) {
815 dw[b] = REG(ptr, 4);
816 ptr += 4;
817 }
818
819 eae->eae_flags = val;
820 eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET;
821
822 base = dw[0] & PCIM_EA_FIELD_MASK;
823 max_offset = dw[1] | ~PCIM_EA_FIELD_MASK;
824 b = 2;
825 if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) {
826 base |= (uint64_t)dw[b] << 32UL;
827 b++;
828 }
829 if (((dw[1] & PCIM_EA_IS_64) != 0)
830 && (b < ent_size)) {
831 max_offset |= (uint64_t)dw[b] << 32UL;
832 b++;
833 }
834
835 eae->eae_base = base;
836 eae->eae_max_offset = max_offset;
837
838 STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link);
839
840 if (bootverbose) {
841 printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n",
842 cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags,
843 (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset);
844 }
845 }
846 }
847 #undef REG
848
849 static void
pci_read_cap(device_t pcib,pcicfgregs * cfg)850 pci_read_cap(device_t pcib, pcicfgregs *cfg)
851 {
852 #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w)
853 #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w)
854 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
855 uint64_t addr;
856 #endif
857 uint32_t val;
858 int ptr, nextptr, ptrptr;
859
860 switch (cfg->hdrtype & PCIM_HDRTYPE) {
861 case PCIM_HDRTYPE_NORMAL:
862 case PCIM_HDRTYPE_BRIDGE:
863 ptrptr = PCIR_CAP_PTR;
864 break;
865 case PCIM_HDRTYPE_CARDBUS:
866 ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */
867 break;
868 default:
869 return; /* no extended capabilities support */
870 }
871 nextptr = REG(ptrptr, 1); /* sanity check? */
872
873 /*
874 * Read capability entries.
875 */
876 while (nextptr != 0) {
877 /* Sanity check */
878 if (nextptr > 255) {
879 printf("illegal PCI extended capability offset %d\n",
880 nextptr);
881 return;
882 }
883 /* Find the next entry */
884 ptr = nextptr;
885 nextptr = REG(ptr + PCICAP_NEXTPTR, 1);
886
887 /* Process this entry */
888 switch (REG(ptr + PCICAP_ID, 1)) {
889 case PCIY_PMG: /* PCI power management */
890 if (cfg->pp.pp_cap == 0) {
891 cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2);
892 cfg->pp.pp_status = ptr + PCIR_POWER_STATUS;
893 cfg->pp.pp_bse = ptr + PCIR_POWER_BSE;
894 if ((nextptr - ptr) > PCIR_POWER_DATA)
895 cfg->pp.pp_data = ptr + PCIR_POWER_DATA;
896 }
897 break;
898 case PCIY_HT: /* HyperTransport */
899 /* Determine HT-specific capability type. */
900 val = REG(ptr + PCIR_HT_COMMAND, 2);
901
902 if ((val & 0xe000) == PCIM_HTCAP_SLAVE)
903 cfg->ht.ht_slave = ptr;
904
905 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__)
906 switch (val & PCIM_HTCMD_CAP_MASK) {
907 case PCIM_HTCAP_MSI_MAPPING:
908 if (!(val & PCIM_HTCMD_MSI_FIXED)) {
909 /* Sanity check the mapping window. */
910 addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI,
911 4);
912 addr <<= 32;
913 addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO,
914 4);
915 if (addr != MSI_INTEL_ADDR_BASE)
916 device_printf(pcib,
917 "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n",
918 cfg->domain, cfg->bus,
919 cfg->slot, cfg->func,
920 (long long)addr);
921 } else
922 addr = MSI_INTEL_ADDR_BASE;
923
924 cfg->ht.ht_msimap = ptr;
925 cfg->ht.ht_msictrl = val;
926 cfg->ht.ht_msiaddr = addr;
927 break;
928 }
929 #endif
930 break;
931 case PCIY_MSI: /* PCI MSI */
932 cfg->msi.msi_location = ptr;
933 cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2);
934 cfg->msi.msi_msgnum = 1 << ((cfg->msi.msi_ctrl &
935 PCIM_MSICTRL_MMC_MASK)>>1);
936 break;
937 case PCIY_MSIX: /* PCI MSI-X */
938 cfg->msix.msix_location = ptr;
939 cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2);
940 cfg->msix.msix_msgnum = (cfg->msix.msix_ctrl &
941 PCIM_MSIXCTRL_TABLE_SIZE) + 1;
942 val = REG(ptr + PCIR_MSIX_TABLE, 4);
943 cfg->msix.msix_table_bar = PCIR_BAR(val &
944 PCIM_MSIX_BIR_MASK);
945 cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK;
946 val = REG(ptr + PCIR_MSIX_PBA, 4);
947 cfg->msix.msix_pba_bar = PCIR_BAR(val &
948 PCIM_MSIX_BIR_MASK);
949 cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK;
950 break;
951 case PCIY_VPD: /* PCI Vital Product Data */
952 cfg->vpd.vpd_reg = ptr;
953 break;
954 case PCIY_SUBVENDOR:
955 /* Should always be true. */
956 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
957 PCIM_HDRTYPE_BRIDGE) {
958 val = REG(ptr + PCIR_SUBVENDCAP_ID, 4);
959 cfg->subvendor = val & 0xffff;
960 cfg->subdevice = val >> 16;
961 }
962 break;
963 case PCIY_PCIX: /* PCI-X */
964 /*
965 * Assume we have a PCI-X chipset if we have
966 * at least one PCI-PCI bridge with a PCI-X
967 * capability. Note that some systems with
968 * PCI-express or HT chipsets might match on
969 * this check as well.
970 */
971 if ((cfg->hdrtype & PCIM_HDRTYPE) ==
972 PCIM_HDRTYPE_BRIDGE)
973 pcix_chipset = 1;
974 cfg->pcix.pcix_location = ptr;
975 break;
976 case PCIY_EXPRESS: /* PCI-express */
977 /*
978 * Assume we have a PCI-express chipset if we have
979 * at least one PCI-express device.
980 */
981 pcie_chipset = 1;
982 cfg->pcie.pcie_location = ptr;
983 val = REG(ptr + PCIER_FLAGS, 2);
984 cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE;
985 break;
986 case PCIY_EA: /* Enhanced Allocation */
987 cfg->ea.ea_location = ptr;
988 pci_ea_fill_info(pcib, cfg);
989 break;
990 default:
991 break;
992 }
993 }
994
995 #if defined(__powerpc__)
996 /*
997 * Enable the MSI mapping window for all HyperTransport
998 * slaves. PCI-PCI bridges have their windows enabled via
999 * PCIB_MAP_MSI().
1000 */
1001 if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 &&
1002 !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) {
1003 device_printf(pcib,
1004 "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n",
1005 cfg->domain, cfg->bus, cfg->slot, cfg->func);
1006 cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
1007 WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl,
1008 2);
1009 }
1010 #endif
1011 /* REG and WREG use carry through to next functions */
1012 }
1013
1014 /*
1015 * PCI Vital Product Data
1016 */
1017
1018 #define PCI_VPD_TIMEOUT 1000000
1019
1020 static int
pci_read_vpd_reg(device_t pcib,pcicfgregs * cfg,int reg,uint32_t * data)1021 pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data)
1022 {
1023 int count = PCI_VPD_TIMEOUT;
1024
1025 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1026
1027 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2);
1028
1029 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) {
1030 if (--count < 0)
1031 return (ENXIO);
1032 DELAY(1); /* limit looping */
1033 }
1034 *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4));
1035
1036 return (0);
1037 }
1038
1039 #if 0
1040 static int
1041 pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data)
1042 {
1043 int count = PCI_VPD_TIMEOUT;
1044
1045 KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned"));
1046
1047 WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4);
1048 WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2);
1049 while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) {
1050 if (--count < 0)
1051 return (ENXIO);
1052 DELAY(1); /* limit looping */
1053 }
1054
1055 return (0);
1056 }
1057 #endif
1058
1059 #undef PCI_VPD_TIMEOUT
1060
1061 struct vpd_readstate {
1062 device_t pcib;
1063 pcicfgregs *cfg;
1064 uint32_t val;
1065 int bytesinval;
1066 int off;
1067 uint8_t cksum;
1068 };
1069
1070 /* return 0 and one byte in *data if no read error, -1 else */
1071 static int
vpd_nextbyte(struct vpd_readstate * vrs,uint8_t * data)1072 vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data)
1073 {
1074 uint32_t reg;
1075 uint8_t byte;
1076
1077 if (vrs->bytesinval == 0) {
1078 if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®))
1079 return (-1);
1080 vrs->val = le32toh(reg);
1081 vrs->off += 4;
1082 byte = vrs->val & 0xff;
1083 vrs->bytesinval = 3;
1084 } else {
1085 vrs->val = vrs->val >> 8;
1086 byte = vrs->val & 0xff;
1087 vrs->bytesinval--;
1088 }
1089
1090 vrs->cksum += byte;
1091 *data = byte;
1092 return (0);
1093 }
1094
1095 /* return 0 on match, -1 and "unget" byte on no match */
1096 static int
vpd_expectbyte(struct vpd_readstate * vrs,uint8_t expected)1097 vpd_expectbyte(struct vpd_readstate *vrs, uint8_t expected)
1098 {
1099 uint8_t data;
1100
1101 if (vpd_nextbyte(vrs, &data) != 0)
1102 return (-1);
1103
1104 if (data == expected)
1105 return (0);
1106
1107 vrs->cksum -= data;
1108 vrs->val = (vrs->val << 8) + data;
1109 vrs->bytesinval++;
1110 return (-1);
1111 }
1112
1113 /* return size if tag matches, -1 on no match, -2 on read error */
1114 static int
vpd_read_tag_size(struct vpd_readstate * vrs,uint8_t vpd_tag)1115 vpd_read_tag_size(struct vpd_readstate *vrs, uint8_t vpd_tag)
1116 {
1117 uint8_t byte1, byte2;
1118
1119 if (vpd_expectbyte(vrs, vpd_tag) != 0)
1120 return (-1);
1121
1122 if ((vpd_tag & 0x80) == 0)
1123 return (vpd_tag & 0x07);
1124
1125 if (vpd_nextbyte(vrs, &byte1) != 0)
1126 return (-2);
1127 if (vpd_nextbyte(vrs, &byte2) != 0)
1128 return (-2);
1129
1130 return ((byte2 << 8) + byte1);
1131 }
1132
1133 /* (re)allocate buffer in multiples of 8 elements */
1134 static void*
alloc_buffer(void * buffer,size_t element_size,int needed)1135 alloc_buffer(void* buffer, size_t element_size, int needed)
1136 {
1137 int alloc, new_alloc;
1138
1139 alloc = roundup2(needed, 8);
1140 new_alloc = roundup2(needed + 1, 8);
1141 if (alloc != new_alloc) {
1142 buffer = reallocf(buffer,
1143 new_alloc * element_size, M_DEVBUF, M_WAITOK | M_ZERO);
1144 }
1145
1146 return (buffer);
1147 }
1148
1149 /* read VPD keyword and return element size, return -1 on read error */
1150 static int
vpd_read_elem_head(struct vpd_readstate * vrs,char keyword[2])1151 vpd_read_elem_head(struct vpd_readstate *vrs, char keyword[2])
1152 {
1153 uint8_t data;
1154
1155 if (vpd_nextbyte(vrs, &keyword[0]) != 0)
1156 return (-1);
1157 if (vpd_nextbyte(vrs, &keyword[1]) != 0)
1158 return (-1);
1159 if (vpd_nextbyte(vrs, &data) != 0)
1160 return (-1);
1161
1162 return (data);
1163 }
1164
1165 /* read VPD data element of given size into allocated buffer */
1166 static char *
vpd_read_value(struct vpd_readstate * vrs,int size)1167 vpd_read_value(struct vpd_readstate *vrs, int size)
1168 {
1169 int i;
1170 char char1;
1171 char *value;
1172
1173 value = malloc(size + 1, M_DEVBUF, M_WAITOK);
1174 for (i = 0; i < size; i++) {
1175 if (vpd_nextbyte(vrs, &char1) != 0) {
1176 free(value, M_DEVBUF);
1177 return (NULL);
1178 }
1179 value[i] = char1;
1180 }
1181 value[size] = '\0';
1182
1183 return (value);
1184 }
1185
1186 /* read VPD into *keyword and *value, return length of data element */
1187 static int
vpd_read_elem_data(struct vpd_readstate * vrs,char keyword[2],char ** value,int maxlen)1188 vpd_read_elem_data(struct vpd_readstate *vrs, char keyword[2], char **value, int maxlen)
1189 {
1190 int len;
1191
1192 len = vpd_read_elem_head(vrs, keyword);
1193 if (len > maxlen)
1194 return (-1);
1195 *value = vpd_read_value(vrs, len);
1196
1197 return (len);
1198 }
1199
1200 /* subtract all data following first byte from checksum of RV element */
1201 static void
vpd_fixup_cksum(struct vpd_readstate * vrs,char * rvstring,int len)1202 vpd_fixup_cksum(struct vpd_readstate *vrs, char *rvstring, int len)
1203 {
1204 int i;
1205 uint8_t fixup;
1206
1207 fixup = 0;
1208 for (i = 1; i < len; i++)
1209 fixup += rvstring[i];
1210 vrs->cksum -= fixup;
1211 }
1212
1213 /* fetch one read-only element and return size of heading + data */
1214 static size_t
next_vpd_ro_elem(struct vpd_readstate * vrs,int maxsize)1215 next_vpd_ro_elem(struct vpd_readstate *vrs, int maxsize)
1216 {
1217 struct pcicfg_vpd *vpd;
1218 pcicfgregs *cfg;
1219 struct vpd_readonly *vpd_ros;
1220 int len;
1221
1222 cfg = vrs->cfg;
1223 vpd = &cfg->vpd;
1224
1225 if (maxsize < 3)
1226 return (-1);
1227 vpd->vpd_ros = alloc_buffer(vpd->vpd_ros, sizeof(*vpd->vpd_ros), vpd->vpd_rocnt);
1228 vpd_ros = &vpd->vpd_ros[vpd->vpd_rocnt];
1229 maxsize -= 3;
1230 len = vpd_read_elem_data(vrs, vpd_ros->keyword, &vpd_ros->value, maxsize);
1231 if (vpd_ros->value == NULL)
1232 return (-1);
1233 vpd_ros->len = len;
1234 if (vpd_ros->keyword[0] == 'R' && vpd_ros->keyword[1] == 'V') {
1235 vpd_fixup_cksum(vrs, vpd_ros->value, len);
1236 if (vrs->cksum != 0) {
1237 pci_printf(cfg,
1238 "invalid VPD checksum %#hhx\n", vrs->cksum);
1239 return (-1);
1240 }
1241 }
1242 vpd->vpd_rocnt++;
1243
1244 return (len + 3);
1245 }
1246
1247 /* fetch one writable element and return size of heading + data */
1248 static size_t
next_vpd_rw_elem(struct vpd_readstate * vrs,int maxsize)1249 next_vpd_rw_elem(struct vpd_readstate *vrs, int maxsize)
1250 {
1251 struct pcicfg_vpd *vpd;
1252 pcicfgregs *cfg;
1253 struct vpd_write *vpd_w;
1254 int len;
1255
1256 cfg = vrs->cfg;
1257 vpd = &cfg->vpd;
1258
1259 if (maxsize < 3)
1260 return (-1);
1261 vpd->vpd_w = alloc_buffer(vpd->vpd_w, sizeof(*vpd->vpd_w), vpd->vpd_wcnt);
1262 if (vpd->vpd_w == NULL) {
1263 pci_printf(cfg, "out of memory");
1264 return (-1);
1265 }
1266 vpd_w = &vpd->vpd_w[vpd->vpd_wcnt];
1267 maxsize -= 3;
1268 vpd_w->start = vrs->off + 3 - vrs->bytesinval;
1269 len = vpd_read_elem_data(vrs, vpd_w->keyword, &vpd_w->value, maxsize);
1270 if (vpd_w->value == NULL)
1271 return (-1);
1272 vpd_w->len = len;
1273 vpd->vpd_wcnt++;
1274
1275 return (len + 3);
1276 }
1277
1278 /* free all memory allocated for VPD data */
1279 static void
vpd_free(struct pcicfg_vpd * vpd)1280 vpd_free(struct pcicfg_vpd *vpd)
1281 {
1282 int i;
1283
1284 free(vpd->vpd_ident, M_DEVBUF);
1285 for (i = 0; i < vpd->vpd_rocnt; i++)
1286 free(vpd->vpd_ros[i].value, M_DEVBUF);
1287 free(vpd->vpd_ros, M_DEVBUF);
1288 vpd->vpd_rocnt = 0;
1289 for (i = 0; i < vpd->vpd_wcnt; i++)
1290 free(vpd->vpd_w[i].value, M_DEVBUF);
1291 free(vpd->vpd_w, M_DEVBUF);
1292 vpd->vpd_wcnt = 0;
1293 }
1294
1295 #define VPD_TAG_END ((0x0f << 3) | 0) /* small tag, len == 0 */
1296 #define VPD_TAG_IDENT (0x02 | 0x80) /* large tag */
1297 #define VPD_TAG_RO (0x10 | 0x80) /* large tag */
1298 #define VPD_TAG_RW (0x11 | 0x80) /* large tag */
1299
1300 static int
pci_parse_vpd(device_t pcib,pcicfgregs * cfg)1301 pci_parse_vpd(device_t pcib, pcicfgregs *cfg)
1302 {
1303 struct vpd_readstate vrs;
1304 int cksumvalid;
1305 int size, elem_size;
1306
1307 /* init vpd reader */
1308 vrs.bytesinval = 0;
1309 vrs.off = 0;
1310 vrs.pcib = pcib;
1311 vrs.cfg = cfg;
1312 vrs.cksum = 0;
1313
1314 /* read VPD ident element - mandatory */
1315 size = vpd_read_tag_size(&vrs, VPD_TAG_IDENT);
1316 if (size <= 0) {
1317 pci_printf(cfg, "no VPD ident found\n");
1318 return (0);
1319 }
1320 cfg->vpd.vpd_ident = vpd_read_value(&vrs, size);
1321 if (cfg->vpd.vpd_ident == NULL) {
1322 pci_printf(cfg, "error accessing VPD ident data\n");
1323 return (0);
1324 }
1325
1326 /* read VPD RO elements - mandatory */
1327 size = vpd_read_tag_size(&vrs, VPD_TAG_RO);
1328 if (size <= 0) {
1329 pci_printf(cfg, "no read-only VPD data found\n");
1330 return (0);
1331 }
1332 while (size > 0) {
1333 elem_size = next_vpd_ro_elem(&vrs, size);
1334 if (elem_size < 0) {
1335 pci_printf(cfg, "error accessing read-only VPD data\n");
1336 return (-1);
1337 }
1338 size -= elem_size;
1339 }
1340 cksumvalid = (vrs.cksum == 0);
1341 if (!cksumvalid)
1342 return (-1);
1343
1344 /* read VPD RW elements - optional */
1345 size = vpd_read_tag_size(&vrs, VPD_TAG_RW);
1346 if (size == -2)
1347 return (-1);
1348 while (size > 0) {
1349 elem_size = next_vpd_rw_elem(&vrs, size);
1350 if (elem_size < 0) {
1351 pci_printf(cfg, "error accessing writeable VPD data\n");
1352 return (-1);
1353 }
1354 size -= elem_size;
1355 }
1356
1357 /* read empty END tag - mandatory */
1358 size = vpd_read_tag_size(&vrs, VPD_TAG_END);
1359 if (size != 0) {
1360 pci_printf(cfg, "No valid VPD end tag found\n");
1361 }
1362 return (0);
1363 }
1364
1365 static void
pci_read_vpd(device_t pcib,pcicfgregs * cfg)1366 pci_read_vpd(device_t pcib, pcicfgregs *cfg)
1367 {
1368 int status;
1369
1370 status = pci_parse_vpd(pcib, cfg);
1371 if (status < 0)
1372 vpd_free(&cfg->vpd);
1373 cfg->vpd.vpd_cached = 1;
1374 #undef REG
1375 #undef WREG
1376 }
1377
1378 int
pci_get_vpd_ident_method(device_t dev,device_t child,const char ** identptr)1379 pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr)
1380 {
1381 struct pci_devinfo *dinfo = device_get_ivars(child);
1382 pcicfgregs *cfg = &dinfo->cfg;
1383
1384 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1385 pci_read_vpd(device_get_parent(dev), cfg);
1386
1387 *identptr = cfg->vpd.vpd_ident;
1388
1389 if (*identptr == NULL)
1390 return (ENXIO);
1391
1392 return (0);
1393 }
1394
1395 int
pci_get_vpd_readonly_method(device_t dev,device_t child,const char * kw,const char ** vptr)1396 pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw,
1397 const char **vptr)
1398 {
1399 struct pci_devinfo *dinfo = device_get_ivars(child);
1400 pcicfgregs *cfg = &dinfo->cfg;
1401 int i;
1402
1403 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1404 pci_read_vpd(device_get_parent(dev), cfg);
1405
1406 for (i = 0; i < cfg->vpd.vpd_rocnt; i++)
1407 if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword,
1408 sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) {
1409 *vptr = cfg->vpd.vpd_ros[i].value;
1410 return (0);
1411 }
1412
1413 *vptr = NULL;
1414 return (ENXIO);
1415 }
1416
1417 struct pcicfg_vpd *
pci_fetch_vpd_list(device_t dev)1418 pci_fetch_vpd_list(device_t dev)
1419 {
1420 struct pci_devinfo *dinfo = device_get_ivars(dev);
1421 pcicfgregs *cfg = &dinfo->cfg;
1422
1423 if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0)
1424 pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg);
1425 return (&cfg->vpd);
1426 }
1427
1428 /*
1429 * Find the requested HyperTransport capability and return the offset
1430 * in configuration space via the pointer provided. The function
1431 * returns 0 on success and an error code otherwise.
1432 */
1433 int
pci_find_htcap_method(device_t dev,device_t child,int capability,int * capreg)1434 pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg)
1435 {
1436 int ptr, error;
1437 uint16_t val;
1438
1439 error = pci_find_cap(child, PCIY_HT, &ptr);
1440 if (error)
1441 return (error);
1442
1443 /*
1444 * Traverse the capabilities list checking each HT capability
1445 * to see if it matches the requested HT capability.
1446 */
1447 for (;;) {
1448 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1449 if (capability == PCIM_HTCAP_SLAVE ||
1450 capability == PCIM_HTCAP_HOST)
1451 val &= 0xe000;
1452 else
1453 val &= PCIM_HTCMD_CAP_MASK;
1454 if (val == capability) {
1455 if (capreg != NULL)
1456 *capreg = ptr;
1457 return (0);
1458 }
1459
1460 /* Skip to the next HT capability. */
1461 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1462 break;
1463 }
1464
1465 return (ENOENT);
1466 }
1467
1468 /*
1469 * Find the next requested HyperTransport capability after start and return
1470 * the offset in configuration space via the pointer provided. The function
1471 * returns 0 on success and an error code otherwise.
1472 */
1473 int
pci_find_next_htcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1474 pci_find_next_htcap_method(device_t dev, device_t child, int capability,
1475 int start, int *capreg)
1476 {
1477 int ptr;
1478 uint16_t val;
1479
1480 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT,
1481 ("start capability is not HyperTransport capability"));
1482 ptr = start;
1483
1484 /*
1485 * Traverse the capabilities list checking each HT capability
1486 * to see if it matches the requested HT capability.
1487 */
1488 for (;;) {
1489 /* Skip to the next HT capability. */
1490 if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0)
1491 break;
1492
1493 val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2);
1494 if (capability == PCIM_HTCAP_SLAVE ||
1495 capability == PCIM_HTCAP_HOST)
1496 val &= 0xe000;
1497 else
1498 val &= PCIM_HTCMD_CAP_MASK;
1499 if (val == capability) {
1500 if (capreg != NULL)
1501 *capreg = ptr;
1502 return (0);
1503 }
1504 }
1505
1506 return (ENOENT);
1507 }
1508
1509 /*
1510 * Find the requested capability and return the offset in
1511 * configuration space via the pointer provided. The function returns
1512 * 0 on success and an error code otherwise.
1513 */
1514 int
pci_find_cap_method(device_t dev,device_t child,int capability,int * capreg)1515 pci_find_cap_method(device_t dev, device_t child, int capability,
1516 int *capreg)
1517 {
1518 struct pci_devinfo *dinfo = device_get_ivars(child);
1519 pcicfgregs *cfg = &dinfo->cfg;
1520 uint32_t status;
1521 uint8_t ptr;
1522
1523 /*
1524 * Check the CAP_LIST bit of the PCI status register first.
1525 */
1526 status = pci_read_config(child, PCIR_STATUS, 2);
1527 if (!(status & PCIM_STATUS_CAPPRESENT))
1528 return (ENXIO);
1529
1530 /*
1531 * Determine the start pointer of the capabilities list.
1532 */
1533 switch (cfg->hdrtype & PCIM_HDRTYPE) {
1534 case PCIM_HDRTYPE_NORMAL:
1535 case PCIM_HDRTYPE_BRIDGE:
1536 ptr = PCIR_CAP_PTR;
1537 break;
1538 case PCIM_HDRTYPE_CARDBUS:
1539 ptr = PCIR_CAP_PTR_2;
1540 break;
1541 default:
1542 /* XXX: panic? */
1543 return (ENXIO); /* no extended capabilities support */
1544 }
1545 ptr = pci_read_config(child, ptr, 1);
1546
1547 /*
1548 * Traverse the capabilities list.
1549 */
1550 while (ptr != 0) {
1551 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1552 if (capreg != NULL)
1553 *capreg = ptr;
1554 return (0);
1555 }
1556 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1557 }
1558
1559 return (ENOENT);
1560 }
1561
1562 /*
1563 * Find the next requested capability after start and return the offset in
1564 * configuration space via the pointer provided. The function returns
1565 * 0 on success and an error code otherwise.
1566 */
1567 int
pci_find_next_cap_method(device_t dev,device_t child,int capability,int start,int * capreg)1568 pci_find_next_cap_method(device_t dev, device_t child, int capability,
1569 int start, int *capreg)
1570 {
1571 uint8_t ptr;
1572
1573 KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability,
1574 ("start capability is not expected capability"));
1575
1576 ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1);
1577 while (ptr != 0) {
1578 if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) {
1579 if (capreg != NULL)
1580 *capreg = ptr;
1581 return (0);
1582 }
1583 ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1);
1584 }
1585
1586 return (ENOENT);
1587 }
1588
1589 /*
1590 * Find the requested extended capability and return the offset in
1591 * configuration space via the pointer provided. The function returns
1592 * 0 on success and an error code otherwise.
1593 */
1594 int
pci_find_extcap_method(device_t dev,device_t child,int capability,int * capreg)1595 pci_find_extcap_method(device_t dev, device_t child, int capability,
1596 int *capreg)
1597 {
1598 struct pci_devinfo *dinfo = device_get_ivars(child);
1599 pcicfgregs *cfg = &dinfo->cfg;
1600 uint32_t ecap;
1601 uint16_t ptr;
1602
1603 /* Only supported for PCI-express devices. */
1604 if (cfg->pcie.pcie_location == 0)
1605 return (ENXIO);
1606
1607 ptr = PCIR_EXTCAP;
1608 ecap = pci_read_config(child, ptr, 4);
1609 if (ecap == 0xffffffff || ecap == 0)
1610 return (ENOENT);
1611 for (;;) {
1612 if (PCI_EXTCAP_ID(ecap) == capability) {
1613 if (capreg != NULL)
1614 *capreg = ptr;
1615 return (0);
1616 }
1617 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1618 if (ptr == 0)
1619 break;
1620 ecap = pci_read_config(child, ptr, 4);
1621 }
1622
1623 return (ENOENT);
1624 }
1625
1626 /*
1627 * Find the next requested extended capability after start and return the
1628 * offset in configuration space via the pointer provided. The function
1629 * returns 0 on success and an error code otherwise.
1630 */
1631 int
pci_find_next_extcap_method(device_t dev,device_t child,int capability,int start,int * capreg)1632 pci_find_next_extcap_method(device_t dev, device_t child, int capability,
1633 int start, int *capreg)
1634 {
1635 struct pci_devinfo *dinfo = device_get_ivars(child);
1636 pcicfgregs *cfg = &dinfo->cfg;
1637 uint32_t ecap;
1638 uint16_t ptr;
1639
1640 /* Only supported for PCI-express devices. */
1641 if (cfg->pcie.pcie_location == 0)
1642 return (ENXIO);
1643
1644 ecap = pci_read_config(child, start, 4);
1645 KASSERT(PCI_EXTCAP_ID(ecap) == capability,
1646 ("start extended capability is not expected capability"));
1647 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1648 while (ptr != 0) {
1649 ecap = pci_read_config(child, ptr, 4);
1650 if (PCI_EXTCAP_ID(ecap) == capability) {
1651 if (capreg != NULL)
1652 *capreg = ptr;
1653 return (0);
1654 }
1655 ptr = PCI_EXTCAP_NEXTPTR(ecap);
1656 }
1657
1658 return (ENOENT);
1659 }
1660
1661 /*
1662 * Support for MSI-X message interrupts.
1663 */
1664 static void
pci_write_msix_entry(device_t dev,u_int index,uint64_t address,uint32_t data)1665 pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data)
1666 {
1667 struct pci_devinfo *dinfo = device_get_ivars(dev);
1668 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1669 uint32_t offset;
1670
1671 KASSERT(msix->msix_table_len > index, ("bogus index"));
1672 offset = msix->msix_table_offset + index * 16;
1673 bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
1674 bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
1675 bus_write_4(msix->msix_table_res, offset + 8, data);
1676 }
1677
1678 void
pci_enable_msix_method(device_t dev,device_t child,u_int index,uint64_t address,uint32_t data)1679 pci_enable_msix_method(device_t dev, device_t child, u_int index,
1680 uint64_t address, uint32_t data)
1681 {
1682
1683 if (pci_msix_rewrite_table) {
1684 struct pci_devinfo *dinfo = device_get_ivars(child);
1685 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1686
1687 /*
1688 * Some VM hosts require MSIX to be disabled in the
1689 * control register before updating the MSIX table
1690 * entries are allowed. It is not enough to only
1691 * disable MSIX while updating a single entry. MSIX
1692 * must be disabled while updating all entries in the
1693 * table.
1694 */
1695 pci_write_config(child,
1696 msix->msix_location + PCIR_MSIX_CTRL,
1697 msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
1698 pci_resume_msix(child);
1699 } else
1700 pci_write_msix_entry(child, index, address, data);
1701
1702 /* Enable MSI -> HT mapping. */
1703 pci_ht_map_msi(child, address);
1704 }
1705
1706 void
pci_mask_msix(device_t dev,u_int index)1707 pci_mask_msix(device_t dev, u_int index)
1708 {
1709 struct pci_devinfo *dinfo = device_get_ivars(dev);
1710 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1711 uint32_t offset, val;
1712
1713 KASSERT(msix->msix_msgnum > index, ("bogus index"));
1714 offset = msix->msix_table_offset + index * 16 + 12;
1715 val = bus_read_4(msix->msix_table_res, offset);
1716 val |= PCIM_MSIX_VCTRL_MASK;
1717
1718 /*
1719 * Some devices (e.g. Samsung PM961) do not support reads of this
1720 * register, so always write the new value.
1721 */
1722 bus_write_4(msix->msix_table_res, offset, val);
1723 }
1724
1725 void
pci_unmask_msix(device_t dev,u_int index)1726 pci_unmask_msix(device_t dev, u_int index)
1727 {
1728 struct pci_devinfo *dinfo = device_get_ivars(dev);
1729 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1730 uint32_t offset, val;
1731
1732 KASSERT(msix->msix_table_len > index, ("bogus index"));
1733 offset = msix->msix_table_offset + index * 16 + 12;
1734 val = bus_read_4(msix->msix_table_res, offset);
1735 val &= ~PCIM_MSIX_VCTRL_MASK;
1736
1737 /*
1738 * Some devices (e.g. Samsung PM961) do not support reads of this
1739 * register, so always write the new value.
1740 */
1741 bus_write_4(msix->msix_table_res, offset, val);
1742 }
1743
1744 int
pci_pending_msix(device_t dev,u_int index)1745 pci_pending_msix(device_t dev, u_int index)
1746 {
1747 struct pci_devinfo *dinfo = device_get_ivars(dev);
1748 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1749 uint32_t offset, bit;
1750
1751 KASSERT(msix->msix_table_len > index, ("bogus index"));
1752 offset = msix->msix_pba_offset + (index / 32) * 4;
1753 bit = 1 << index % 32;
1754 return (bus_read_4(msix->msix_pba_res, offset) & bit);
1755 }
1756
1757 /*
1758 * Restore MSI-X registers and table during resume. If MSI-X is
1759 * enabled then walk the virtual table to restore the actual MSI-X
1760 * table.
1761 */
1762 static void
pci_resume_msix(device_t dev)1763 pci_resume_msix(device_t dev)
1764 {
1765 struct pci_devinfo *dinfo = device_get_ivars(dev);
1766 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1767 struct msix_table_entry *mte;
1768 struct msix_vector *mv;
1769 int i;
1770
1771 if (msix->msix_alloc > 0) {
1772 /* First, mask all vectors. */
1773 for (i = 0; i < msix->msix_msgnum; i++)
1774 pci_mask_msix(dev, i);
1775
1776 /* Second, program any messages with at least one handler. */
1777 for (i = 0; i < msix->msix_table_len; i++) {
1778 mte = &msix->msix_table[i];
1779 if (mte->mte_vector == 0 || mte->mte_handlers == 0)
1780 continue;
1781 mv = &msix->msix_vectors[mte->mte_vector - 1];
1782 pci_write_msix_entry(dev, i, mv->mv_address,
1783 mv->mv_data);
1784 pci_unmask_msix(dev, i);
1785 }
1786 }
1787 pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
1788 msix->msix_ctrl, 2);
1789 }
1790
1791 /*
1792 * Attempt to allocate *count MSI-X messages. The actual number allocated is
1793 * returned in *count. After this function returns, each message will be
1794 * available to the driver as SYS_RES_IRQ resources starting at rid 1.
1795 */
1796 int
pci_alloc_msix_method(device_t dev,device_t child,int * count)1797 pci_alloc_msix_method(device_t dev, device_t child, int *count)
1798 {
1799 struct pci_devinfo *dinfo = device_get_ivars(child);
1800 pcicfgregs *cfg = &dinfo->cfg;
1801 struct resource_list_entry *rle;
1802 int actual, error, i, irq, max;
1803
1804 /* Don't let count == 0 get us into trouble. */
1805 if (*count == 0)
1806 return (EINVAL);
1807
1808 /* If rid 0 is allocated, then fail. */
1809 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
1810 if (rle != NULL && rle->res != NULL)
1811 return (ENXIO);
1812
1813 /* Already have allocated messages? */
1814 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
1815 return (ENXIO);
1816
1817 /* If MSI-X is blacklisted for this system, fail. */
1818 if (pci_msix_blacklisted())
1819 return (ENXIO);
1820
1821 /* MSI-X capability present? */
1822 if (cfg->msix.msix_location == 0 || !pci_do_msix)
1823 return (ENODEV);
1824
1825 /* Make sure the appropriate BARs are mapped. */
1826 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1827 cfg->msix.msix_table_bar);
1828 if (rle == NULL || rle->res == NULL ||
1829 !(rman_get_flags(rle->res) & RF_ACTIVE))
1830 return (ENXIO);
1831 cfg->msix.msix_table_res = rle->res;
1832 if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) {
1833 rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY,
1834 cfg->msix.msix_pba_bar);
1835 if (rle == NULL || rle->res == NULL ||
1836 !(rman_get_flags(rle->res) & RF_ACTIVE))
1837 return (ENXIO);
1838 }
1839 cfg->msix.msix_pba_res = rle->res;
1840
1841 if (bootverbose)
1842 device_printf(child,
1843 "attempting to allocate %d MSI-X vectors (%d supported)\n",
1844 *count, cfg->msix.msix_msgnum);
1845 max = min(*count, cfg->msix.msix_msgnum);
1846 for (i = 0; i < max; i++) {
1847 /* Allocate a message. */
1848 error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
1849 if (error) {
1850 if (i == 0)
1851 return (error);
1852 break;
1853 }
1854 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
1855 irq, 1);
1856 }
1857 actual = i;
1858
1859 if (bootverbose) {
1860 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1);
1861 if (actual == 1)
1862 device_printf(child, "using IRQ %ju for MSI-X\n",
1863 rle->start);
1864 else {
1865 int run;
1866
1867 /*
1868 * Be fancy and try to print contiguous runs of
1869 * IRQ values as ranges. 'irq' is the previous IRQ.
1870 * 'run' is true if we are in a range.
1871 */
1872 device_printf(child, "using IRQs %ju", rle->start);
1873 irq = rle->start;
1874 run = 0;
1875 for (i = 1; i < actual; i++) {
1876 rle = resource_list_find(&dinfo->resources,
1877 SYS_RES_IRQ, i + 1);
1878
1879 /* Still in a run? */
1880 if (rle->start == irq + 1) {
1881 run = 1;
1882 irq++;
1883 continue;
1884 }
1885
1886 /* Finish previous range. */
1887 if (run) {
1888 printf("-%d", irq);
1889 run = 0;
1890 }
1891
1892 /* Start new range. */
1893 printf(",%ju", rle->start);
1894 irq = rle->start;
1895 }
1896
1897 /* Unfinished range? */
1898 if (run)
1899 printf("-%d", irq);
1900 printf(" for MSI-X\n");
1901 }
1902 }
1903
1904 /* Mask all vectors. */
1905 for (i = 0; i < cfg->msix.msix_msgnum; i++)
1906 pci_mask_msix(child, i);
1907
1908 /* Allocate and initialize vector data and virtual table. */
1909 cfg->msix.msix_vectors = malloc(sizeof(struct msix_vector) * actual,
1910 M_DEVBUF, M_WAITOK | M_ZERO);
1911 cfg->msix.msix_table = malloc(sizeof(struct msix_table_entry) * actual,
1912 M_DEVBUF, M_WAITOK | M_ZERO);
1913 for (i = 0; i < actual; i++) {
1914 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
1915 cfg->msix.msix_vectors[i].mv_irq = rle->start;
1916 cfg->msix.msix_table[i].mte_vector = i + 1;
1917 }
1918
1919 /* Update control register to enable MSI-X. */
1920 cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
1921 pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
1922 cfg->msix.msix_ctrl, 2);
1923
1924 /* Update counts of alloc'd messages. */
1925 cfg->msix.msix_alloc = actual;
1926 cfg->msix.msix_table_len = actual;
1927 *count = actual;
1928 return (0);
1929 }
1930
1931 /*
1932 * By default, pci_alloc_msix() will assign the allocated IRQ
1933 * resources consecutively to the first N messages in the MSI-X table.
1934 * However, device drivers may want to use different layouts if they
1935 * either receive fewer messages than they asked for, or they wish to
1936 * populate the MSI-X table sparsely. This method allows the driver
1937 * to specify what layout it wants. It must be called after a
1938 * successful pci_alloc_msix() but before any of the associated
1939 * SYS_RES_IRQ resources are allocated via bus_alloc_resource().
1940 *
1941 * The 'vectors' array contains 'count' message vectors. The array
1942 * maps directly to the MSI-X table in that index 0 in the array
1943 * specifies the vector for the first message in the MSI-X table, etc.
1944 * The vector value in each array index can either be 0 to indicate
1945 * that no vector should be assigned to a message slot, or it can be a
1946 * number from 1 to N (where N is the count returned from a
1947 * succcessful call to pci_alloc_msix()) to indicate which message
1948 * vector (IRQ) to be used for the corresponding message.
1949 *
1950 * On successful return, each message with a non-zero vector will have
1951 * an associated SYS_RES_IRQ whose rid is equal to the array index +
1952 * 1. Additionally, if any of the IRQs allocated via the previous
1953 * call to pci_alloc_msix() are not used in the mapping, those IRQs
1954 * will be freed back to the system automatically.
1955 *
1956 * For example, suppose a driver has a MSI-X table with 6 messages and
1957 * asks for 6 messages, but pci_alloc_msix() only returns a count of
1958 * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and
1959 * C. After the call to pci_alloc_msix(), the device will be setup to
1960 * have an MSI-X table of ABC--- (where - means no vector assigned).
1961 * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 },
1962 * then the MSI-X table will look like A-AB-B, and the 'C' vector will
1963 * be freed back to the system. This device will also have valid
1964 * SYS_RES_IRQ rids of 1, 3, 4, and 6.
1965 *
1966 * In any case, the SYS_RES_IRQ rid X will always map to the message
1967 * at MSI-X table index X - 1 and will only be valid if a vector is
1968 * assigned to that table entry.
1969 */
1970 int
pci_remap_msix_method(device_t dev,device_t child,int count,const u_int * vectors)1971 pci_remap_msix_method(device_t dev, device_t child, int count,
1972 const u_int *vectors)
1973 {
1974 struct pci_devinfo *dinfo = device_get_ivars(child);
1975 struct pcicfg_msix *msix = &dinfo->cfg.msix;
1976 struct resource_list_entry *rle;
1977 int i, irq, j, *used;
1978
1979 /*
1980 * Have to have at least one message in the table but the
1981 * table can't be bigger than the actual MSI-X table in the
1982 * device.
1983 */
1984 if (count == 0 || count > msix->msix_msgnum)
1985 return (EINVAL);
1986
1987 /* Sanity check the vectors. */
1988 for (i = 0; i < count; i++)
1989 if (vectors[i] > msix->msix_alloc)
1990 return (EINVAL);
1991
1992 /*
1993 * Make sure there aren't any holes in the vectors to be used.
1994 * It's a big pain to support it, and it doesn't really make
1995 * sense anyway. Also, at least one vector must be used.
1996 */
1997 used = malloc(sizeof(int) * msix->msix_alloc, M_DEVBUF, M_WAITOK |
1998 M_ZERO);
1999 for (i = 0; i < count; i++)
2000 if (vectors[i] != 0)
2001 used[vectors[i] - 1] = 1;
2002 for (i = 0; i < msix->msix_alloc - 1; i++)
2003 if (used[i] == 0 && used[i + 1] == 1) {
2004 free(used, M_DEVBUF);
2005 return (EINVAL);
2006 }
2007 if (used[0] != 1) {
2008 free(used, M_DEVBUF);
2009 return (EINVAL);
2010 }
2011
2012 /* Make sure none of the resources are allocated. */
2013 for (i = 0; i < msix->msix_table_len; i++) {
2014 if (msix->msix_table[i].mte_vector == 0)
2015 continue;
2016 if (msix->msix_table[i].mte_handlers > 0) {
2017 free(used, M_DEVBUF);
2018 return (EBUSY);
2019 }
2020 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2021 KASSERT(rle != NULL, ("missing resource"));
2022 if (rle->res != NULL) {
2023 free(used, M_DEVBUF);
2024 return (EBUSY);
2025 }
2026 }
2027
2028 /* Free the existing resource list entries. */
2029 for (i = 0; i < msix->msix_table_len; i++) {
2030 if (msix->msix_table[i].mte_vector == 0)
2031 continue;
2032 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2033 }
2034
2035 /*
2036 * Build the new virtual table keeping track of which vectors are
2037 * used.
2038 */
2039 free(msix->msix_table, M_DEVBUF);
2040 msix->msix_table = malloc(sizeof(struct msix_table_entry) * count,
2041 M_DEVBUF, M_WAITOK | M_ZERO);
2042 for (i = 0; i < count; i++)
2043 msix->msix_table[i].mte_vector = vectors[i];
2044 msix->msix_table_len = count;
2045
2046 /* Free any unused IRQs and resize the vectors array if necessary. */
2047 j = msix->msix_alloc - 1;
2048 if (used[j] == 0) {
2049 struct msix_vector *vec;
2050
2051 while (used[j] == 0) {
2052 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2053 msix->msix_vectors[j].mv_irq);
2054 j--;
2055 }
2056 vec = malloc(sizeof(struct msix_vector) * (j + 1), M_DEVBUF,
2057 M_WAITOK);
2058 bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) *
2059 (j + 1));
2060 free(msix->msix_vectors, M_DEVBUF);
2061 msix->msix_vectors = vec;
2062 msix->msix_alloc = j + 1;
2063 }
2064 free(used, M_DEVBUF);
2065
2066 /* Map the IRQs onto the rids. */
2067 for (i = 0; i < count; i++) {
2068 if (vectors[i] == 0)
2069 continue;
2070 irq = msix->msix_vectors[vectors[i] - 1].mv_irq;
2071 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
2072 irq, 1);
2073 }
2074
2075 if (bootverbose) {
2076 device_printf(child, "Remapped MSI-X IRQs as: ");
2077 for (i = 0; i < count; i++) {
2078 if (i != 0)
2079 printf(", ");
2080 if (vectors[i] == 0)
2081 printf("---");
2082 else
2083 printf("%d",
2084 msix->msix_vectors[vectors[i] - 1].mv_irq);
2085 }
2086 printf("\n");
2087 }
2088
2089 return (0);
2090 }
2091
2092 static int
pci_release_msix(device_t dev,device_t child)2093 pci_release_msix(device_t dev, device_t child)
2094 {
2095 struct pci_devinfo *dinfo = device_get_ivars(child);
2096 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2097 struct resource_list_entry *rle;
2098 int i;
2099
2100 /* Do we have any messages to release? */
2101 if (msix->msix_alloc == 0)
2102 return (ENODEV);
2103
2104 /* Make sure none of the resources are allocated. */
2105 for (i = 0; i < msix->msix_table_len; i++) {
2106 if (msix->msix_table[i].mte_vector == 0)
2107 continue;
2108 if (msix->msix_table[i].mte_handlers > 0)
2109 return (EBUSY);
2110 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2111 KASSERT(rle != NULL, ("missing resource"));
2112 if (rle->res != NULL)
2113 return (EBUSY);
2114 }
2115
2116 /* Update control register to disable MSI-X. */
2117 msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE;
2118 pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
2119 msix->msix_ctrl, 2);
2120
2121 /* Free the resource list entries. */
2122 for (i = 0; i < msix->msix_table_len; i++) {
2123 if (msix->msix_table[i].mte_vector == 0)
2124 continue;
2125 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2126 }
2127 free(msix->msix_table, M_DEVBUF);
2128 msix->msix_table_len = 0;
2129
2130 /* Release the IRQs. */
2131 for (i = 0; i < msix->msix_alloc; i++)
2132 PCIB_RELEASE_MSIX(device_get_parent(dev), child,
2133 msix->msix_vectors[i].mv_irq);
2134 free(msix->msix_vectors, M_DEVBUF);
2135 msix->msix_alloc = 0;
2136 return (0);
2137 }
2138
2139 /*
2140 * Return the max supported MSI-X messages this device supports.
2141 * Basically, assuming the MD code can alloc messages, this function
2142 * should return the maximum value that pci_alloc_msix() can return.
2143 * Thus, it is subject to the tunables, etc.
2144 */
2145 int
pci_msix_count_method(device_t dev,device_t child)2146 pci_msix_count_method(device_t dev, device_t child)
2147 {
2148 struct pci_devinfo *dinfo = device_get_ivars(child);
2149 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2150
2151 if (pci_do_msix && msix->msix_location != 0)
2152 return (msix->msix_msgnum);
2153 return (0);
2154 }
2155
2156 int
pci_msix_pba_bar_method(device_t dev,device_t child)2157 pci_msix_pba_bar_method(device_t dev, device_t child)
2158 {
2159 struct pci_devinfo *dinfo = device_get_ivars(child);
2160 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2161
2162 if (pci_do_msix && msix->msix_location != 0)
2163 return (msix->msix_pba_bar);
2164 return (-1);
2165 }
2166
2167 int
pci_msix_table_bar_method(device_t dev,device_t child)2168 pci_msix_table_bar_method(device_t dev, device_t child)
2169 {
2170 struct pci_devinfo *dinfo = device_get_ivars(child);
2171 struct pcicfg_msix *msix = &dinfo->cfg.msix;
2172
2173 if (pci_do_msix && msix->msix_location != 0)
2174 return (msix->msix_table_bar);
2175 return (-1);
2176 }
2177
2178 /*
2179 * HyperTransport MSI mapping control
2180 */
2181 void
pci_ht_map_msi(device_t dev,uint64_t addr)2182 pci_ht_map_msi(device_t dev, uint64_t addr)
2183 {
2184 struct pci_devinfo *dinfo = device_get_ivars(dev);
2185 struct pcicfg_ht *ht = &dinfo->cfg.ht;
2186
2187 if (!ht->ht_msimap)
2188 return;
2189
2190 if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) &&
2191 ht->ht_msiaddr >> 20 == addr >> 20) {
2192 /* Enable MSI -> HT mapping. */
2193 ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE;
2194 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2195 ht->ht_msictrl, 2);
2196 }
2197
2198 if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) {
2199 /* Disable MSI -> HT mapping. */
2200 ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE;
2201 pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND,
2202 ht->ht_msictrl, 2);
2203 }
2204 }
2205
2206 int
pci_get_relaxed_ordering_enabled(device_t dev)2207 pci_get_relaxed_ordering_enabled(device_t dev)
2208 {
2209 struct pci_devinfo *dinfo = device_get_ivars(dev);
2210 int cap;
2211 uint16_t val;
2212
2213 cap = dinfo->cfg.pcie.pcie_location;
2214 if (cap == 0)
2215 return (0);
2216 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2217 val &= PCIEM_CTL_RELAXED_ORD_ENABLE;
2218 return (val != 0);
2219 }
2220
2221 int
pci_get_max_payload(device_t dev)2222 pci_get_max_payload(device_t dev)
2223 {
2224 struct pci_devinfo *dinfo = device_get_ivars(dev);
2225 int cap;
2226 uint16_t val;
2227
2228 cap = dinfo->cfg.pcie.pcie_location;
2229 if (cap == 0)
2230 return (0);
2231 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2232 val &= PCIEM_CTL_MAX_PAYLOAD;
2233 val >>= 5;
2234 return (1 << (val + 7));
2235 }
2236
2237 int
pci_get_max_read_req(device_t dev)2238 pci_get_max_read_req(device_t dev)
2239 {
2240 struct pci_devinfo *dinfo = device_get_ivars(dev);
2241 int cap;
2242 uint16_t val;
2243
2244 cap = dinfo->cfg.pcie.pcie_location;
2245 if (cap == 0)
2246 return (0);
2247 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2248 val &= PCIEM_CTL_MAX_READ_REQUEST;
2249 val >>= 12;
2250 return (1 << (val + 7));
2251 }
2252
2253 int
pci_set_max_read_req(device_t dev,int size)2254 pci_set_max_read_req(device_t dev, int size)
2255 {
2256 struct pci_devinfo *dinfo = device_get_ivars(dev);
2257 int cap;
2258 uint16_t val;
2259
2260 cap = dinfo->cfg.pcie.pcie_location;
2261 if (cap == 0)
2262 return (0);
2263 if (size < 128)
2264 size = 128;
2265 if (size > 4096)
2266 size = 4096;
2267 size = (1 << (fls(size) - 1));
2268 val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
2269 val &= ~PCIEM_CTL_MAX_READ_REQUEST;
2270 val |= (fls(size) - 8) << 12;
2271 pci_write_config(dev, cap + PCIER_DEVICE_CTL, val, 2);
2272 return (size);
2273 }
2274
2275 uint32_t
pcie_read_config(device_t dev,int reg,int width)2276 pcie_read_config(device_t dev, int reg, int width)
2277 {
2278 struct pci_devinfo *dinfo = device_get_ivars(dev);
2279 int cap;
2280
2281 cap = dinfo->cfg.pcie.pcie_location;
2282 if (cap == 0) {
2283 if (width == 2)
2284 return (0xffff);
2285 return (0xffffffff);
2286 }
2287
2288 return (pci_read_config(dev, cap + reg, width));
2289 }
2290
2291 void
pcie_write_config(device_t dev,int reg,uint32_t value,int width)2292 pcie_write_config(device_t dev, int reg, uint32_t value, int width)
2293 {
2294 struct pci_devinfo *dinfo = device_get_ivars(dev);
2295 int cap;
2296
2297 cap = dinfo->cfg.pcie.pcie_location;
2298 if (cap == 0)
2299 return;
2300 pci_write_config(dev, cap + reg, value, width);
2301 }
2302
2303 /*
2304 * Adjusts a PCI-e capability register by clearing the bits in mask
2305 * and setting the bits in (value & mask). Bits not set in mask are
2306 * not adjusted.
2307 *
2308 * Returns the old value on success or all ones on failure.
2309 */
2310 uint32_t
pcie_adjust_config(device_t dev,int reg,uint32_t mask,uint32_t value,int width)2311 pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value,
2312 int width)
2313 {
2314 struct pci_devinfo *dinfo = device_get_ivars(dev);
2315 uint32_t old, new;
2316 int cap;
2317
2318 cap = dinfo->cfg.pcie.pcie_location;
2319 if (cap == 0) {
2320 if (width == 2)
2321 return (0xffff);
2322 return (0xffffffff);
2323 }
2324
2325 old = pci_read_config(dev, cap + reg, width);
2326 new = old & ~mask;
2327 new |= (value & mask);
2328 pci_write_config(dev, cap + reg, new, width);
2329 return (old);
2330 }
2331
2332 /*
2333 * Support for MSI message signalled interrupts.
2334 */
2335 void
pci_enable_msi_method(device_t dev,device_t child,uint64_t address,uint16_t data)2336 pci_enable_msi_method(device_t dev, device_t child, uint64_t address,
2337 uint16_t data)
2338 {
2339 struct pci_devinfo *dinfo = device_get_ivars(child);
2340 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2341
2342 /* Write data and address values. */
2343 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR,
2344 address & 0xffffffff, 4);
2345 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2346 pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH,
2347 address >> 32, 4);
2348 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT,
2349 data, 2);
2350 } else
2351 pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data,
2352 2);
2353
2354 /* Enable MSI in the control register. */
2355 msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE;
2356 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2357 msi->msi_ctrl, 2);
2358
2359 /* Enable MSI -> HT mapping. */
2360 pci_ht_map_msi(child, address);
2361 }
2362
2363 void
pci_disable_msi_method(device_t dev,device_t child)2364 pci_disable_msi_method(device_t dev, device_t child)
2365 {
2366 struct pci_devinfo *dinfo = device_get_ivars(child);
2367 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2368
2369 /* Disable MSI -> HT mapping. */
2370 pci_ht_map_msi(child, 0);
2371
2372 /* Disable MSI in the control register. */
2373 msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
2374 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2375 msi->msi_ctrl, 2);
2376 }
2377
2378 /*
2379 * Restore MSI registers during resume. If MSI is enabled then
2380 * restore the data and address registers in addition to the control
2381 * register.
2382 */
2383 static void
pci_resume_msi(device_t dev)2384 pci_resume_msi(device_t dev)
2385 {
2386 struct pci_devinfo *dinfo = device_get_ivars(dev);
2387 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2388 uint64_t address;
2389 uint16_t data;
2390
2391 if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) {
2392 address = msi->msi_addr;
2393 data = msi->msi_data;
2394 pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
2395 address & 0xffffffff, 4);
2396 if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
2397 pci_write_config(dev, msi->msi_location +
2398 PCIR_MSI_ADDR_HIGH, address >> 32, 4);
2399 pci_write_config(dev, msi->msi_location +
2400 PCIR_MSI_DATA_64BIT, data, 2);
2401 } else
2402 pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA,
2403 data, 2);
2404 }
2405 pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
2406 2);
2407 }
2408
2409 static int
pci_remap_intr_method(device_t bus,device_t dev,u_int irq)2410 pci_remap_intr_method(device_t bus, device_t dev, u_int irq)
2411 {
2412 struct pci_devinfo *dinfo = device_get_ivars(dev);
2413 pcicfgregs *cfg = &dinfo->cfg;
2414 struct resource_list_entry *rle;
2415 struct msix_table_entry *mte;
2416 struct msix_vector *mv;
2417 uint64_t addr;
2418 uint32_t data;
2419 int error, i, j;
2420
2421 /*
2422 * Handle MSI first. We try to find this IRQ among our list
2423 * of MSI IRQs. If we find it, we request updated address and
2424 * data registers and apply the results.
2425 */
2426 if (cfg->msi.msi_alloc > 0) {
2427 /* If we don't have any active handlers, nothing to do. */
2428 if (cfg->msi.msi_handlers == 0)
2429 return (0);
2430 for (i = 0; i < cfg->msi.msi_alloc; i++) {
2431 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
2432 i + 1);
2433 if (rle->start == irq) {
2434 error = PCIB_MAP_MSI(device_get_parent(bus),
2435 dev, irq, &addr, &data);
2436 if (error)
2437 return (error);
2438 pci_disable_msi(dev);
2439 dinfo->cfg.msi.msi_addr = addr;
2440 dinfo->cfg.msi.msi_data = data;
2441 pci_enable_msi(dev, addr, data);
2442 return (0);
2443 }
2444 }
2445 return (ENOENT);
2446 }
2447
2448 /*
2449 * For MSI-X, we check to see if we have this IRQ. If we do,
2450 * we request the updated mapping info. If that works, we go
2451 * through all the slots that use this IRQ and update them.
2452 */
2453 if (cfg->msix.msix_alloc > 0) {
2454 bool found = false;
2455
2456 for (i = 0; i < cfg->msix.msix_alloc; i++) {
2457 mv = &cfg->msix.msix_vectors[i];
2458 if (mv->mv_irq == irq) {
2459 error = PCIB_MAP_MSI(device_get_parent(bus),
2460 dev, irq, &addr, &data);
2461 if (error)
2462 return (error);
2463 mv->mv_address = addr;
2464 mv->mv_data = data;
2465 for (j = 0; j < cfg->msix.msix_table_len; j++) {
2466 mte = &cfg->msix.msix_table[j];
2467 if (mte->mte_vector != i + 1)
2468 continue;
2469 if (mte->mte_handlers == 0)
2470 continue;
2471 pci_mask_msix(dev, j);
2472 pci_enable_msix(dev, j, addr, data);
2473 pci_unmask_msix(dev, j);
2474 }
2475 found = true;
2476 }
2477 }
2478 return (found ? 0 : ENOENT);
2479 }
2480
2481 return (ENOENT);
2482 }
2483
2484 /*
2485 * Returns true if the specified device is blacklisted because MSI
2486 * doesn't work.
2487 */
2488 int
pci_msi_device_blacklisted(device_t dev)2489 pci_msi_device_blacklisted(device_t dev)
2490 {
2491
2492 if (!pci_honor_msi_blacklist)
2493 return (0);
2494
2495 return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
2496 }
2497
2498 /*
2499 * Determine if MSI is blacklisted globally on this system. Currently,
2500 * we just check for blacklisted chipsets as represented by the
2501 * host-PCI bridge at device 0:0:0. In the future, it may become
2502 * necessary to check other system attributes, such as the kenv values
2503 * that give the motherboard manufacturer and model number.
2504 */
2505 static int
pci_msi_blacklisted(void)2506 pci_msi_blacklisted(void)
2507 {
2508 device_t dev;
2509
2510 if (!pci_honor_msi_blacklist)
2511 return (0);
2512
2513 /* Blacklist all non-PCI-express and non-PCI-X chipsets. */
2514 if (!(pcie_chipset || pcix_chipset)) {
2515 if (vm_guest != VM_GUEST_NO) {
2516 /*
2517 * Whitelist older chipsets in virtual
2518 * machines known to support MSI.
2519 */
2520 dev = pci_find_bsf(0, 0, 0);
2521 if (dev != NULL)
2522 return (!pci_has_quirk(pci_get_devid(dev),
2523 PCI_QUIRK_ENABLE_MSI_VM));
2524 }
2525 return (1);
2526 }
2527
2528 dev = pci_find_bsf(0, 0, 0);
2529 if (dev != NULL)
2530 return (pci_msi_device_blacklisted(dev));
2531 return (0);
2532 }
2533
2534 /*
2535 * Returns true if the specified device is blacklisted because MSI-X
2536 * doesn't work. Note that this assumes that if MSI doesn't work,
2537 * MSI-X doesn't either.
2538 */
2539 int
pci_msix_device_blacklisted(device_t dev)2540 pci_msix_device_blacklisted(device_t dev)
2541 {
2542
2543 if (!pci_honor_msi_blacklist)
2544 return (0);
2545
2546 if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
2547 return (1);
2548
2549 return (pci_msi_device_blacklisted(dev));
2550 }
2551
2552 /*
2553 * Determine if MSI-X is blacklisted globally on this system. If MSI
2554 * is blacklisted, assume that MSI-X is as well. Check for additional
2555 * chipsets where MSI works but MSI-X does not.
2556 */
2557 static int
pci_msix_blacklisted(void)2558 pci_msix_blacklisted(void)
2559 {
2560 device_t dev;
2561
2562 if (!pci_honor_msi_blacklist)
2563 return (0);
2564
2565 dev = pci_find_bsf(0, 0, 0);
2566 if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
2567 PCI_QUIRK_DISABLE_MSIX))
2568 return (1);
2569
2570 return (pci_msi_blacklisted());
2571 }
2572
2573 /*
2574 * Attempt to allocate *count MSI messages. The actual number allocated is
2575 * returned in *count. After this function returns, each message will be
2576 * available to the driver as SYS_RES_IRQ resources starting at a rid 1.
2577 */
2578 int
pci_alloc_msi_method(device_t dev,device_t child,int * count)2579 pci_alloc_msi_method(device_t dev, device_t child, int *count)
2580 {
2581 struct pci_devinfo *dinfo = device_get_ivars(child);
2582 pcicfgregs *cfg = &dinfo->cfg;
2583 struct resource_list_entry *rle;
2584 int actual, error, i, irqs[32];
2585 uint16_t ctrl;
2586
2587 /* Don't let count == 0 get us into trouble. */
2588 if (*count == 0)
2589 return (EINVAL);
2590
2591 /* If rid 0 is allocated, then fail. */
2592 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
2593 if (rle != NULL && rle->res != NULL)
2594 return (ENXIO);
2595
2596 /* Already have allocated messages? */
2597 if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
2598 return (ENXIO);
2599
2600 /* If MSI is blacklisted for this system, fail. */
2601 if (pci_msi_blacklisted())
2602 return (ENXIO);
2603
2604 /* MSI capability present? */
2605 if (cfg->msi.msi_location == 0 || !pci_do_msi)
2606 return (ENODEV);
2607
2608 if (bootverbose)
2609 device_printf(child,
2610 "attempting to allocate %d MSI vectors (%d supported)\n",
2611 *count, cfg->msi.msi_msgnum);
2612
2613 /* Don't ask for more than the device supports. */
2614 actual = min(*count, cfg->msi.msi_msgnum);
2615
2616 /* Don't ask for more than 32 messages. */
2617 actual = min(actual, 32);
2618
2619 /* MSI requires power of 2 number of messages. */
2620 if (!powerof2(actual))
2621 return (EINVAL);
2622
2623 for (;;) {
2624 /* Try to allocate N messages. */
2625 error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual,
2626 actual, irqs);
2627 if (error == 0)
2628 break;
2629 if (actual == 1)
2630 return (error);
2631
2632 /* Try N / 2. */
2633 actual >>= 1;
2634 }
2635
2636 /*
2637 * We now have N actual messages mapped onto SYS_RES_IRQ
2638 * resources in the irqs[] array, so add new resources
2639 * starting at rid 1.
2640 */
2641 for (i = 0; i < actual; i++)
2642 resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1,
2643 irqs[i], irqs[i], 1);
2644
2645 if (bootverbose) {
2646 if (actual == 1)
2647 device_printf(child, "using IRQ %d for MSI\n", irqs[0]);
2648 else {
2649 int run;
2650
2651 /*
2652 * Be fancy and try to print contiguous runs
2653 * of IRQ values as ranges. 'run' is true if
2654 * we are in a range.
2655 */
2656 device_printf(child, "using IRQs %d", irqs[0]);
2657 run = 0;
2658 for (i = 1; i < actual; i++) {
2659 /* Still in a run? */
2660 if (irqs[i] == irqs[i - 1] + 1) {
2661 run = 1;
2662 continue;
2663 }
2664
2665 /* Finish previous range. */
2666 if (run) {
2667 printf("-%d", irqs[i - 1]);
2668 run = 0;
2669 }
2670
2671 /* Start new range. */
2672 printf(",%d", irqs[i]);
2673 }
2674
2675 /* Unfinished range? */
2676 if (run)
2677 printf("-%d", irqs[actual - 1]);
2678 printf(" for MSI\n");
2679 }
2680 }
2681
2682 /* Update control register with actual count. */
2683 ctrl = cfg->msi.msi_ctrl;
2684 ctrl &= ~PCIM_MSICTRL_MME_MASK;
2685 ctrl |= (ffs(actual) - 1) << 4;
2686 cfg->msi.msi_ctrl = ctrl;
2687 pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2);
2688
2689 /* Update counts of alloc'd messages. */
2690 cfg->msi.msi_alloc = actual;
2691 cfg->msi.msi_handlers = 0;
2692 *count = actual;
2693 return (0);
2694 }
2695
2696 /* Release the MSI messages associated with this device. */
2697 int
pci_release_msi_method(device_t dev,device_t child)2698 pci_release_msi_method(device_t dev, device_t child)
2699 {
2700 struct pci_devinfo *dinfo = device_get_ivars(child);
2701 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2702 struct resource_list_entry *rle;
2703 int error, i, irqs[32];
2704
2705 /* Try MSI-X first. */
2706 error = pci_release_msix(dev, child);
2707 if (error != ENODEV)
2708 return (error);
2709
2710 /* Do we have any messages to release? */
2711 if (msi->msi_alloc == 0)
2712 return (ENODEV);
2713 KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));
2714
2715 /* Make sure none of the resources are allocated. */
2716 if (msi->msi_handlers > 0)
2717 return (EBUSY);
2718 for (i = 0; i < msi->msi_alloc; i++) {
2719 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
2720 KASSERT(rle != NULL, ("missing MSI resource"));
2721 if (rle->res != NULL)
2722 return (EBUSY);
2723 irqs[i] = rle->start;
2724 }
2725
2726 /* Update control register with 0 count. */
2727 KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
2728 ("%s: MSI still enabled", __func__));
2729 msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
2730 pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
2731 msi->msi_ctrl, 2);
2732
2733 /* Release the messages. */
2734 PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs);
2735 for (i = 0; i < msi->msi_alloc; i++)
2736 resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
2737
2738 /* Update alloc count. */
2739 msi->msi_alloc = 0;
2740 msi->msi_addr = 0;
2741 msi->msi_data = 0;
2742 return (0);
2743 }
2744
2745 /*
2746 * Return the max supported MSI messages this device supports.
2747 * Basically, assuming the MD code can alloc messages, this function
2748 * should return the maximum value that pci_alloc_msi() can return.
2749 * Thus, it is subject to the tunables, etc.
2750 */
2751 int
pci_msi_count_method(device_t dev,device_t child)2752 pci_msi_count_method(device_t dev, device_t child)
2753 {
2754 struct pci_devinfo *dinfo = device_get_ivars(child);
2755 struct pcicfg_msi *msi = &dinfo->cfg.msi;
2756
2757 if (pci_do_msi && msi->msi_location != 0)
2758 return (msi->msi_msgnum);
2759 return (0);
2760 }
2761
2762 /* free pcicfgregs structure and all depending data structures */
2763
2764 int
pci_freecfg(struct pci_devinfo * dinfo)2765 pci_freecfg(struct pci_devinfo *dinfo)
2766 {
2767 struct devlist *devlist_head;
2768 struct pci_map *pm, *next;
2769
2770 devlist_head = &pci_devq;
2771
2772 if (dinfo->cfg.vpd.vpd_reg)
2773 vpd_free(&dinfo->cfg.vpd);
2774
2775 STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) {
2776 free(pm, M_DEVBUF);
2777 }
2778 STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links);
2779 free(dinfo, M_DEVBUF);
2780
2781 /* increment the generation count */
2782 pci_generation++;
2783
2784 /* we're losing one device */
2785 pci_numdevs--;
2786 return (0);
2787 }
2788
2789 /*
2790 * PCI power manangement
2791 */
2792 int
pci_set_powerstate_method(device_t dev,device_t child,int state)2793 pci_set_powerstate_method(device_t dev, device_t child, int state)
2794 {
2795 struct pci_devinfo *dinfo = device_get_ivars(child);
2796 pcicfgregs *cfg = &dinfo->cfg;
2797 uint16_t status;
2798 int oldstate, highest, delay;
2799
2800 if (cfg->pp.pp_cap == 0)
2801 return (EOPNOTSUPP);
2802
2803 /*
2804 * Optimize a no state change request away. While it would be OK to
2805 * write to the hardware in theory, some devices have shown odd
2806 * behavior when going from D3 -> D3.
2807 */
2808 oldstate = pci_get_powerstate(child);
2809 if (oldstate == state)
2810 return (0);
2811
2812 /*
2813 * The PCI power management specification states that after a state
2814 * transition between PCI power states, system software must
2815 * guarantee a minimal delay before the function accesses the device.
2816 * Compute the worst case delay that we need to guarantee before we
2817 * access the device. Many devices will be responsive much more
2818 * quickly than this delay, but there are some that don't respond
2819 * instantly to state changes. Transitions to/from D3 state require
2820 * 10ms, while D2 requires 200us, and D0/1 require none. The delay
2821 * is done below with DELAY rather than a sleeper function because
2822 * this function can be called from contexts where we cannot sleep.
2823 */
2824 highest = (oldstate > state) ? oldstate : state;
2825 if (highest == PCI_POWERSTATE_D3)
2826 delay = 10000;
2827 else if (highest == PCI_POWERSTATE_D2)
2828 delay = 200;
2829 else
2830 delay = 0;
2831 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2)
2832 & ~PCIM_PSTAT_DMASK;
2833 switch (state) {
2834 case PCI_POWERSTATE_D0:
2835 status |= PCIM_PSTAT_D0;
2836 break;
2837 case PCI_POWERSTATE_D1:
2838 if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0)
2839 return (EOPNOTSUPP);
2840 status |= PCIM_PSTAT_D1;
2841 break;
2842 case PCI_POWERSTATE_D2:
2843 if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0)
2844 return (EOPNOTSUPP);
2845 status |= PCIM_PSTAT_D2;
2846 break;
2847 case PCI_POWERSTATE_D3:
2848 status |= PCIM_PSTAT_D3;
2849 break;
2850 default:
2851 return (EINVAL);
2852 }
2853
2854 if (bootverbose)
2855 pci_printf(cfg, "Transition from D%d to D%d\n", oldstate,
2856 state);
2857
2858 PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_status, status, 2);
2859 if (delay)
2860 DELAY(delay);
2861 return (0);
2862 }
2863
2864 int
pci_get_powerstate_method(device_t dev,device_t child)2865 pci_get_powerstate_method(device_t dev, device_t child)
2866 {
2867 struct pci_devinfo *dinfo = device_get_ivars(child);
2868 pcicfgregs *cfg = &dinfo->cfg;
2869 uint16_t status;
2870 int result;
2871
2872 if (cfg->pp.pp_cap != 0) {
2873 status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2);
2874 switch (status & PCIM_PSTAT_DMASK) {
2875 case PCIM_PSTAT_D0:
2876 result = PCI_POWERSTATE_D0;
2877 break;
2878 case PCIM_PSTAT_D1:
2879 result = PCI_POWERSTATE_D1;
2880 break;
2881 case PCIM_PSTAT_D2:
2882 result = PCI_POWERSTATE_D2;
2883 break;
2884 case PCIM_PSTAT_D3:
2885 result = PCI_POWERSTATE_D3;
2886 break;
2887 default:
2888 result = PCI_POWERSTATE_UNKNOWN;
2889 break;
2890 }
2891 } else {
2892 /* No support, device is always at D0 */
2893 result = PCI_POWERSTATE_D0;
2894 }
2895 return (result);
2896 }
2897
2898 /*
2899 * Some convenience functions for PCI device drivers.
2900 */
2901
2902 static __inline void
pci_set_command_bit(device_t dev,device_t child,uint16_t bit)2903 pci_set_command_bit(device_t dev, device_t child, uint16_t bit)
2904 {
2905 uint16_t command;
2906
2907 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
2908 command |= bit;
2909 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
2910 }
2911
2912 static __inline void
pci_clear_command_bit(device_t dev,device_t child,uint16_t bit)2913 pci_clear_command_bit(device_t dev, device_t child, uint16_t bit)
2914 {
2915 uint16_t command;
2916
2917 command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2);
2918 command &= ~bit;
2919 PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2);
2920 }
2921
2922 int
pci_enable_busmaster_method(device_t dev,device_t child)2923 pci_enable_busmaster_method(device_t dev, device_t child)
2924 {
2925 pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
2926 return (0);
2927 }
2928
2929 int
pci_disable_busmaster_method(device_t dev,device_t child)2930 pci_disable_busmaster_method(device_t dev, device_t child)
2931 {
2932 pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN);
2933 return (0);
2934 }
2935
2936 int
pci_enable_io_method(device_t dev,device_t child,int space)2937 pci_enable_io_method(device_t dev, device_t child, int space)
2938 {
2939 uint16_t bit;
2940
2941 switch(space) {
2942 case SYS_RES_IOPORT:
2943 bit = PCIM_CMD_PORTEN;
2944 break;
2945 case SYS_RES_MEMORY:
2946 bit = PCIM_CMD_MEMEN;
2947 break;
2948 default:
2949 return (EINVAL);
2950 }
2951 pci_set_command_bit(dev, child, bit);
2952 return (0);
2953 }
2954
2955 int
pci_disable_io_method(device_t dev,device_t child,int space)2956 pci_disable_io_method(device_t dev, device_t child, int space)
2957 {
2958 uint16_t bit;
2959
2960 switch(space) {
2961 case SYS_RES_IOPORT:
2962 bit = PCIM_CMD_PORTEN;
2963 break;
2964 case SYS_RES_MEMORY:
2965 bit = PCIM_CMD_MEMEN;
2966 break;
2967 default:
2968 return (EINVAL);
2969 }
2970 pci_clear_command_bit(dev, child, bit);
2971 return (0);
2972 }
2973
2974 /*
2975 * New style pci driver. Parent device is either a pci-host-bridge or a
2976 * pci-pci-bridge. Both kinds are represented by instances of pcib.
2977 */
2978
2979 void
pci_print_verbose(struct pci_devinfo * dinfo)2980 pci_print_verbose(struct pci_devinfo *dinfo)
2981 {
2982
2983 if (bootverbose) {
2984 pcicfgregs *cfg = &dinfo->cfg;
2985
2986 printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n",
2987 cfg->vendor, cfg->device, cfg->revid);
2988 printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n",
2989 cfg->domain, cfg->bus, cfg->slot, cfg->func);
2990 printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n",
2991 cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype,
2992 cfg->mfdev);
2993 printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n",
2994 cfg->cmdreg, cfg->statreg, cfg->cachelnsz);
2995 printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n",
2996 cfg->lattimer, cfg->lattimer * 30, cfg->mingnt,
2997 cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250);
2998 if (cfg->intpin > 0)
2999 printf("\tintpin=%c, irq=%d\n",
3000 cfg->intpin +'a' -1, cfg->intline);
3001 if (cfg->pp.pp_cap) {
3002 uint16_t status;
3003
3004 status = pci_read_config(cfg->dev, cfg->pp.pp_status, 2);
3005 printf("\tpowerspec %d supports D0%s%s D3 current D%d\n",
3006 cfg->pp.pp_cap & PCIM_PCAP_SPEC,
3007 cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "",
3008 cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "",
3009 status & PCIM_PSTAT_DMASK);
3010 }
3011 if (cfg->msi.msi_location) {
3012 int ctrl;
3013
3014 ctrl = cfg->msi.msi_ctrl;
3015 printf("\tMSI supports %d message%s%s%s\n",
3016 cfg->msi.msi_msgnum,
3017 (cfg->msi.msi_msgnum == 1) ? "" : "s",
3018 (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "",
3019 (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":"");
3020 }
3021 if (cfg->msix.msix_location) {
3022 printf("\tMSI-X supports %d message%s ",
3023 cfg->msix.msix_msgnum,
3024 (cfg->msix.msix_msgnum == 1) ? "" : "s");
3025 if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar)
3026 printf("in map 0x%x\n",
3027 cfg->msix.msix_table_bar);
3028 else
3029 printf("in maps 0x%x and 0x%x\n",
3030 cfg->msix.msix_table_bar,
3031 cfg->msix.msix_pba_bar);
3032 }
3033 }
3034 }
3035
3036 static int
pci_porten(device_t dev)3037 pci_porten(device_t dev)
3038 {
3039 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0;
3040 }
3041
3042 static int
pci_memen(device_t dev)3043 pci_memen(device_t dev)
3044 {
3045 return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0;
3046 }
3047
3048 void
pci_read_bar(device_t dev,int reg,pci_addr_t * mapp,pci_addr_t * testvalp,int * bar64)3049 pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp,
3050 int *bar64)
3051 {
3052 struct pci_devinfo *dinfo;
3053 pci_addr_t map, testval;
3054 int ln2range;
3055 uint16_t cmd;
3056
3057 /*
3058 * The device ROM BAR is special. It is always a 32-bit
3059 * memory BAR. Bit 0 is special and should not be set when
3060 * sizing the BAR.
3061 */
3062 dinfo = device_get_ivars(dev);
3063 if (PCIR_IS_BIOS(&dinfo->cfg, reg)) {
3064 map = pci_read_config(dev, reg, 4);
3065 pci_write_config(dev, reg, 0xfffffffe, 4);
3066 testval = pci_read_config(dev, reg, 4);
3067 pci_write_config(dev, reg, map, 4);
3068 *mapp = map;
3069 *testvalp = testval;
3070 if (bar64 != NULL)
3071 *bar64 = 0;
3072 return;
3073 }
3074
3075 map = pci_read_config(dev, reg, 4);
3076 ln2range = pci_maprange(map);
3077 if (ln2range == 64)
3078 map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3079
3080 /*
3081 * Disable decoding via the command register before
3082 * determining the BAR's length since we will be placing it in
3083 * a weird state.
3084 */
3085 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3086 pci_write_config(dev, PCIR_COMMAND,
3087 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
3088
3089 /*
3090 * Determine the BAR's length by writing all 1's. The bottom
3091 * log_2(size) bits of the BAR will stick as 0 when we read
3092 * the value back.
3093 *
3094 * NB: according to the PCI Local Bus Specification, rev. 3.0:
3095 * "Software writes 0FFFFFFFFh to both registers, reads them back,
3096 * and combines the result into a 64-bit value." (section 6.2.5.1)
3097 *
3098 * Writes to both registers must be performed before attempting to
3099 * read back the size value.
3100 */
3101 testval = 0;
3102 pci_write_config(dev, reg, 0xffffffff, 4);
3103 if (ln2range == 64) {
3104 pci_write_config(dev, reg + 4, 0xffffffff, 4);
3105 testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32;
3106 }
3107 testval |= pci_read_config(dev, reg, 4);
3108
3109 /*
3110 * Restore the original value of the BAR. We may have reprogrammed
3111 * the BAR of the low-level console device and when booting verbose,
3112 * we need the console device addressable.
3113 */
3114 pci_write_config(dev, reg, map, 4);
3115 if (ln2range == 64)
3116 pci_write_config(dev, reg + 4, map >> 32, 4);
3117 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3118
3119 *mapp = map;
3120 *testvalp = testval;
3121 if (bar64 != NULL)
3122 *bar64 = (ln2range == 64);
3123 }
3124
3125 static void
pci_write_bar(device_t dev,struct pci_map * pm,pci_addr_t base)3126 pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base)
3127 {
3128 struct pci_devinfo *dinfo;
3129 int ln2range;
3130
3131 /* The device ROM BAR is always a 32-bit memory BAR. */
3132 dinfo = device_get_ivars(dev);
3133 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3134 ln2range = 32;
3135 else
3136 ln2range = pci_maprange(pm->pm_value);
3137 pci_write_config(dev, pm->pm_reg, base, 4);
3138 if (ln2range == 64)
3139 pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4);
3140 pm->pm_value = pci_read_config(dev, pm->pm_reg, 4);
3141 if (ln2range == 64)
3142 pm->pm_value |= (pci_addr_t)pci_read_config(dev,
3143 pm->pm_reg + 4, 4) << 32;
3144 }
3145
3146 struct pci_map *
pci_find_bar(device_t dev,int reg)3147 pci_find_bar(device_t dev, int reg)
3148 {
3149 struct pci_devinfo *dinfo;
3150 struct pci_map *pm;
3151
3152 dinfo = device_get_ivars(dev);
3153 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3154 if (pm->pm_reg == reg)
3155 return (pm);
3156 }
3157 return (NULL);
3158 }
3159
3160 struct pci_map *
pci_first_bar(device_t dev)3161 pci_first_bar(device_t dev)
3162 {
3163 struct pci_devinfo *dinfo;
3164
3165 dinfo = device_get_ivars(dev);
3166 return (STAILQ_FIRST(&dinfo->cfg.maps));
3167 }
3168
3169 struct pci_map *
pci_next_bar(struct pci_map * pm)3170 pci_next_bar(struct pci_map *pm)
3171 {
3172 return (STAILQ_NEXT(pm, pm_link));
3173 }
3174
3175 int
pci_bar_enabled(device_t dev,struct pci_map * pm)3176 pci_bar_enabled(device_t dev, struct pci_map *pm)
3177 {
3178 struct pci_devinfo *dinfo;
3179 uint16_t cmd;
3180
3181 dinfo = device_get_ivars(dev);
3182 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) &&
3183 !(pm->pm_value & PCIM_BIOS_ENABLE))
3184 return (0);
3185 #ifdef PCI_IOV
3186 if ((dinfo->cfg.flags & PCICFG_VF) != 0) {
3187 struct pcicfg_iov *iov;
3188
3189 iov = dinfo->cfg.iov;
3190 cmd = pci_read_config(iov->iov_pf,
3191 iov->iov_pos + PCIR_SRIOV_CTL, 2);
3192 return ((cmd & PCIM_SRIOV_VF_MSE) != 0);
3193 }
3194 #endif
3195 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3196 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value))
3197 return ((cmd & PCIM_CMD_MEMEN) != 0);
3198 else
3199 return ((cmd & PCIM_CMD_PORTEN) != 0);
3200 }
3201
3202 struct pci_map *
pci_add_bar(device_t dev,int reg,pci_addr_t value,pci_addr_t size)3203 pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size)
3204 {
3205 struct pci_devinfo *dinfo;
3206 struct pci_map *pm, *prev;
3207
3208 dinfo = device_get_ivars(dev);
3209 pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO);
3210 pm->pm_reg = reg;
3211 pm->pm_value = value;
3212 pm->pm_size = size;
3213 STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) {
3214 KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x",
3215 reg));
3216 if (STAILQ_NEXT(prev, pm_link) == NULL ||
3217 STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg)
3218 break;
3219 }
3220 if (prev != NULL)
3221 STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link);
3222 else
3223 STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link);
3224 return (pm);
3225 }
3226
3227 static void
pci_restore_bars(device_t dev)3228 pci_restore_bars(device_t dev)
3229 {
3230 struct pci_devinfo *dinfo;
3231 struct pci_map *pm;
3232 int ln2range;
3233
3234 dinfo = device_get_ivars(dev);
3235 STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) {
3236 if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg))
3237 ln2range = 32;
3238 else
3239 ln2range = pci_maprange(pm->pm_value);
3240 pci_write_config(dev, pm->pm_reg, pm->pm_value, 4);
3241 if (ln2range == 64)
3242 pci_write_config(dev, pm->pm_reg + 4,
3243 pm->pm_value >> 32, 4);
3244 }
3245 }
3246
3247 /*
3248 * Add a resource based on a pci map register. Return 1 if the map
3249 * register is a 32bit map register or 2 if it is a 64bit register.
3250 */
3251 static int
pci_add_map(device_t bus,device_t dev,int reg,struct resource_list * rl,int force,int prefetch)3252 pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
3253 int force, int prefetch)
3254 {
3255 struct pci_map *pm;
3256 pci_addr_t base, map, testval;
3257 pci_addr_t start, end, count;
3258 int barlen, basezero, flags, maprange, mapsize, type;
3259 uint16_t cmd;
3260 struct resource *res;
3261
3262 /*
3263 * The BAR may already exist if the device is a CardBus card
3264 * whose CIS is stored in this BAR.
3265 */
3266 pm = pci_find_bar(dev, reg);
3267 if (pm != NULL) {
3268 maprange = pci_maprange(pm->pm_value);
3269 barlen = maprange == 64 ? 2 : 1;
3270 return (barlen);
3271 }
3272
3273 pci_read_bar(dev, reg, &map, &testval, NULL);
3274 if (PCI_BAR_MEM(map)) {
3275 type = SYS_RES_MEMORY;
3276 if (map & PCIM_BAR_MEM_PREFETCH)
3277 prefetch = 1;
3278 } else
3279 type = SYS_RES_IOPORT;
3280 mapsize = pci_mapsize(testval);
3281 base = pci_mapbase(map);
3282 #ifdef __PCI_BAR_ZERO_VALID
3283 basezero = 0;
3284 #else
3285 basezero = base == 0;
3286 #endif
3287 maprange = pci_maprange(map);
3288 barlen = maprange == 64 ? 2 : 1;
3289
3290 /*
3291 * For I/O registers, if bottom bit is set, and the next bit up
3292 * isn't clear, we know we have a BAR that doesn't conform to the
3293 * spec, so ignore it. Also, sanity check the size of the data
3294 * areas to the type of memory involved. Memory must be at least
3295 * 16 bytes in size, while I/O ranges must be at least 4.
3296 */
3297 if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0)
3298 return (barlen);
3299 if ((type == SYS_RES_MEMORY && mapsize < 4) ||
3300 (type == SYS_RES_IOPORT && mapsize < 2))
3301 return (barlen);
3302
3303 /* Save a record of this BAR. */
3304 pm = pci_add_bar(dev, reg, map, mapsize);
3305 if (bootverbose) {
3306 printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d",
3307 reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize);
3308 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3309 printf(", port disabled\n");
3310 else if (type == SYS_RES_MEMORY && !pci_memen(dev))
3311 printf(", memory disabled\n");
3312 else
3313 printf(", enabled\n");
3314 }
3315
3316 /*
3317 * If base is 0, then we have problems if this architecture does
3318 * not allow that. It is best to ignore such entries for the
3319 * moment. These will be allocated later if the driver specifically
3320 * requests them. However, some removable buses look better when
3321 * all resources are allocated, so allow '0' to be overridden.
3322 *
3323 * Similarly treat maps whose values is the same as the test value
3324 * read back. These maps have had all f's written to them by the
3325 * BIOS in an attempt to disable the resources.
3326 */
3327 if (!force && (basezero || map == testval))
3328 return (barlen);
3329 if ((u_long)base != base) {
3330 device_printf(bus,
3331 "pci%d:%d:%d:%d bar %#x too many address bits",
3332 pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev),
3333 pci_get_function(dev), reg);
3334 return (barlen);
3335 }
3336
3337 /*
3338 * This code theoretically does the right thing, but has
3339 * undesirable side effects in some cases where peripherals
3340 * respond oddly to having these bits enabled. Let the user
3341 * be able to turn them off (since pci_enable_io_modes is 1 by
3342 * default).
3343 */
3344 if (pci_enable_io_modes) {
3345 /* Turn on resources that have been left off by a lazy BIOS */
3346 if (type == SYS_RES_IOPORT && !pci_porten(dev)) {
3347 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3348 cmd |= PCIM_CMD_PORTEN;
3349 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3350 }
3351 if (type == SYS_RES_MEMORY && !pci_memen(dev)) {
3352 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
3353 cmd |= PCIM_CMD_MEMEN;
3354 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
3355 }
3356 } else {
3357 if (type == SYS_RES_IOPORT && !pci_porten(dev))
3358 return (barlen);
3359 if (type == SYS_RES_MEMORY && !pci_memen(dev))
3360 return (barlen);
3361 }
3362
3363 count = (pci_addr_t)1 << mapsize;
3364 flags = RF_ALIGNMENT_LOG2(mapsize);
3365 if (prefetch)
3366 flags |= RF_PREFETCHABLE;
3367 if (basezero || base == pci_mapbase(testval) || pci_clear_bars) {
3368 start = 0; /* Let the parent decide. */
3369 end = ~0;
3370 } else {
3371 start = base;
3372 end = base + count - 1;
3373 }
3374 resource_list_add(rl, type, reg, start, end, count);
3375
3376 /*
3377 * Try to allocate the resource for this BAR from our parent
3378 * so that this resource range is already reserved. The
3379 * driver for this device will later inherit this resource in
3380 * pci_alloc_resource().
3381 */
3382 res = resource_list_reserve(rl, bus, dev, type, ®, start, end, count,
3383 flags);
3384 if ((pci_do_realloc_bars
3385 || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR))
3386 && res == NULL && (start != 0 || end != ~0)) {
3387 /*
3388 * If the allocation fails, try to allocate a resource for
3389 * this BAR using any available range. The firmware felt
3390 * it was important enough to assign a resource, so don't
3391 * disable decoding if we can help it.
3392 */
3393 resource_list_delete(rl, type, reg);
3394 resource_list_add(rl, type, reg, 0, ~0, count);
3395 res = resource_list_reserve(rl, bus, dev, type, ®, 0, ~0,
3396 count, flags);
3397 }
3398 if (res == NULL) {
3399 /*
3400 * If the allocation fails, delete the resource list entry
3401 * and disable decoding for this device.
3402 *
3403 * If the driver requests this resource in the future,
3404 * pci_reserve_map() will try to allocate a fresh
3405 * resource range.
3406 */
3407 resource_list_delete(rl, type, reg);
3408 pci_disable_io(dev, type);
3409 if (bootverbose)
3410 device_printf(bus,
3411 "pci%d:%d:%d:%d bar %#x failed to allocate\n",
3412 pci_get_domain(dev), pci_get_bus(dev),
3413 pci_get_slot(dev), pci_get_function(dev), reg);
3414 } else {
3415 start = rman_get_start(res);
3416 pci_write_bar(dev, pm, start);
3417 }
3418 return (barlen);
3419 }
3420
3421 /*
3422 * For ATA devices we need to decide early what addressing mode to use.
3423 * Legacy demands that the primary and secondary ATA ports sits on the
3424 * same addresses that old ISA hardware did. This dictates that we use
3425 * those addresses and ignore the BAR's if we cannot set PCI native
3426 * addressing mode.
3427 */
3428 static void
pci_ata_maps(device_t bus,device_t dev,struct resource_list * rl,int force,uint32_t prefetchmask)3429 pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
3430 uint32_t prefetchmask)
3431 {
3432 int rid, type, progif;
3433 #if 0
3434 /* if this device supports PCI native addressing use it */
3435 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3436 if ((progif & 0x8a) == 0x8a) {
3437 if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) &&
3438 pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) {
3439 printf("Trying ATA native PCI addressing mode\n");
3440 pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1);
3441 }
3442 }
3443 #endif
3444 progif = pci_read_config(dev, PCIR_PROGIF, 1);
3445 type = SYS_RES_IOPORT;
3446 if (progif & PCIP_STORAGE_IDE_MODEPRIM) {
3447 pci_add_map(bus, dev, PCIR_BAR(0), rl, force,
3448 prefetchmask & (1 << 0));
3449 pci_add_map(bus, dev, PCIR_BAR(1), rl, force,
3450 prefetchmask & (1 << 1));
3451 } else {
3452 rid = PCIR_BAR(0);
3453 resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
3454 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0,
3455 0x1f7, 8, 0);
3456 rid = PCIR_BAR(1);
3457 resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
3458 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6,
3459 0x3f6, 1, 0);
3460 }
3461 if (progif & PCIP_STORAGE_IDE_MODESEC) {
3462 pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
3463 prefetchmask & (1 << 2));
3464 pci_add_map(bus, dev, PCIR_BAR(3), rl, force,
3465 prefetchmask & (1 << 3));
3466 } else {
3467 rid = PCIR_BAR(2);
3468 resource_list_add(rl, type, rid, 0x170, 0x177, 8);
3469 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x170,
3470 0x177, 8, 0);
3471 rid = PCIR_BAR(3);
3472 resource_list_add(rl, type, rid, 0x376, 0x376, 1);
3473 (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x376,
3474 0x376, 1, 0);
3475 }
3476 pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
3477 prefetchmask & (1 << 4));
3478 pci_add_map(bus, dev, PCIR_BAR(5), rl, force,
3479 prefetchmask & (1 << 5));
3480 }
3481
3482 static void
pci_assign_interrupt(device_t bus,device_t dev,int force_route)3483 pci_assign_interrupt(device_t bus, device_t dev, int force_route)
3484 {
3485 struct pci_devinfo *dinfo = device_get_ivars(dev);
3486 pcicfgregs *cfg = &dinfo->cfg;
3487 char tunable_name[64];
3488 int irq;
3489
3490 /* Has to have an intpin to have an interrupt. */
3491 if (cfg->intpin == 0)
3492 return;
3493
3494 /* Let the user override the IRQ with a tunable. */
3495 irq = PCI_INVALID_IRQ;
3496 snprintf(tunable_name, sizeof(tunable_name),
3497 "hw.pci%d.%d.%d.INT%c.irq",
3498 cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1);
3499 if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0))
3500 irq = PCI_INVALID_IRQ;
3501
3502 /*
3503 * If we didn't get an IRQ via the tunable, then we either use the
3504 * IRQ value in the intline register or we ask the bus to route an
3505 * interrupt for us. If force_route is true, then we only use the
3506 * value in the intline register if the bus was unable to assign an
3507 * IRQ.
3508 */
3509 if (!PCI_INTERRUPT_VALID(irq)) {
3510 if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route)
3511 irq = PCI_ASSIGN_INTERRUPT(bus, dev);
3512 if (!PCI_INTERRUPT_VALID(irq))
3513 irq = cfg->intline;
3514 }
3515
3516 /* If after all that we don't have an IRQ, just bail. */
3517 if (!PCI_INTERRUPT_VALID(irq))
3518 return;
3519
3520 /* Update the config register if it changed. */
3521 if (irq != cfg->intline) {
3522 cfg->intline = irq;
3523 pci_write_config(dev, PCIR_INTLINE, irq, 1);
3524 }
3525
3526 /* Add this IRQ as rid 0 interrupt resource. */
3527 resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1);
3528 }
3529
3530 /* Perform early OHCI takeover from SMM. */
3531 static void
ohci_early_takeover(device_t self)3532 ohci_early_takeover(device_t self)
3533 {
3534 struct resource *res;
3535 uint32_t ctl;
3536 int rid;
3537 int i;
3538
3539 rid = PCIR_BAR(0);
3540 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3541 if (res == NULL)
3542 return;
3543
3544 ctl = bus_read_4(res, OHCI_CONTROL);
3545 if (ctl & OHCI_IR) {
3546 if (bootverbose)
3547 printf("ohci early: "
3548 "SMM active, request owner change\n");
3549 bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR);
3550 for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) {
3551 DELAY(1000);
3552 ctl = bus_read_4(res, OHCI_CONTROL);
3553 }
3554 if (ctl & OHCI_IR) {
3555 if (bootverbose)
3556 printf("ohci early: "
3557 "SMM does not respond, resetting\n");
3558 bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET);
3559 }
3560 /* Disable interrupts */
3561 bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS);
3562 }
3563
3564 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3565 }
3566
3567 /* Perform early UHCI takeover from SMM. */
3568 static void
uhci_early_takeover(device_t self)3569 uhci_early_takeover(device_t self)
3570 {
3571 struct resource *res;
3572 int rid;
3573
3574 /*
3575 * Set the PIRQD enable bit and switch off all the others. We don't
3576 * want legacy support to interfere with us XXX Does this also mean
3577 * that the BIOS won't touch the keyboard anymore if it is connected
3578 * to the ports of the root hub?
3579 */
3580 pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2);
3581
3582 /* Disable interrupts */
3583 rid = PCI_UHCI_BASE_REG;
3584 res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE);
3585 if (res != NULL) {
3586 bus_write_2(res, UHCI_INTR, 0);
3587 bus_release_resource(self, SYS_RES_IOPORT, rid, res);
3588 }
3589 }
3590
3591 /* Perform early EHCI takeover from SMM. */
3592 static void
ehci_early_takeover(device_t self)3593 ehci_early_takeover(device_t self)
3594 {
3595 struct resource *res;
3596 uint32_t cparams;
3597 uint32_t eec;
3598 uint8_t eecp;
3599 uint8_t bios_sem;
3600 uint8_t offs;
3601 int rid;
3602 int i;
3603
3604 rid = PCIR_BAR(0);
3605 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3606 if (res == NULL)
3607 return;
3608
3609 cparams = bus_read_4(res, EHCI_HCCPARAMS);
3610
3611 /* Synchronise with the BIOS if it owns the controller. */
3612 for (eecp = EHCI_HCC_EECP(cparams); eecp != 0;
3613 eecp = EHCI_EECP_NEXT(eec)) {
3614 eec = pci_read_config(self, eecp, 4);
3615 if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) {
3616 continue;
3617 }
3618 bios_sem = pci_read_config(self, eecp +
3619 EHCI_LEGSUP_BIOS_SEM, 1);
3620 if (bios_sem == 0) {
3621 continue;
3622 }
3623 if (bootverbose)
3624 printf("ehci early: "
3625 "SMM active, request owner change\n");
3626
3627 pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1);
3628
3629 for (i = 0; (i < 100) && (bios_sem != 0); i++) {
3630 DELAY(1000);
3631 bios_sem = pci_read_config(self, eecp +
3632 EHCI_LEGSUP_BIOS_SEM, 1);
3633 }
3634
3635 if (bios_sem != 0) {
3636 if (bootverbose)
3637 printf("ehci early: "
3638 "SMM does not respond\n");
3639 }
3640 /* Disable interrupts */
3641 offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION));
3642 bus_write_4(res, offs + EHCI_USBINTR, 0);
3643 }
3644 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3645 }
3646
3647 /* Perform early XHCI takeover from SMM. */
3648 static void
xhci_early_takeover(device_t self)3649 xhci_early_takeover(device_t self)
3650 {
3651 struct resource *res;
3652 uint32_t cparams;
3653 uint32_t eec;
3654 uint8_t eecp;
3655 uint8_t bios_sem;
3656 uint8_t offs;
3657 int rid;
3658 int i;
3659
3660 rid = PCIR_BAR(0);
3661 res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE);
3662 if (res == NULL)
3663 return;
3664
3665 cparams = bus_read_4(res, XHCI_HCSPARAMS0);
3666
3667 eec = -1;
3668
3669 /* Synchronise with the BIOS if it owns the controller. */
3670 for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec);
3671 eecp += XHCI_XECP_NEXT(eec) << 2) {
3672 eec = bus_read_4(res, eecp);
3673
3674 if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY)
3675 continue;
3676
3677 bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM);
3678 if (bios_sem == 0)
3679 continue;
3680
3681 if (bootverbose)
3682 printf("xhci early: "
3683 "SMM active, request owner change\n");
3684
3685 bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1);
3686
3687 /* wait a maximum of 5 second */
3688
3689 for (i = 0; (i < 5000) && (bios_sem != 0); i++) {
3690 DELAY(1000);
3691 bios_sem = bus_read_1(res, eecp +
3692 XHCI_XECP_BIOS_SEM);
3693 }
3694
3695 if (bios_sem != 0) {
3696 if (bootverbose)
3697 printf("xhci early: "
3698 "SMM does not respond\n");
3699 }
3700
3701 /* Disable interrupts */
3702 offs = bus_read_1(res, XHCI_CAPLENGTH);
3703 bus_write_4(res, offs + XHCI_USBCMD, 0);
3704 bus_read_4(res, offs + XHCI_USBSTS);
3705 }
3706 bus_release_resource(self, SYS_RES_MEMORY, rid, res);
3707 }
3708
3709 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
3710 static void
pci_reserve_secbus(device_t bus,device_t dev,pcicfgregs * cfg,struct resource_list * rl)3711 pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg,
3712 struct resource_list *rl)
3713 {
3714 struct resource *res;
3715 char *cp;
3716 rman_res_t start, end, count;
3717 int rid, sec_bus, sec_reg, sub_bus, sub_reg, sup_bus;
3718
3719 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3720 case PCIM_HDRTYPE_BRIDGE:
3721 sec_reg = PCIR_SECBUS_1;
3722 sub_reg = PCIR_SUBBUS_1;
3723 break;
3724 case PCIM_HDRTYPE_CARDBUS:
3725 sec_reg = PCIR_SECBUS_2;
3726 sub_reg = PCIR_SUBBUS_2;
3727 break;
3728 default:
3729 return;
3730 }
3731
3732 /*
3733 * If the existing bus range is valid, attempt to reserve it
3734 * from our parent. If this fails for any reason, clear the
3735 * secbus and subbus registers.
3736 *
3737 * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus?
3738 * This would at least preserve the existing sec_bus if it is
3739 * valid.
3740 */
3741 sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1);
3742 sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1);
3743
3744 /* Quirk handling. */
3745 switch (pci_get_devid(dev)) {
3746 case 0x12258086: /* Intel 82454KX/GX (Orion) */
3747 sup_bus = pci_read_config(dev, 0x41, 1);
3748 if (sup_bus != 0xff) {
3749 sec_bus = sup_bus + 1;
3750 sub_bus = sup_bus + 1;
3751 PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1);
3752 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3753 }
3754 break;
3755
3756 case 0x00dd10de:
3757 /* Compaq R3000 BIOS sets wrong subordinate bus number. */
3758 if ((cp = kern_getenv("smbios.planar.maker")) == NULL)
3759 break;
3760 if (strncmp(cp, "Compal", 6) != 0) {
3761 freeenv(cp);
3762 break;
3763 }
3764 freeenv(cp);
3765 if ((cp = kern_getenv("smbios.planar.product")) == NULL)
3766 break;
3767 if (strncmp(cp, "08A0", 4) != 0) {
3768 freeenv(cp);
3769 break;
3770 }
3771 freeenv(cp);
3772 if (sub_bus < 0xa) {
3773 sub_bus = 0xa;
3774 PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1);
3775 }
3776 break;
3777 }
3778
3779 if (bootverbose)
3780 printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus);
3781 if (sec_bus > 0 && sub_bus >= sec_bus) {
3782 start = sec_bus;
3783 end = sub_bus;
3784 count = end - start + 1;
3785
3786 resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count);
3787
3788 /*
3789 * If requested, clear secondary bus registers in
3790 * bridge devices to force a complete renumbering
3791 * rather than reserving the existing range. However,
3792 * preserve the existing size.
3793 */
3794 if (pci_clear_buses)
3795 goto clear;
3796
3797 rid = 0;
3798 res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, &rid,
3799 start, end, count, 0);
3800 if (res != NULL)
3801 return;
3802
3803 if (bootverbose)
3804 device_printf(bus,
3805 "pci%d:%d:%d:%d secbus failed to allocate\n",
3806 pci_get_domain(dev), pci_get_bus(dev),
3807 pci_get_slot(dev), pci_get_function(dev));
3808 }
3809
3810 clear:
3811 PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1);
3812 PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1);
3813 }
3814
3815 static struct resource *
pci_alloc_secbus(device_t dev,device_t child,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)3816 pci_alloc_secbus(device_t dev, device_t child, int *rid, rman_res_t start,
3817 rman_res_t end, rman_res_t count, u_int flags)
3818 {
3819 struct pci_devinfo *dinfo;
3820 pcicfgregs *cfg;
3821 struct resource_list *rl;
3822 struct resource *res;
3823 int sec_reg, sub_reg;
3824
3825 dinfo = device_get_ivars(child);
3826 cfg = &dinfo->cfg;
3827 rl = &dinfo->resources;
3828 switch (cfg->hdrtype & PCIM_HDRTYPE) {
3829 case PCIM_HDRTYPE_BRIDGE:
3830 sec_reg = PCIR_SECBUS_1;
3831 sub_reg = PCIR_SUBBUS_1;
3832 break;
3833 case PCIM_HDRTYPE_CARDBUS:
3834 sec_reg = PCIR_SECBUS_2;
3835 sub_reg = PCIR_SUBBUS_2;
3836 break;
3837 default:
3838 return (NULL);
3839 }
3840
3841 if (*rid != 0)
3842 return (NULL);
3843
3844 if (resource_list_find(rl, PCI_RES_BUS, *rid) == NULL)
3845 resource_list_add(rl, PCI_RES_BUS, *rid, start, end, count);
3846 if (!resource_list_reserved(rl, PCI_RES_BUS, *rid)) {
3847 res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid,
3848 start, end, count, flags & ~RF_ACTIVE);
3849 if (res == NULL) {
3850 resource_list_delete(rl, PCI_RES_BUS, *rid);
3851 device_printf(child, "allocating %ju bus%s failed\n",
3852 count, count == 1 ? "" : "es");
3853 return (NULL);
3854 }
3855 if (bootverbose)
3856 device_printf(child,
3857 "Lazy allocation of %ju bus%s at %ju\n", count,
3858 count == 1 ? "" : "es", rman_get_start(res));
3859 PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1);
3860 PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1);
3861 }
3862 return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start,
3863 end, count, flags));
3864 }
3865 #endif
3866
3867 static int
pci_ea_bei_to_rid(device_t dev,int bei)3868 pci_ea_bei_to_rid(device_t dev, int bei)
3869 {
3870 #ifdef PCI_IOV
3871 struct pci_devinfo *dinfo;
3872 int iov_pos;
3873 struct pcicfg_iov *iov;
3874
3875 dinfo = device_get_ivars(dev);
3876 iov = dinfo->cfg.iov;
3877 if (iov != NULL)
3878 iov_pos = iov->iov_pos;
3879 else
3880 iov_pos = 0;
3881 #endif
3882
3883 /* Check if matches BAR */
3884 if ((bei >= PCIM_EA_BEI_BAR_0) &&
3885 (bei <= PCIM_EA_BEI_BAR_5))
3886 return (PCIR_BAR(bei));
3887
3888 /* Check ROM */
3889 if (bei == PCIM_EA_BEI_ROM)
3890 return (PCIR_BIOS);
3891
3892 #ifdef PCI_IOV
3893 /* Check if matches VF_BAR */
3894 if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) &&
3895 (bei <= PCIM_EA_BEI_VF_BAR_5))
3896 return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) +
3897 iov_pos);
3898 #endif
3899
3900 return (-1);
3901 }
3902
3903 int
pci_ea_is_enabled(device_t dev,int rid)3904 pci_ea_is_enabled(device_t dev, int rid)
3905 {
3906 struct pci_ea_entry *ea;
3907 struct pci_devinfo *dinfo;
3908
3909 dinfo = device_get_ivars(dev);
3910
3911 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
3912 if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid)
3913 return ((ea->eae_flags & PCIM_EA_ENABLE) > 0);
3914 }
3915
3916 return (0);
3917 }
3918
3919 void
pci_add_resources_ea(device_t bus,device_t dev,int alloc_iov)3920 pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov)
3921 {
3922 struct pci_ea_entry *ea;
3923 struct pci_devinfo *dinfo;
3924 pci_addr_t start, end, count;
3925 struct resource_list *rl;
3926 int type, flags, rid;
3927 struct resource *res;
3928 uint32_t tmp;
3929 #ifdef PCI_IOV
3930 struct pcicfg_iov *iov;
3931 #endif
3932
3933 dinfo = device_get_ivars(dev);
3934 rl = &dinfo->resources;
3935 flags = 0;
3936
3937 #ifdef PCI_IOV
3938 iov = dinfo->cfg.iov;
3939 #endif
3940
3941 if (dinfo->cfg.ea.ea_location == 0)
3942 return;
3943
3944 STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) {
3945 /*
3946 * TODO: Ignore EA-BAR if is not enabled.
3947 * Currently the EA implementation supports
3948 * only situation, where EA structure contains
3949 * predefined entries. In case they are not enabled
3950 * leave them unallocated and proceed with
3951 * a legacy-BAR mechanism.
3952 */
3953 if ((ea->eae_flags & PCIM_EA_ENABLE) == 0)
3954 continue;
3955
3956 switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) {
3957 case PCIM_EA_P_MEM_PREFETCH:
3958 case PCIM_EA_P_VF_MEM_PREFETCH:
3959 flags = RF_PREFETCHABLE;
3960 /* FALLTHROUGH */
3961 case PCIM_EA_P_VF_MEM:
3962 case PCIM_EA_P_MEM:
3963 type = SYS_RES_MEMORY;
3964 break;
3965 case PCIM_EA_P_IO:
3966 type = SYS_RES_IOPORT;
3967 break;
3968 default:
3969 continue;
3970 }
3971
3972 if (alloc_iov != 0) {
3973 #ifdef PCI_IOV
3974 /* Allocating IOV, confirm BEI matches */
3975 if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) ||
3976 (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5))
3977 continue;
3978 #else
3979 continue;
3980 #endif
3981 } else {
3982 /* Allocating BAR, confirm BEI matches */
3983 if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) ||
3984 (ea->eae_bei > PCIM_EA_BEI_BAR_5)) &&
3985 (ea->eae_bei != PCIM_EA_BEI_ROM))
3986 continue;
3987 }
3988
3989 rid = pci_ea_bei_to_rid(dev, ea->eae_bei);
3990 if (rid < 0)
3991 continue;
3992
3993 /* Skip resources already allocated by EA */
3994 if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) ||
3995 (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL))
3996 continue;
3997
3998 start = ea->eae_base;
3999 count = ea->eae_max_offset + 1;
4000 #ifdef PCI_IOV
4001 if (iov != NULL)
4002 count = count * iov->iov_num_vfs;
4003 #endif
4004 end = start + count - 1;
4005 if (count == 0)
4006 continue;
4007
4008 resource_list_add(rl, type, rid, start, end, count);
4009 res = resource_list_reserve(rl, bus, dev, type, &rid, start, end, count,
4010 flags);
4011 if (res == NULL) {
4012 resource_list_delete(rl, type, rid);
4013
4014 /*
4015 * Failed to allocate using EA, disable entry.
4016 * Another attempt to allocation will be performed
4017 * further, but this time using legacy BAR registers
4018 */
4019 tmp = pci_read_config(dev, ea->eae_cfg_offset, 4);
4020 tmp &= ~PCIM_EA_ENABLE;
4021 pci_write_config(dev, ea->eae_cfg_offset, tmp, 4);
4022
4023 /*
4024 * Disabling entry might fail in case it is hardwired.
4025 * Read flags again to match current status.
4026 */
4027 ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4);
4028
4029 continue;
4030 }
4031
4032 /* As per specification, fill BAR with zeros */
4033 pci_write_config(dev, rid, 0, 4);
4034 }
4035 }
4036
4037 void
pci_add_resources(device_t bus,device_t dev,int force,uint32_t prefetchmask)4038 pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask)
4039 {
4040 struct pci_devinfo *dinfo;
4041 pcicfgregs *cfg;
4042 struct resource_list *rl;
4043 const struct pci_quirk *q;
4044 uint32_t devid;
4045 int i;
4046
4047 dinfo = device_get_ivars(dev);
4048 cfg = &dinfo->cfg;
4049 rl = &dinfo->resources;
4050 devid = (cfg->device << 16) | cfg->vendor;
4051
4052 /* Allocate resources using Enhanced Allocation */
4053 pci_add_resources_ea(bus, dev, 0);
4054
4055 /* ATA devices needs special map treatment */
4056 if ((pci_get_class(dev) == PCIC_STORAGE) &&
4057 (pci_get_subclass(dev) == PCIS_STORAGE_IDE) &&
4058 ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) ||
4059 (!pci_read_config(dev, PCIR_BAR(0), 4) &&
4060 !pci_read_config(dev, PCIR_BAR(2), 4))) )
4061 pci_ata_maps(bus, dev, rl, force, prefetchmask);
4062 else
4063 for (i = 0; i < cfg->nummaps;) {
4064 /* Skip resources already managed by EA */
4065 if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) ||
4066 (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) ||
4067 pci_ea_is_enabled(dev, PCIR_BAR(i))) {
4068 i++;
4069 continue;
4070 }
4071
4072 /*
4073 * Skip quirked resources.
4074 */
4075 for (q = &pci_quirks[0]; q->devid != 0; q++)
4076 if (q->devid == devid &&
4077 q->type == PCI_QUIRK_UNMAP_REG &&
4078 q->arg1 == PCIR_BAR(i))
4079 break;
4080 if (q->devid != 0) {
4081 i++;
4082 continue;
4083 }
4084 i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force,
4085 prefetchmask & (1 << i));
4086 }
4087
4088 /*
4089 * Add additional, quirked resources.
4090 */
4091 for (q = &pci_quirks[0]; q->devid != 0; q++)
4092 if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG)
4093 pci_add_map(bus, dev, q->arg1, rl, force, 0);
4094
4095 if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline)) {
4096 #ifdef __PCI_REROUTE_INTERRUPT
4097 /*
4098 * Try to re-route interrupts. Sometimes the BIOS or
4099 * firmware may leave bogus values in these registers.
4100 * If the re-route fails, then just stick with what we
4101 * have.
4102 */
4103 pci_assign_interrupt(bus, dev, 1);
4104 #else
4105 pci_assign_interrupt(bus, dev, 0);
4106 #endif
4107 }
4108
4109 if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS &&
4110 pci_get_subclass(dev) == PCIS_SERIALBUS_USB) {
4111 if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI)
4112 xhci_early_takeover(dev);
4113 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI)
4114 ehci_early_takeover(dev);
4115 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI)
4116 ohci_early_takeover(dev);
4117 else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI)
4118 uhci_early_takeover(dev);
4119 }
4120
4121 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
4122 /*
4123 * Reserve resources for secondary bus ranges behind bridge
4124 * devices.
4125 */
4126 pci_reserve_secbus(bus, dev, cfg, rl);
4127 #endif
4128 }
4129
4130 static struct pci_devinfo *
pci_identify_function(device_t pcib,device_t dev,int domain,int busno,int slot,int func)4131 pci_identify_function(device_t pcib, device_t dev, int domain, int busno,
4132 int slot, int func)
4133 {
4134 struct pci_devinfo *dinfo;
4135
4136 dinfo = pci_read_device(pcib, dev, domain, busno, slot, func);
4137 if (dinfo != NULL)
4138 pci_add_child(dev, dinfo);
4139
4140 return (dinfo);
4141 }
4142
4143 void
pci_add_children(device_t dev,int domain,int busno)4144 pci_add_children(device_t dev, int domain, int busno)
4145 {
4146 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4147 device_t pcib = device_get_parent(dev);
4148 struct pci_devinfo *dinfo;
4149 int maxslots;
4150 int s, f, pcifunchigh;
4151 uint8_t hdrtype;
4152 int first_func;
4153
4154 /*
4155 * Try to detect a device at slot 0, function 0. If it exists, try to
4156 * enable ARI. We must enable ARI before detecting the rest of the
4157 * functions on this bus as ARI changes the set of slots and functions
4158 * that are legal on this bus.
4159 */
4160 dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0);
4161 if (dinfo != NULL && pci_enable_ari)
4162 PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev);
4163
4164 /*
4165 * Start looking for new devices on slot 0 at function 1 because we
4166 * just identified the device at slot 0, function 0.
4167 */
4168 first_func = 1;
4169
4170 maxslots = PCIB_MAXSLOTS(pcib);
4171 for (s = 0; s <= maxslots; s++, first_func = 0) {
4172 pcifunchigh = 0;
4173 f = 0;
4174 DELAY(1);
4175
4176 /* If function 0 is not present, skip to the next slot. */
4177 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4178 continue;
4179 hdrtype = REG(PCIR_HDRTYPE, 1);
4180 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4181 continue;
4182 if (hdrtype & PCIM_MFDEV)
4183 pcifunchigh = PCIB_MAXFUNCS(pcib);
4184 for (f = first_func; f <= pcifunchigh; f++)
4185 pci_identify_function(pcib, dev, domain, busno, s, f);
4186 }
4187 #undef REG
4188 }
4189
4190 int
pci_rescan_method(device_t dev)4191 pci_rescan_method(device_t dev)
4192 {
4193 #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w)
4194 device_t pcib = device_get_parent(dev);
4195 device_t child, *devlist, *unchanged;
4196 int devcount, error, i, j, maxslots, oldcount;
4197 int busno, domain, s, f, pcifunchigh;
4198 uint8_t hdrtype;
4199
4200 /* No need to check for ARI on a rescan. */
4201 error = device_get_children(dev, &devlist, &devcount);
4202 if (error)
4203 return (error);
4204 if (devcount != 0) {
4205 unchanged = malloc(devcount * sizeof(device_t), M_TEMP,
4206 M_NOWAIT | M_ZERO);
4207 if (unchanged == NULL) {
4208 free(devlist, M_TEMP);
4209 return (ENOMEM);
4210 }
4211 } else
4212 unchanged = NULL;
4213
4214 domain = pcib_get_domain(dev);
4215 busno = pcib_get_bus(dev);
4216 maxslots = PCIB_MAXSLOTS(pcib);
4217 for (s = 0; s <= maxslots; s++) {
4218 /* If function 0 is not present, skip to the next slot. */
4219 f = 0;
4220 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4221 continue;
4222 pcifunchigh = 0;
4223 hdrtype = REG(PCIR_HDRTYPE, 1);
4224 if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE)
4225 continue;
4226 if (hdrtype & PCIM_MFDEV)
4227 pcifunchigh = PCIB_MAXFUNCS(pcib);
4228 for (f = 0; f <= pcifunchigh; f++) {
4229 if (REG(PCIR_VENDOR, 2) == PCIV_INVALID)
4230 continue;
4231
4232 /*
4233 * Found a valid function. Check if a
4234 * device_t for this device already exists.
4235 */
4236 for (i = 0; i < devcount; i++) {
4237 child = devlist[i];
4238 if (child == NULL)
4239 continue;
4240 if (pci_get_slot(child) == s &&
4241 pci_get_function(child) == f) {
4242 unchanged[i] = child;
4243 goto next_func;
4244 }
4245 }
4246
4247 pci_identify_function(pcib, dev, domain, busno, s, f);
4248 next_func:;
4249 }
4250 }
4251
4252 /* Remove devices that are no longer present. */
4253 for (i = 0; i < devcount; i++) {
4254 if (unchanged[i] != NULL)
4255 continue;
4256 device_delete_child(dev, devlist[i]);
4257 }
4258
4259 free(devlist, M_TEMP);
4260 oldcount = devcount;
4261
4262 /* Try to attach the devices just added. */
4263 error = device_get_children(dev, &devlist, &devcount);
4264 if (error) {
4265 free(unchanged, M_TEMP);
4266 return (error);
4267 }
4268
4269 for (i = 0; i < devcount; i++) {
4270 for (j = 0; j < oldcount; j++) {
4271 if (devlist[i] == unchanged[j])
4272 goto next_device;
4273 }
4274
4275 device_probe_and_attach(devlist[i]);
4276 next_device:;
4277 }
4278
4279 free(unchanged, M_TEMP);
4280 free(devlist, M_TEMP);
4281 return (0);
4282 #undef REG
4283 }
4284
4285 #ifdef PCI_IOV
4286 device_t
pci_add_iov_child(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4287 pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid,
4288 uint16_t did)
4289 {
4290 struct pci_devinfo *vf_dinfo;
4291 device_t pcib;
4292 int busno, slot, func;
4293
4294 pcib = device_get_parent(bus);
4295
4296 PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func);
4297
4298 vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno,
4299 slot, func, vid, did);
4300
4301 vf_dinfo->cfg.flags |= PCICFG_VF;
4302 pci_add_child(bus, vf_dinfo);
4303
4304 return (vf_dinfo->cfg.dev);
4305 }
4306
4307 device_t
pci_create_iov_child_method(device_t bus,device_t pf,uint16_t rid,uint16_t vid,uint16_t did)4308 pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid,
4309 uint16_t vid, uint16_t did)
4310 {
4311
4312 return (pci_add_iov_child(bus, pf, rid, vid, did));
4313 }
4314 #endif
4315
4316 /*
4317 * For PCIe device set Max_Payload_Size to match PCIe root's.
4318 */
4319 static void
pcie_setup_mps(device_t dev)4320 pcie_setup_mps(device_t dev)
4321 {
4322 struct pci_devinfo *dinfo = device_get_ivars(dev);
4323 device_t root;
4324 uint16_t rmps, mmps, mps;
4325
4326 if (dinfo->cfg.pcie.pcie_location == 0)
4327 return;
4328 root = pci_find_pcie_root_port(dev);
4329 if (root == NULL)
4330 return;
4331 /* Check whether the MPS is already configured. */
4332 rmps = pcie_read_config(root, PCIER_DEVICE_CTL, 2) &
4333 PCIEM_CTL_MAX_PAYLOAD;
4334 mps = pcie_read_config(dev, PCIER_DEVICE_CTL, 2) &
4335 PCIEM_CTL_MAX_PAYLOAD;
4336 if (mps == rmps)
4337 return;
4338 /* Check whether the device is capable of the root's MPS. */
4339 mmps = (pcie_read_config(dev, PCIER_DEVICE_CAP, 2) &
4340 PCIEM_CAP_MAX_PAYLOAD) << 5;
4341 if (rmps > mmps) {
4342 /*
4343 * The device is unable to handle root's MPS. Limit root.
4344 * XXX: We should traverse through all the tree, applying
4345 * it to all the devices.
4346 */
4347 pcie_adjust_config(root, PCIER_DEVICE_CTL,
4348 PCIEM_CTL_MAX_PAYLOAD, mmps, 2);
4349 } else {
4350 pcie_adjust_config(dev, PCIER_DEVICE_CTL,
4351 PCIEM_CTL_MAX_PAYLOAD, rmps, 2);
4352 }
4353 }
4354
4355 static void
pci_add_child_clear_aer(device_t dev,struct pci_devinfo * dinfo)4356 pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo)
4357 {
4358 int aer;
4359 uint32_t r;
4360 uint16_t r2;
4361
4362 if (dinfo->cfg.pcie.pcie_location != 0 &&
4363 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) {
4364 r2 = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
4365 PCIER_ROOT_CTL, 2);
4366 r2 &= ~(PCIEM_ROOT_CTL_SERR_CORR |
4367 PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL);
4368 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
4369 PCIER_ROOT_CTL, r2, 2);
4370 }
4371 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
4372 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
4373 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
4374 if (r != 0 && bootverbose) {
4375 pci_printf(&dinfo->cfg,
4376 "clearing AER UC 0x%08x -> 0x%08x\n",
4377 r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS,
4378 4));
4379 }
4380
4381 r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4);
4382 r &= ~(PCIM_AER_UC_TRAINING_ERROR |
4383 PCIM_AER_UC_DL_PROTOCOL_ERROR |
4384 PCIM_AER_UC_SURPRISE_LINK_DOWN |
4385 PCIM_AER_UC_POISONED_TLP |
4386 PCIM_AER_UC_FC_PROTOCOL_ERROR |
4387 PCIM_AER_UC_COMPLETION_TIMEOUT |
4388 PCIM_AER_UC_COMPLETER_ABORT |
4389 PCIM_AER_UC_UNEXPECTED_COMPLETION |
4390 PCIM_AER_UC_RECEIVER_OVERFLOW |
4391 PCIM_AER_UC_MALFORMED_TLP |
4392 PCIM_AER_UC_ECRC_ERROR |
4393 PCIM_AER_UC_UNSUPPORTED_REQUEST |
4394 PCIM_AER_UC_ACS_VIOLATION |
4395 PCIM_AER_UC_INTERNAL_ERROR |
4396 PCIM_AER_UC_MC_BLOCKED_TLP |
4397 PCIM_AER_UC_ATOMIC_EGRESS_BLK |
4398 PCIM_AER_UC_TLP_PREFIX_BLOCKED);
4399 pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4);
4400
4401 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
4402 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
4403 if (r != 0 && bootverbose) {
4404 pci_printf(&dinfo->cfg,
4405 "clearing AER COR 0x%08x -> 0x%08x\n",
4406 r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS,
4407 4));
4408 }
4409
4410 r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4);
4411 r &= ~(PCIM_AER_COR_RECEIVER_ERROR |
4412 PCIM_AER_COR_BAD_TLP |
4413 PCIM_AER_COR_BAD_DLLP |
4414 PCIM_AER_COR_REPLAY_ROLLOVER |
4415 PCIM_AER_COR_REPLAY_TIMEOUT |
4416 PCIM_AER_COR_ADVISORY_NF_ERROR |
4417 PCIM_AER_COR_INTERNAL_ERROR |
4418 PCIM_AER_COR_HEADER_LOG_OVFLOW);
4419 pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4);
4420
4421 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
4422 PCIER_DEVICE_CTL, 2);
4423 r |= PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE |
4424 PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE;
4425 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
4426 PCIER_DEVICE_CTL, r, 2);
4427 }
4428 }
4429
4430 void
pci_add_child(device_t bus,struct pci_devinfo * dinfo)4431 pci_add_child(device_t bus, struct pci_devinfo *dinfo)
4432 {
4433 device_t dev;
4434
4435 dinfo->cfg.dev = dev = device_add_child(bus, NULL, -1);
4436 device_set_ivars(dev, dinfo);
4437 resource_list_init(&dinfo->resources);
4438 pci_cfg_save(dev, dinfo, 0);
4439 pci_cfg_restore(dev, dinfo);
4440 pci_print_verbose(dinfo);
4441 pci_add_resources(bus, dev, 0, 0);
4442 if (pci_enable_mps_tune)
4443 pcie_setup_mps(dev);
4444 pci_child_added(dinfo->cfg.dev);
4445
4446 if (pci_clear_aer_on_attach)
4447 pci_add_child_clear_aer(dev, dinfo);
4448
4449 EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
4450 }
4451
4452 void
pci_child_added_method(device_t dev,device_t child)4453 pci_child_added_method(device_t dev, device_t child)
4454 {
4455
4456 }
4457
4458 static int
pci_probe(device_t dev)4459 pci_probe(device_t dev)
4460 {
4461
4462 device_set_desc(dev, "PCI bus");
4463
4464 /* Allow other subclasses to override this driver. */
4465 return (BUS_PROBE_GENERIC);
4466 }
4467
4468 int
pci_attach_common(device_t dev)4469 pci_attach_common(device_t dev)
4470 {
4471 struct pci_softc *sc;
4472 int busno, domain;
4473 #ifdef PCI_RES_BUS
4474 int rid;
4475 #endif
4476
4477 sc = device_get_softc(dev);
4478 domain = pcib_get_domain(dev);
4479 busno = pcib_get_bus(dev);
4480 #ifdef PCI_RES_BUS
4481 rid = 0;
4482 sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno,
4483 1, 0);
4484 if (sc->sc_bus == NULL) {
4485 device_printf(dev, "failed to allocate bus number\n");
4486 return (ENXIO);
4487 }
4488 #endif
4489 if (bootverbose)
4490 device_printf(dev, "domain=%d, physical bus=%d\n",
4491 domain, busno);
4492 sc->sc_dma_tag = bus_get_dma_tag(dev);
4493 return (0);
4494 }
4495
4496 int
pci_attach(device_t dev)4497 pci_attach(device_t dev)
4498 {
4499 int busno, domain, error;
4500
4501 error = pci_attach_common(dev);
4502 if (error)
4503 return (error);
4504
4505 /*
4506 * Since there can be multiple independently numbered PCI
4507 * buses on systems with multiple PCI domains, we can't use
4508 * the unit number to decide which bus we are probing. We ask
4509 * the parent pcib what our domain and bus numbers are.
4510 */
4511 domain = pcib_get_domain(dev);
4512 busno = pcib_get_bus(dev);
4513 pci_add_children(dev, domain, busno);
4514 return (bus_generic_attach(dev));
4515 }
4516
4517 int
pci_detach(device_t dev)4518 pci_detach(device_t dev)
4519 {
4520 #ifdef PCI_RES_BUS
4521 struct pci_softc *sc;
4522 #endif
4523 int error;
4524
4525 error = bus_generic_detach(dev);
4526 if (error)
4527 return (error);
4528 #ifdef PCI_RES_BUS
4529 sc = device_get_softc(dev);
4530 error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus);
4531 if (error)
4532 return (error);
4533 #endif
4534 return (device_delete_children(dev));
4535 }
4536
4537 static void
pci_hint_device_unit(device_t dev,device_t child,const char * name,int * unitp)4538 pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp)
4539 {
4540 int line, unit;
4541 const char *at;
4542 char me1[24], me2[32];
4543 uint8_t b, s, f;
4544 uint32_t d;
4545 device_location_cache_t *cache;
4546
4547 d = pci_get_domain(child);
4548 b = pci_get_bus(child);
4549 s = pci_get_slot(child);
4550 f = pci_get_function(child);
4551 snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f);
4552 snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f);
4553 line = 0;
4554 cache = dev_wired_cache_init();
4555 while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) {
4556 resource_string_value(name, unit, "at", &at);
4557 if (strcmp(at, me1) == 0 || strcmp(at, me2) == 0) {
4558 *unitp = unit;
4559 break;
4560 }
4561 if (dev_wired_cache_match(cache, child, at)) {
4562 *unitp = unit;
4563 break;
4564 }
4565 }
4566 dev_wired_cache_fini(cache);
4567 }
4568
4569 static void
pci_set_power_child(device_t dev,device_t child,int state)4570 pci_set_power_child(device_t dev, device_t child, int state)
4571 {
4572 device_t pcib;
4573 int dstate;
4574
4575 /*
4576 * Set the device to the given state. If the firmware suggests
4577 * a different power state, use it instead. If power management
4578 * is not present, the firmware is responsible for managing
4579 * device power. Skip children who aren't attached since they
4580 * are handled separately.
4581 */
4582 pcib = device_get_parent(dev);
4583 dstate = state;
4584 if (device_is_attached(child) &&
4585 PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0)
4586 pci_set_powerstate(child, dstate);
4587 }
4588
4589 int
pci_suspend_child(device_t dev,device_t child)4590 pci_suspend_child(device_t dev, device_t child)
4591 {
4592 struct pci_devinfo *dinfo;
4593 struct resource_list_entry *rle;
4594 int error;
4595
4596 dinfo = device_get_ivars(child);
4597
4598 /*
4599 * Save the PCI configuration space for the child and set the
4600 * device in the appropriate power state for this sleep state.
4601 */
4602 pci_cfg_save(child, dinfo, 0);
4603
4604 /* Suspend devices before potentially powering them down. */
4605 error = bus_generic_suspend_child(dev, child);
4606
4607 if (error)
4608 return (error);
4609
4610 if (pci_do_power_suspend) {
4611 /*
4612 * Make sure this device's interrupt handler is not invoked
4613 * in the case the device uses a shared interrupt that can
4614 * be raised by some other device.
4615 * This is applicable only to regular (legacy) PCI interrupts
4616 * as MSI/MSI-X interrupts are never shared.
4617 */
4618 rle = resource_list_find(&dinfo->resources,
4619 SYS_RES_IRQ, 0);
4620 if (rle != NULL && rle->res != NULL)
4621 (void)bus_suspend_intr(child, rle->res);
4622 pci_set_power_child(dev, child, PCI_POWERSTATE_D3);
4623 }
4624
4625 return (0);
4626 }
4627
4628 int
pci_resume_child(device_t dev,device_t child)4629 pci_resume_child(device_t dev, device_t child)
4630 {
4631 struct pci_devinfo *dinfo;
4632 struct resource_list_entry *rle;
4633
4634 if (pci_do_power_resume)
4635 pci_set_power_child(dev, child, PCI_POWERSTATE_D0);
4636
4637 dinfo = device_get_ivars(child);
4638 pci_cfg_restore(child, dinfo);
4639 if (!device_is_attached(child))
4640 pci_cfg_save(child, dinfo, 1);
4641
4642 bus_generic_resume_child(dev, child);
4643
4644 /*
4645 * Allow interrupts only after fully resuming the driver and hardware.
4646 */
4647 if (pci_do_power_suspend) {
4648 /* See pci_suspend_child for details. */
4649 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0);
4650 if (rle != NULL && rle->res != NULL)
4651 (void)bus_resume_intr(child, rle->res);
4652 }
4653
4654 return (0);
4655 }
4656
4657 int
pci_resume(device_t dev)4658 pci_resume(device_t dev)
4659 {
4660 device_t child, *devlist;
4661 int error, i, numdevs;
4662
4663 if ((error = device_get_children(dev, &devlist, &numdevs)) != 0)
4664 return (error);
4665
4666 /*
4667 * Resume critical devices first, then everything else later.
4668 */
4669 for (i = 0; i < numdevs; i++) {
4670 child = devlist[i];
4671 switch (pci_get_class(child)) {
4672 case PCIC_DISPLAY:
4673 case PCIC_MEMORY:
4674 case PCIC_BRIDGE:
4675 case PCIC_BASEPERIPH:
4676 BUS_RESUME_CHILD(dev, child);
4677 break;
4678 }
4679 }
4680 for (i = 0; i < numdevs; i++) {
4681 child = devlist[i];
4682 switch (pci_get_class(child)) {
4683 case PCIC_DISPLAY:
4684 case PCIC_MEMORY:
4685 case PCIC_BRIDGE:
4686 case PCIC_BASEPERIPH:
4687 break;
4688 default:
4689 BUS_RESUME_CHILD(dev, child);
4690 }
4691 }
4692 free(devlist, M_TEMP);
4693 return (0);
4694 }
4695
4696 static void
pci_load_vendor_data(void)4697 pci_load_vendor_data(void)
4698 {
4699 caddr_t data;
4700 void *ptr;
4701 size_t sz;
4702
4703 data = preload_search_by_type("pci_vendor_data");
4704 if (data != NULL) {
4705 ptr = preload_fetch_addr(data);
4706 sz = preload_fetch_size(data);
4707 if (ptr != NULL && sz != 0) {
4708 pci_vendordata = ptr;
4709 pci_vendordata_size = sz;
4710 /* terminate the database */
4711 pci_vendordata[pci_vendordata_size] = '\n';
4712 }
4713 }
4714 }
4715
4716 void
pci_driver_added(device_t dev,driver_t * driver)4717 pci_driver_added(device_t dev, driver_t *driver)
4718 {
4719 int numdevs;
4720 device_t *devlist;
4721 device_t child;
4722 struct pci_devinfo *dinfo;
4723 int i;
4724
4725 if (bootverbose)
4726 device_printf(dev, "driver added\n");
4727 DEVICE_IDENTIFY(driver, dev);
4728 if (device_get_children(dev, &devlist, &numdevs) != 0)
4729 return;
4730 for (i = 0; i < numdevs; i++) {
4731 child = devlist[i];
4732 if (device_get_state(child) != DS_NOTPRESENT)
4733 continue;
4734 dinfo = device_get_ivars(child);
4735 pci_print_verbose(dinfo);
4736 if (bootverbose)
4737 pci_printf(&dinfo->cfg, "reprobing on driver added\n");
4738 pci_cfg_restore(child, dinfo);
4739 if (device_probe_and_attach(child) != 0)
4740 pci_child_detached(dev, child);
4741 }
4742 free(devlist, M_TEMP);
4743 }
4744
4745 int
pci_setup_intr(device_t dev,device_t child,struct resource * irq,int flags,driver_filter_t * filter,driver_intr_t * intr,void * arg,void ** cookiep)4746 pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
4747 driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep)
4748 {
4749 struct pci_devinfo *dinfo;
4750 struct msix_table_entry *mte;
4751 struct msix_vector *mv;
4752 uint64_t addr;
4753 uint32_t data;
4754 void *cookie;
4755 int error, rid;
4756
4757 error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr,
4758 arg, &cookie);
4759 if (error)
4760 return (error);
4761
4762 /* If this is not a direct child, just bail out. */
4763 if (device_get_parent(child) != dev) {
4764 *cookiep = cookie;
4765 return(0);
4766 }
4767
4768 rid = rman_get_rid(irq);
4769 if (rid == 0) {
4770 /* Make sure that INTx is enabled */
4771 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
4772 } else {
4773 /*
4774 * Check to see if the interrupt is MSI or MSI-X.
4775 * Ask our parent to map the MSI and give
4776 * us the address and data register values.
4777 * If we fail for some reason, teardown the
4778 * interrupt handler.
4779 */
4780 dinfo = device_get_ivars(child);
4781 if (dinfo->cfg.msi.msi_alloc > 0) {
4782 if (dinfo->cfg.msi.msi_addr == 0) {
4783 KASSERT(dinfo->cfg.msi.msi_handlers == 0,
4784 ("MSI has handlers, but vectors not mapped"));
4785 error = PCIB_MAP_MSI(device_get_parent(dev),
4786 child, rman_get_start(irq), &addr, &data);
4787 if (error)
4788 goto bad;
4789 dinfo->cfg.msi.msi_addr = addr;
4790 dinfo->cfg.msi.msi_data = data;
4791 }
4792 if (dinfo->cfg.msi.msi_handlers == 0)
4793 pci_enable_msi(child, dinfo->cfg.msi.msi_addr,
4794 dinfo->cfg.msi.msi_data);
4795 dinfo->cfg.msi.msi_handlers++;
4796 } else {
4797 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
4798 ("No MSI or MSI-X interrupts allocated"));
4799 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
4800 ("MSI-X index too high"));
4801 mte = &dinfo->cfg.msix.msix_table[rid - 1];
4802 KASSERT(mte->mte_vector != 0, ("no message vector"));
4803 mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
4804 KASSERT(mv->mv_irq == rman_get_start(irq),
4805 ("IRQ mismatch"));
4806 if (mv->mv_address == 0) {
4807 KASSERT(mte->mte_handlers == 0,
4808 ("MSI-X table entry has handlers, but vector not mapped"));
4809 error = PCIB_MAP_MSI(device_get_parent(dev),
4810 child, rman_get_start(irq), &addr, &data);
4811 if (error)
4812 goto bad;
4813 mv->mv_address = addr;
4814 mv->mv_data = data;
4815 }
4816
4817 /*
4818 * The MSIX table entry must be made valid by
4819 * incrementing the mte_handlers before
4820 * calling pci_enable_msix() and
4821 * pci_resume_msix(). Else the MSIX rewrite
4822 * table quirk will not work as expected.
4823 */
4824 mte->mte_handlers++;
4825 if (mte->mte_handlers == 1) {
4826 pci_enable_msix(child, rid - 1, mv->mv_address,
4827 mv->mv_data);
4828 pci_unmask_msix(child, rid - 1);
4829 }
4830 }
4831
4832 /*
4833 * Make sure that INTx is disabled if we are using MSI/MSI-X,
4834 * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG,
4835 * in which case we "enable" INTx so MSI/MSI-X actually works.
4836 */
4837 if (!pci_has_quirk(pci_get_devid(child),
4838 PCI_QUIRK_MSI_INTX_BUG))
4839 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
4840 else
4841 pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS);
4842 bad:
4843 if (error) {
4844 (void)bus_generic_teardown_intr(dev, child, irq,
4845 cookie);
4846 return (error);
4847 }
4848 }
4849 *cookiep = cookie;
4850 return (0);
4851 }
4852
4853 int
pci_teardown_intr(device_t dev,device_t child,struct resource * irq,void * cookie)4854 pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
4855 void *cookie)
4856 {
4857 struct msix_table_entry *mte;
4858 struct resource_list_entry *rle;
4859 struct pci_devinfo *dinfo;
4860 int error, rid;
4861
4862 if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
4863 return (EINVAL);
4864
4865 /* If this isn't a direct child, just bail out */
4866 if (device_get_parent(child) != dev)
4867 return(bus_generic_teardown_intr(dev, child, irq, cookie));
4868
4869 rid = rman_get_rid(irq);
4870 if (rid == 0) {
4871 /* Mask INTx */
4872 pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS);
4873 } else {
4874 /*
4875 * Check to see if the interrupt is MSI or MSI-X. If so,
4876 * decrement the appropriate handlers count and mask the
4877 * MSI-X message, or disable MSI messages if the count
4878 * drops to 0.
4879 */
4880 dinfo = device_get_ivars(child);
4881 rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
4882 if (rle->res != irq)
4883 return (EINVAL);
4884 if (dinfo->cfg.msi.msi_alloc > 0) {
4885 KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
4886 ("MSI-X index too high"));
4887 if (dinfo->cfg.msi.msi_handlers == 0)
4888 return (EINVAL);
4889 dinfo->cfg.msi.msi_handlers--;
4890 if (dinfo->cfg.msi.msi_handlers == 0)
4891 pci_disable_msi(child);
4892 } else {
4893 KASSERT(dinfo->cfg.msix.msix_alloc > 0,
4894 ("No MSI or MSI-X interrupts allocated"));
4895 KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
4896 ("MSI-X index too high"));
4897 mte = &dinfo->cfg.msix.msix_table[rid - 1];
4898 if (mte->mte_handlers == 0)
4899 return (EINVAL);
4900 mte->mte_handlers--;
4901 if (mte->mte_handlers == 0)
4902 pci_mask_msix(child, rid - 1);
4903 }
4904 }
4905 error = bus_generic_teardown_intr(dev, child, irq, cookie);
4906 if (rid > 0)
4907 KASSERT(error == 0,
4908 ("%s: generic teardown failed for MSI/MSI-X", __func__));
4909 return (error);
4910 }
4911
4912 int
pci_print_child(device_t dev,device_t child)4913 pci_print_child(device_t dev, device_t child)
4914 {
4915 struct pci_devinfo *dinfo;
4916 struct resource_list *rl;
4917 int retval = 0;
4918
4919 dinfo = device_get_ivars(child);
4920 rl = &dinfo->resources;
4921
4922 retval += bus_print_child_header(dev, child);
4923
4924 retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx");
4925 retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx");
4926 retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd");
4927 if (device_get_flags(dev))
4928 retval += printf(" flags %#x", device_get_flags(dev));
4929
4930 retval += printf(" at device %d.%d", pci_get_slot(child),
4931 pci_get_function(child));
4932
4933 retval += bus_print_child_domain(dev, child);
4934 retval += bus_print_child_footer(dev, child);
4935
4936 return (retval);
4937 }
4938
4939 static const struct
4940 {
4941 int class;
4942 int subclass;
4943 int report; /* 0 = bootverbose, 1 = always */
4944 const char *desc;
4945 } pci_nomatch_tab[] = {
4946 {PCIC_OLD, -1, 1, "old"},
4947 {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"},
4948 {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"},
4949 {PCIC_STORAGE, -1, 1, "mass storage"},
4950 {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"},
4951 {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"},
4952 {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"},
4953 {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"},
4954 {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"},
4955 {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"},
4956 {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"},
4957 {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"},
4958 {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"},
4959 {PCIC_NETWORK, -1, 1, "network"},
4960 {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"},
4961 {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"},
4962 {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"},
4963 {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"},
4964 {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"},
4965 {PCIC_DISPLAY, -1, 1, "display"},
4966 {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"},
4967 {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"},
4968 {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"},
4969 {PCIC_MULTIMEDIA, -1, 1, "multimedia"},
4970 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"},
4971 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"},
4972 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"},
4973 {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"},
4974 {PCIC_MEMORY, -1, 1, "memory"},
4975 {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"},
4976 {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"},
4977 {PCIC_BRIDGE, -1, 1, "bridge"},
4978 {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"},
4979 {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"},
4980 {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"},
4981 {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"},
4982 {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"},
4983 {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"},
4984 {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"},
4985 {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"},
4986 {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"},
4987 {PCIC_SIMPLECOMM, -1, 1, "simple comms"},
4988 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */
4989 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"},
4990 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"},
4991 {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"},
4992 {PCIC_BASEPERIPH, -1, 0, "base peripheral"},
4993 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"},
4994 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"},
4995 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"},
4996 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"},
4997 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"},
4998 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"},
4999 {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"},
5000 {PCIC_INPUTDEV, -1, 1, "input device"},
5001 {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"},
5002 {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"},
5003 {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"},
5004 {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"},
5005 {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"},
5006 {PCIC_DOCKING, -1, 1, "docking station"},
5007 {PCIC_PROCESSOR, -1, 1, "processor"},
5008 {PCIC_SERIALBUS, -1, 1, "serial bus"},
5009 {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"},
5010 {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"},
5011 {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"},
5012 {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"},
5013 {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"},
5014 {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"},
5015 {PCIC_WIRELESS, -1, 1, "wireless controller"},
5016 {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"},
5017 {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"},
5018 {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"},
5019 {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"},
5020 {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"},
5021 {PCIC_SATCOM, -1, 1, "satellite communication"},
5022 {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"},
5023 {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"},
5024 {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"},
5025 {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"},
5026 {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"},
5027 {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"},
5028 {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"},
5029 {PCIC_DASP, -1, 0, "dasp"},
5030 {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"},
5031 {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"},
5032 {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"},
5033 {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"},
5034 {PCIC_INSTRUMENT, -1, 0, "non-essential instrumentation"},
5035 {0, 0, 0, NULL}
5036 };
5037
5038 void
pci_probe_nomatch(device_t dev,device_t child)5039 pci_probe_nomatch(device_t dev, device_t child)
5040 {
5041 int i, report;
5042 const char *cp, *scp;
5043 char *device;
5044
5045 /*
5046 * Look for a listing for this device in a loaded device database.
5047 */
5048 report = 1;
5049 if ((device = pci_describe_device(child)) != NULL) {
5050 device_printf(dev, "<%s>", device);
5051 free(device, M_DEVBUF);
5052 } else {
5053 /*
5054 * Scan the class/subclass descriptions for a general
5055 * description.
5056 */
5057 cp = "unknown";
5058 scp = NULL;
5059 for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) {
5060 if (pci_nomatch_tab[i].class == pci_get_class(child)) {
5061 if (pci_nomatch_tab[i].subclass == -1) {
5062 cp = pci_nomatch_tab[i].desc;
5063 report = pci_nomatch_tab[i].report;
5064 } else if (pci_nomatch_tab[i].subclass ==
5065 pci_get_subclass(child)) {
5066 scp = pci_nomatch_tab[i].desc;
5067 report = pci_nomatch_tab[i].report;
5068 }
5069 }
5070 }
5071 if (report || bootverbose) {
5072 device_printf(dev, "<%s%s%s>",
5073 cp ? cp : "",
5074 ((cp != NULL) && (scp != NULL)) ? ", " : "",
5075 scp ? scp : "");
5076 }
5077 }
5078 if (report || bootverbose) {
5079 printf(" at device %d.%d (no driver attached)\n",
5080 pci_get_slot(child), pci_get_function(child));
5081 }
5082 pci_cfg_save(child, device_get_ivars(child), 1);
5083 }
5084
5085 void
pci_child_detached(device_t dev,device_t child)5086 pci_child_detached(device_t dev, device_t child)
5087 {
5088 struct pci_devinfo *dinfo;
5089 struct resource_list *rl;
5090
5091 dinfo = device_get_ivars(child);
5092 rl = &dinfo->resources;
5093
5094 /*
5095 * Have to deallocate IRQs before releasing any MSI messages and
5096 * have to release MSI messages before deallocating any memory
5097 * BARs.
5098 */
5099 if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0)
5100 pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n");
5101 if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) {
5102 if (dinfo->cfg.msi.msi_alloc != 0)
5103 pci_printf(&dinfo->cfg, "Device leaked %d MSI "
5104 "vectors\n", dinfo->cfg.msi.msi_alloc);
5105 else
5106 pci_printf(&dinfo->cfg, "Device leaked %d MSI-X "
5107 "vectors\n", dinfo->cfg.msix.msix_alloc);
5108 (void)pci_release_msi(child);
5109 }
5110 if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0)
5111 pci_printf(&dinfo->cfg, "Device leaked memory resources\n");
5112 if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0)
5113 pci_printf(&dinfo->cfg, "Device leaked I/O resources\n");
5114 #ifdef PCI_RES_BUS
5115 if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0)
5116 pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n");
5117 #endif
5118
5119 pci_cfg_save(child, dinfo, 1);
5120 }
5121
5122 /*
5123 * Parse the PCI device database, if loaded, and return a pointer to a
5124 * description of the device.
5125 *
5126 * The database is flat text formatted as follows:
5127 *
5128 * Any line not in a valid format is ignored.
5129 * Lines are terminated with newline '\n' characters.
5130 *
5131 * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then
5132 * the vendor name.
5133 *
5134 * A DEVICE line is entered immediately below the corresponding VENDOR ID.
5135 * - devices cannot be listed without a corresponding VENDOR line.
5136 * A DEVICE line consists of a TAB, the 4 digit (hex) device code,
5137 * another TAB, then the device name.
5138 */
5139
5140 /*
5141 * Assuming (ptr) points to the beginning of a line in the database,
5142 * return the vendor or device and description of the next entry.
5143 * The value of (vendor) or (device) inappropriate for the entry type
5144 * is set to -1. Returns nonzero at the end of the database.
5145 *
5146 * Note that this is slightly unrobust in the face of corrupt data;
5147 * we attempt to safeguard against this by spamming the end of the
5148 * database with a newline when we initialise.
5149 */
5150 static int
pci_describe_parse_line(char ** ptr,int * vendor,int * device,char ** desc)5151 pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc)
5152 {
5153 char *cp = *ptr;
5154 int left;
5155
5156 *device = -1;
5157 *vendor = -1;
5158 **desc = '\0';
5159 for (;;) {
5160 left = pci_vendordata_size - (cp - pci_vendordata);
5161 if (left <= 0) {
5162 *ptr = cp;
5163 return(1);
5164 }
5165
5166 /* vendor entry? */
5167 if (*cp != '\t' &&
5168 sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2)
5169 break;
5170 /* device entry? */
5171 if (*cp == '\t' &&
5172 sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2)
5173 break;
5174
5175 /* skip to next line */
5176 while (*cp != '\n' && left > 0) {
5177 cp++;
5178 left--;
5179 }
5180 if (*cp == '\n') {
5181 cp++;
5182 left--;
5183 }
5184 }
5185 /* skip to next line */
5186 while (*cp != '\n' && left > 0) {
5187 cp++;
5188 left--;
5189 }
5190 if (*cp == '\n' && left > 0)
5191 cp++;
5192 *ptr = cp;
5193 return(0);
5194 }
5195
5196 static char *
pci_describe_device(device_t dev)5197 pci_describe_device(device_t dev)
5198 {
5199 int vendor, device;
5200 char *desc, *vp, *dp, *line;
5201
5202 desc = vp = dp = NULL;
5203
5204 /*
5205 * If we have no vendor data, we can't do anything.
5206 */
5207 if (pci_vendordata == NULL)
5208 goto out;
5209
5210 /*
5211 * Scan the vendor data looking for this device
5212 */
5213 line = pci_vendordata;
5214 if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5215 goto out;
5216 for (;;) {
5217 if (pci_describe_parse_line(&line, &vendor, &device, &vp))
5218 goto out;
5219 if (vendor == pci_get_vendor(dev))
5220 break;
5221 }
5222 if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL)
5223 goto out;
5224 for (;;) {
5225 if (pci_describe_parse_line(&line, &vendor, &device, &dp)) {
5226 *dp = 0;
5227 break;
5228 }
5229 if (vendor != -1) {
5230 *dp = 0;
5231 break;
5232 }
5233 if (device == pci_get_device(dev))
5234 break;
5235 }
5236 if (dp[0] == '\0')
5237 snprintf(dp, 80, "0x%x", pci_get_device(dev));
5238 if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) !=
5239 NULL)
5240 sprintf(desc, "%s, %s", vp, dp);
5241 out:
5242 if (vp != NULL)
5243 free(vp, M_DEVBUF);
5244 if (dp != NULL)
5245 free(dp, M_DEVBUF);
5246 return(desc);
5247 }
5248
5249 int
pci_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)5250 pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
5251 {
5252 struct pci_devinfo *dinfo;
5253 pcicfgregs *cfg;
5254
5255 dinfo = device_get_ivars(child);
5256 cfg = &dinfo->cfg;
5257
5258 switch (which) {
5259 case PCI_IVAR_ETHADDR:
5260 /*
5261 * The generic accessor doesn't deal with failure, so
5262 * we set the return value, then return an error.
5263 */
5264 *((uint8_t **) result) = NULL;
5265 return (EINVAL);
5266 case PCI_IVAR_SUBVENDOR:
5267 *result = cfg->subvendor;
5268 break;
5269 case PCI_IVAR_SUBDEVICE:
5270 *result = cfg->subdevice;
5271 break;
5272 case PCI_IVAR_VENDOR:
5273 *result = cfg->vendor;
5274 break;
5275 case PCI_IVAR_DEVICE:
5276 *result = cfg->device;
5277 break;
5278 case PCI_IVAR_DEVID:
5279 *result = (cfg->device << 16) | cfg->vendor;
5280 break;
5281 case PCI_IVAR_CLASS:
5282 *result = cfg->baseclass;
5283 break;
5284 case PCI_IVAR_SUBCLASS:
5285 *result = cfg->subclass;
5286 break;
5287 case PCI_IVAR_PROGIF:
5288 *result = cfg->progif;
5289 break;
5290 case PCI_IVAR_REVID:
5291 *result = cfg->revid;
5292 break;
5293 case PCI_IVAR_INTPIN:
5294 *result = cfg->intpin;
5295 break;
5296 case PCI_IVAR_IRQ:
5297 *result = cfg->intline;
5298 break;
5299 case PCI_IVAR_DOMAIN:
5300 *result = cfg->domain;
5301 break;
5302 case PCI_IVAR_BUS:
5303 *result = cfg->bus;
5304 break;
5305 case PCI_IVAR_SLOT:
5306 *result = cfg->slot;
5307 break;
5308 case PCI_IVAR_FUNCTION:
5309 *result = cfg->func;
5310 break;
5311 case PCI_IVAR_CMDREG:
5312 *result = cfg->cmdreg;
5313 break;
5314 case PCI_IVAR_CACHELNSZ:
5315 *result = cfg->cachelnsz;
5316 break;
5317 case PCI_IVAR_MINGNT:
5318 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5319 *result = -1;
5320 return (EINVAL);
5321 }
5322 *result = cfg->mingnt;
5323 break;
5324 case PCI_IVAR_MAXLAT:
5325 if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) {
5326 *result = -1;
5327 return (EINVAL);
5328 }
5329 *result = cfg->maxlat;
5330 break;
5331 case PCI_IVAR_LATTIMER:
5332 *result = cfg->lattimer;
5333 break;
5334 default:
5335 return (ENOENT);
5336 }
5337 return (0);
5338 }
5339
5340 int
pci_write_ivar(device_t dev,device_t child,int which,uintptr_t value)5341 pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value)
5342 {
5343 struct pci_devinfo *dinfo;
5344
5345 dinfo = device_get_ivars(child);
5346
5347 switch (which) {
5348 case PCI_IVAR_INTPIN:
5349 dinfo->cfg.intpin = value;
5350 return (0);
5351 case PCI_IVAR_ETHADDR:
5352 case PCI_IVAR_SUBVENDOR:
5353 case PCI_IVAR_SUBDEVICE:
5354 case PCI_IVAR_VENDOR:
5355 case PCI_IVAR_DEVICE:
5356 case PCI_IVAR_DEVID:
5357 case PCI_IVAR_CLASS:
5358 case PCI_IVAR_SUBCLASS:
5359 case PCI_IVAR_PROGIF:
5360 case PCI_IVAR_REVID:
5361 case PCI_IVAR_IRQ:
5362 case PCI_IVAR_DOMAIN:
5363 case PCI_IVAR_BUS:
5364 case PCI_IVAR_SLOT:
5365 case PCI_IVAR_FUNCTION:
5366 return (EINVAL); /* disallow for now */
5367
5368 default:
5369 return (ENOENT);
5370 }
5371 }
5372
5373 #include "opt_ddb.h"
5374 #ifdef DDB
5375 #include <ddb/ddb.h>
5376 #include <sys/cons.h>
5377
5378 /*
5379 * List resources based on pci map registers, used for within ddb
5380 */
5381
DB_SHOW_COMMAND_FLAGS(pciregs,db_pci_dump,DB_CMD_MEMSAFE)5382 DB_SHOW_COMMAND_FLAGS(pciregs, db_pci_dump, DB_CMD_MEMSAFE)
5383 {
5384 struct pci_devinfo *dinfo;
5385 struct devlist *devlist_head;
5386 struct pci_conf *p;
5387 const char *name;
5388 int i, error, none_count;
5389
5390 none_count = 0;
5391 /* get the head of the device queue */
5392 devlist_head = &pci_devq;
5393
5394 /*
5395 * Go through the list of devices and print out devices
5396 */
5397 for (error = 0, i = 0,
5398 dinfo = STAILQ_FIRST(devlist_head);
5399 (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit;
5400 dinfo = STAILQ_NEXT(dinfo, pci_links), i++) {
5401 /* Populate pd_name and pd_unit */
5402 name = NULL;
5403 if (dinfo->cfg.dev)
5404 name = device_get_name(dinfo->cfg.dev);
5405
5406 p = &dinfo->conf;
5407 db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x "
5408 "chip=0x%08x rev=0x%02x hdr=0x%02x\n",
5409 (name && *name) ? name : "none",
5410 (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) :
5411 none_count++,
5412 p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev,
5413 p->pc_sel.pc_func, (p->pc_class << 16) |
5414 (p->pc_subclass << 8) | p->pc_progif,
5415 (p->pc_subdevice << 16) | p->pc_subvendor,
5416 (p->pc_device << 16) | p->pc_vendor,
5417 p->pc_revid, p->pc_hdr);
5418 }
5419 }
5420 #endif /* DDB */
5421
5422 struct resource *
pci_reserve_map(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int num,u_int flags)5423 pci_reserve_map(device_t dev, device_t child, int type, int *rid,
5424 rman_res_t start, rman_res_t end, rman_res_t count, u_int num,
5425 u_int flags)
5426 {
5427 struct pci_devinfo *dinfo = device_get_ivars(child);
5428 struct resource_list *rl = &dinfo->resources;
5429 struct resource *res;
5430 struct pci_map *pm;
5431 uint16_t cmd;
5432 pci_addr_t map, testval;
5433 int mapsize;
5434
5435 res = NULL;
5436
5437 /* If rid is managed by EA, ignore it */
5438 if (pci_ea_is_enabled(child, *rid))
5439 goto out;
5440
5441 pm = pci_find_bar(child, *rid);
5442 if (pm != NULL) {
5443 /* This is a BAR that we failed to allocate earlier. */
5444 mapsize = pm->pm_size;
5445 map = pm->pm_value;
5446 } else {
5447 /*
5448 * Weed out the bogons, and figure out how large the
5449 * BAR/map is. BARs that read back 0 here are bogus
5450 * and unimplemented. Note: atapci in legacy mode are
5451 * special and handled elsewhere in the code. If you
5452 * have a atapci device in legacy mode and it fails
5453 * here, that other code is broken.
5454 */
5455 pci_read_bar(child, *rid, &map, &testval, NULL);
5456
5457 /*
5458 * Determine the size of the BAR and ignore BARs with a size
5459 * of 0. Device ROM BARs use a different mask value.
5460 */
5461 if (PCIR_IS_BIOS(&dinfo->cfg, *rid))
5462 mapsize = pci_romsize(testval);
5463 else
5464 mapsize = pci_mapsize(testval);
5465 if (mapsize == 0)
5466 goto out;
5467 pm = pci_add_bar(child, *rid, map, mapsize);
5468 }
5469
5470 if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, *rid)) {
5471 if (type != SYS_RES_MEMORY) {
5472 if (bootverbose)
5473 device_printf(dev,
5474 "child %s requested type %d for rid %#x,"
5475 " but the BAR says it is an memio\n",
5476 device_get_nameunit(child), type, *rid);
5477 goto out;
5478 }
5479 } else {
5480 if (type != SYS_RES_IOPORT) {
5481 if (bootverbose)
5482 device_printf(dev,
5483 "child %s requested type %d for rid %#x,"
5484 " but the BAR says it is an ioport\n",
5485 device_get_nameunit(child), type, *rid);
5486 goto out;
5487 }
5488 }
5489
5490 /*
5491 * For real BARs, we need to override the size that
5492 * the driver requests, because that's what the BAR
5493 * actually uses and we would otherwise have a
5494 * situation where we might allocate the excess to
5495 * another driver, which won't work.
5496 */
5497 count = ((pci_addr_t)1 << mapsize) * num;
5498 if (RF_ALIGNMENT(flags) < mapsize)
5499 flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize);
5500 if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH))
5501 flags |= RF_PREFETCHABLE;
5502
5503 /*
5504 * Allocate enough resource, and then write back the
5505 * appropriate BAR for that resource.
5506 */
5507 resource_list_add(rl, type, *rid, start, end, count);
5508 res = resource_list_reserve(rl, dev, child, type, rid, start, end,
5509 count, flags & ~RF_ACTIVE);
5510 if (res == NULL) {
5511 resource_list_delete(rl, type, *rid);
5512 device_printf(child,
5513 "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n",
5514 count, *rid, type, start, end);
5515 goto out;
5516 }
5517 if (bootverbose)
5518 device_printf(child,
5519 "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n",
5520 count, *rid, type, rman_get_start(res));
5521
5522 /* Disable decoding via the CMD register before updating the BAR */
5523 cmd = pci_read_config(child, PCIR_COMMAND, 2);
5524 pci_write_config(child, PCIR_COMMAND,
5525 cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2);
5526
5527 map = rman_get_start(res);
5528 pci_write_bar(child, pm, map);
5529
5530 /* Restore the original value of the CMD register */
5531 pci_write_config(child, PCIR_COMMAND, cmd, 2);
5532 out:
5533 return (res);
5534 }
5535
5536 struct resource *
pci_alloc_multi_resource(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_long num,u_int flags)5537 pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid,
5538 rman_res_t start, rman_res_t end, rman_res_t count, u_long num,
5539 u_int flags)
5540 {
5541 struct pci_devinfo *dinfo;
5542 struct resource_list *rl;
5543 struct resource_list_entry *rle;
5544 struct resource *res;
5545 pcicfgregs *cfg;
5546
5547 /*
5548 * Perform lazy resource allocation
5549 */
5550 dinfo = device_get_ivars(child);
5551 rl = &dinfo->resources;
5552 cfg = &dinfo->cfg;
5553 switch (type) {
5554 #if defined(NEW_PCIB) && defined(PCI_RES_BUS)
5555 case PCI_RES_BUS:
5556 return (pci_alloc_secbus(dev, child, rid, start, end, count,
5557 flags));
5558 #endif
5559 case SYS_RES_IRQ:
5560 /*
5561 * Can't alloc legacy interrupt once MSI messages have
5562 * been allocated.
5563 */
5564 if (*rid == 0 && (cfg->msi.msi_alloc > 0 ||
5565 cfg->msix.msix_alloc > 0))
5566 return (NULL);
5567
5568 /*
5569 * If the child device doesn't have an interrupt
5570 * routed and is deserving of an interrupt, try to
5571 * assign it one.
5572 */
5573 if (*rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) &&
5574 (cfg->intpin != 0))
5575 pci_assign_interrupt(dev, child, 0);
5576 break;
5577 case SYS_RES_IOPORT:
5578 case SYS_RES_MEMORY:
5579 #ifdef NEW_PCIB
5580 /*
5581 * PCI-PCI bridge I/O window resources are not BARs.
5582 * For those allocations just pass the request up the
5583 * tree.
5584 */
5585 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) {
5586 switch (*rid) {
5587 case PCIR_IOBASEL_1:
5588 case PCIR_MEMBASE_1:
5589 case PCIR_PMBASEL_1:
5590 /*
5591 * XXX: Should we bother creating a resource
5592 * list entry?
5593 */
5594 return (bus_generic_alloc_resource(dev, child,
5595 type, rid, start, end, count, flags));
5596 }
5597 }
5598 #endif
5599 /* Reserve resources for this BAR if needed. */
5600 rle = resource_list_find(rl, type, *rid);
5601 if (rle == NULL) {
5602 res = pci_reserve_map(dev, child, type, rid, start, end,
5603 count, num, flags);
5604 if (res == NULL)
5605 return (NULL);
5606 }
5607 }
5608 return (resource_list_alloc(rl, dev, child, type, rid,
5609 start, end, count, flags));
5610 }
5611
5612 struct resource *
pci_alloc_resource(device_t dev,device_t child,int type,int * rid,rman_res_t start,rman_res_t end,rman_res_t count,u_int flags)5613 pci_alloc_resource(device_t dev, device_t child, int type, int *rid,
5614 rman_res_t start, rman_res_t end, rman_res_t count, u_int flags)
5615 {
5616 #ifdef PCI_IOV
5617 struct pci_devinfo *dinfo;
5618 #endif
5619
5620 if (device_get_parent(child) != dev)
5621 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child,
5622 type, rid, start, end, count, flags));
5623
5624 #ifdef PCI_IOV
5625 dinfo = device_get_ivars(child);
5626 if (dinfo->cfg.flags & PCICFG_VF) {
5627 switch (type) {
5628 /* VFs can't have I/O BARs. */
5629 case SYS_RES_IOPORT:
5630 return (NULL);
5631 case SYS_RES_MEMORY:
5632 return (pci_vf_alloc_mem_resource(dev, child, rid,
5633 start, end, count, flags));
5634 }
5635
5636 /* Fall through for other types of resource allocations. */
5637 }
5638 #endif
5639
5640 return (pci_alloc_multi_resource(dev, child, type, rid, start, end,
5641 count, 1, flags));
5642 }
5643
5644 int
pci_release_resource(device_t dev,device_t child,struct resource * r)5645 pci_release_resource(device_t dev, device_t child, struct resource *r)
5646 {
5647 struct pci_devinfo *dinfo;
5648 struct resource_list *rl;
5649 pcicfgregs *cfg __unused;
5650
5651 if (device_get_parent(child) != dev)
5652 return (bus_generic_release_resource(dev, child, r));
5653
5654 dinfo = device_get_ivars(child);
5655 cfg = &dinfo->cfg;
5656
5657 #ifdef PCI_IOV
5658 if (cfg->flags & PCICFG_VF) {
5659 switch (rman_get_type(r)) {
5660 /* VFs can't have I/O BARs. */
5661 case SYS_RES_IOPORT:
5662 return (EDOOFUS);
5663 case SYS_RES_MEMORY:
5664 return (pci_vf_release_mem_resource(dev, child, r));
5665 }
5666
5667 /* Fall through for other types of resource allocations. */
5668 }
5669 #endif
5670
5671 #ifdef NEW_PCIB
5672 /*
5673 * PCI-PCI bridge I/O window resources are not BARs. For
5674 * those allocations just pass the request up the tree.
5675 */
5676 if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE &&
5677 (rman_get_type(r) == SYS_RES_IOPORT ||
5678 rman_get_type(r) == SYS_RES_MEMORY)) {
5679 switch (rman_get_rid(r)) {
5680 case PCIR_IOBASEL_1:
5681 case PCIR_MEMBASE_1:
5682 case PCIR_PMBASEL_1:
5683 return (bus_generic_release_resource(dev, child, r));
5684 }
5685 }
5686 #endif
5687
5688 rl = &dinfo->resources;
5689 return (resource_list_release(rl, dev, child, r));
5690 }
5691
5692 int
pci_activate_resource(device_t dev,device_t child,struct resource * r)5693 pci_activate_resource(device_t dev, device_t child, struct resource *r)
5694 {
5695 struct pci_devinfo *dinfo;
5696 int error, rid, type;
5697
5698 if (device_get_parent(child) != dev)
5699 return (bus_generic_activate_resource(dev, child, r));
5700
5701 dinfo = device_get_ivars(child);
5702 #ifdef PCI_IOV
5703 if (dinfo->cfg.flags & PCICFG_VF) {
5704 switch (rman_get_type(r)) {
5705 /* VFs can't have I/O BARs. */
5706 case SYS_RES_IOPORT:
5707 error = EINVAL;
5708 break;
5709 case SYS_RES_MEMORY:
5710 error = pci_vf_activate_mem_resource(dev, child, r);
5711 break;
5712 default:
5713 error = bus_generic_activate_resource(dev, child, r);
5714 break;
5715 }
5716 } else
5717 #endif
5718 error = bus_generic_activate_resource(dev, child, r);
5719 if (error)
5720 return (error);
5721
5722 rid = rman_get_rid(r);
5723 type = rman_get_type(r);
5724
5725 /* Device ROMs need their decoding explicitly enabled. */
5726 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
5727 pci_write_bar(child, pci_find_bar(child, rid),
5728 rman_get_start(r) | PCIM_BIOS_ENABLE);
5729
5730 /* Enable decoding in the command register when activating BARs. */
5731 switch (type) {
5732 case SYS_RES_IOPORT:
5733 case SYS_RES_MEMORY:
5734 error = PCI_ENABLE_IO(dev, child, type);
5735 break;
5736 }
5737 return (error);
5738 }
5739
5740 int
pci_deactivate_resource(device_t dev,device_t child,struct resource * r)5741 pci_deactivate_resource(device_t dev, device_t child, struct resource *r)
5742 {
5743 struct pci_devinfo *dinfo;
5744 int error, rid, type;
5745
5746 if (device_get_parent(child) != dev)
5747 return (bus_generic_deactivate_resource(dev, child, r));
5748
5749 dinfo = device_get_ivars(child);
5750 #ifdef PCI_IOV
5751 if (dinfo->cfg.flags & PCICFG_VF) {
5752 switch (rman_get_type(r)) {
5753 /* VFs can't have I/O BARs. */
5754 case SYS_RES_IOPORT:
5755 error = EINVAL;
5756 break;
5757 case SYS_RES_MEMORY:
5758 error = pci_vf_deactivate_mem_resource(dev, child, r);
5759 break;
5760 default:
5761 error = bus_generic_deactivate_resource(dev, child, r);
5762 break;
5763 }
5764 } else
5765 #endif
5766 error = bus_generic_deactivate_resource(dev, child, r);
5767 if (error)
5768 return (error);
5769
5770 /* Disable decoding for device ROMs. */
5771 rid = rman_get_rid(r);
5772 type = rman_get_type(r);
5773 if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid))
5774 pci_write_bar(child, pci_find_bar(child, rid),
5775 rman_get_start(r));
5776 return (0);
5777 }
5778
5779 #ifdef PCI_IOV
5780 int
pci_adjust_resource(device_t dev,device_t child,struct resource * r,rman_res_t start,rman_res_t end)5781 pci_adjust_resource(device_t dev, device_t child, struct resource *r,
5782 rman_res_t start, rman_res_t end)
5783 {
5784 struct pci_devinfo *dinfo;
5785
5786 if (device_get_parent(child) != dev)
5787 return (bus_generic_adjust_resource(dev, child, r, start,
5788 end));
5789
5790 dinfo = device_get_ivars(child);
5791 if (dinfo->cfg.flags & PCICFG_VF) {
5792 switch (rman_get_type(r)) {
5793 /* VFs can't have I/O BARs. */
5794 case SYS_RES_IOPORT:
5795 return (EINVAL);
5796 case SYS_RES_MEMORY:
5797 return (pci_vf_adjust_mem_resource(dev, child, r,
5798 start, end));
5799 }
5800
5801 /* Fall through for other types of resource allocations. */
5802 }
5803
5804 return (bus_generic_adjust_resource(dev, child, r, start, end));
5805 }
5806
5807 int
pci_map_resource(device_t dev,device_t child,struct resource * r,struct resource_map_request * argsp,struct resource_map * map)5808 pci_map_resource(device_t dev, device_t child, struct resource *r,
5809 struct resource_map_request *argsp, struct resource_map *map)
5810 {
5811 struct pci_devinfo *dinfo;
5812
5813 if (device_get_parent(child) != dev)
5814 return (bus_generic_map_resource(dev, child, r, argsp,
5815 map));
5816
5817 dinfo = device_get_ivars(child);
5818 if (dinfo->cfg.flags & PCICFG_VF) {
5819 switch (rman_get_type(r)) {
5820 /* VFs can't have I/O BARs. */
5821 case SYS_RES_IOPORT:
5822 return (EINVAL);
5823 case SYS_RES_MEMORY:
5824 return (pci_vf_map_mem_resource(dev, child, r, argsp,
5825 map));
5826 }
5827
5828 /* Fall through for other types of resource allocations. */
5829 }
5830
5831 return (bus_generic_map_resource(dev, child, r, argsp, map));
5832 }
5833
5834 int
pci_unmap_resource(device_t dev,device_t child,struct resource * r,struct resource_map * map)5835 pci_unmap_resource(device_t dev, device_t child, struct resource *r,
5836 struct resource_map *map)
5837 {
5838 struct pci_devinfo *dinfo;
5839
5840 if (device_get_parent(child) != dev)
5841 return (bus_generic_unmap_resource(dev, child, r, map));
5842
5843 dinfo = device_get_ivars(child);
5844 if (dinfo->cfg.flags & PCICFG_VF) {
5845 switch (rman_get_type(r)) {
5846 /* VFs can't have I/O BARs. */
5847 case SYS_RES_IOPORT:
5848 return (EINVAL);
5849 case SYS_RES_MEMORY:
5850 return (pci_vf_unmap_mem_resource(dev, child, r, map));
5851 }
5852
5853 /* Fall through for other types of resource allocations. */
5854 }
5855
5856 return (bus_generic_unmap_resource(dev, child, r, map));
5857 }
5858 #endif
5859
5860 void
pci_child_deleted(device_t dev,device_t child)5861 pci_child_deleted(device_t dev, device_t child)
5862 {
5863 struct resource_list_entry *rle;
5864 struct resource_list *rl;
5865 struct pci_devinfo *dinfo;
5866
5867 dinfo = device_get_ivars(child);
5868 rl = &dinfo->resources;
5869
5870 EVENTHANDLER_INVOKE(pci_delete_device, child);
5871
5872 /* Turn off access to resources we're about to free */
5873 if (bus_child_present(child) != 0) {
5874 pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
5875 PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2);
5876
5877 pci_disable_busmaster(child);
5878 }
5879
5880 /* Free all allocated resources */
5881 STAILQ_FOREACH(rle, rl, link) {
5882 if (rle->res) {
5883 if (rman_get_flags(rle->res) & RF_ACTIVE ||
5884 resource_list_busy(rl, rle->type, rle->rid)) {
5885 pci_printf(&dinfo->cfg,
5886 "Resource still owned, oops. "
5887 "(type=%d, rid=%d, addr=%lx)\n",
5888 rle->type, rle->rid,
5889 rman_get_start(rle->res));
5890 bus_release_resource(child, rle->type, rle->rid,
5891 rle->res);
5892 }
5893 resource_list_unreserve(rl, dev, child, rle->type,
5894 rle->rid);
5895 }
5896 }
5897 resource_list_free(rl);
5898
5899 pci_freecfg(dinfo);
5900 }
5901
5902 void
pci_delete_resource(device_t dev,device_t child,int type,int rid)5903 pci_delete_resource(device_t dev, device_t child, int type, int rid)
5904 {
5905 struct pci_devinfo *dinfo;
5906 struct resource_list *rl;
5907 struct resource_list_entry *rle;
5908
5909 if (device_get_parent(child) != dev)
5910 return;
5911
5912 dinfo = device_get_ivars(child);
5913 rl = &dinfo->resources;
5914 rle = resource_list_find(rl, type, rid);
5915 if (rle == NULL)
5916 return;
5917
5918 if (rle->res) {
5919 if (rman_get_flags(rle->res) & RF_ACTIVE ||
5920 resource_list_busy(rl, type, rid)) {
5921 device_printf(dev, "delete_resource: "
5922 "Resource still owned by child, oops. "
5923 "(type=%d, rid=%d, addr=%jx)\n",
5924 type, rid, rman_get_start(rle->res));
5925 return;
5926 }
5927 resource_list_unreserve(rl, dev, child, type, rid);
5928 }
5929 resource_list_delete(rl, type, rid);
5930 }
5931
5932 struct resource_list *
pci_get_resource_list(device_t dev,device_t child)5933 pci_get_resource_list (device_t dev, device_t child)
5934 {
5935 struct pci_devinfo *dinfo = device_get_ivars(child);
5936
5937 return (&dinfo->resources);
5938 }
5939
5940 #ifdef IOMMU
5941 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)5942 pci_get_dma_tag(device_t bus, device_t dev)
5943 {
5944 bus_dma_tag_t tag;
5945 struct pci_softc *sc;
5946
5947 if (device_get_parent(dev) == bus) {
5948 /* try iommu and return if it works */
5949 tag = iommu_get_dma_tag(bus, dev);
5950 } else
5951 tag = NULL;
5952 if (tag == NULL) {
5953 sc = device_get_softc(bus);
5954 tag = sc->sc_dma_tag;
5955 }
5956 return (tag);
5957 }
5958 #else
5959 bus_dma_tag_t
pci_get_dma_tag(device_t bus,device_t dev)5960 pci_get_dma_tag(device_t bus, device_t dev)
5961 {
5962 struct pci_softc *sc = device_get_softc(bus);
5963
5964 return (sc->sc_dma_tag);
5965 }
5966 #endif
5967
5968 uint32_t
pci_read_config_method(device_t dev,device_t child,int reg,int width)5969 pci_read_config_method(device_t dev, device_t child, int reg, int width)
5970 {
5971 struct pci_devinfo *dinfo = device_get_ivars(child);
5972 pcicfgregs *cfg = &dinfo->cfg;
5973
5974 #ifdef PCI_IOV
5975 /*
5976 * SR-IOV VFs don't implement the VID or DID registers, so we have to
5977 * emulate them here.
5978 */
5979 if (cfg->flags & PCICFG_VF) {
5980 if (reg == PCIR_VENDOR) {
5981 switch (width) {
5982 case 4:
5983 return (cfg->device << 16 | cfg->vendor);
5984 case 2:
5985 return (cfg->vendor);
5986 case 1:
5987 return (cfg->vendor & 0xff);
5988 default:
5989 return (0xffffffff);
5990 }
5991 } else if (reg == PCIR_DEVICE) {
5992 switch (width) {
5993 /* Note that an unaligned 4-byte read is an error. */
5994 case 2:
5995 return (cfg->device);
5996 case 1:
5997 return (cfg->device & 0xff);
5998 default:
5999 return (0xffffffff);
6000 }
6001 }
6002 }
6003 #endif
6004
6005 return (PCIB_READ_CONFIG(device_get_parent(dev),
6006 cfg->bus, cfg->slot, cfg->func, reg, width));
6007 }
6008
6009 void
pci_write_config_method(device_t dev,device_t child,int reg,uint32_t val,int width)6010 pci_write_config_method(device_t dev, device_t child, int reg,
6011 uint32_t val, int width)
6012 {
6013 struct pci_devinfo *dinfo = device_get_ivars(child);
6014 pcicfgregs *cfg = &dinfo->cfg;
6015
6016 PCIB_WRITE_CONFIG(device_get_parent(dev),
6017 cfg->bus, cfg->slot, cfg->func, reg, val, width);
6018 }
6019
6020 int
pci_child_location_method(device_t dev,device_t child,struct sbuf * sb)6021 pci_child_location_method(device_t dev, device_t child, struct sbuf *sb)
6022 {
6023
6024 sbuf_printf(sb, "slot=%d function=%d dbsf=pci%d:%d:%d:%d",
6025 pci_get_slot(child), pci_get_function(child), pci_get_domain(child),
6026 pci_get_bus(child), pci_get_slot(child), pci_get_function(child));
6027 return (0);
6028 }
6029
6030 int
pci_child_pnpinfo_method(device_t dev,device_t child,struct sbuf * sb)6031 pci_child_pnpinfo_method(device_t dev, device_t child, struct sbuf *sb)
6032 {
6033 struct pci_devinfo *dinfo;
6034 pcicfgregs *cfg;
6035
6036 dinfo = device_get_ivars(child);
6037 cfg = &dinfo->cfg;
6038 sbuf_printf(sb, "vendor=0x%04x device=0x%04x subvendor=0x%04x "
6039 "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device,
6040 cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass,
6041 cfg->progif);
6042 return (0);
6043 }
6044
6045 int
pci_get_device_path_method(device_t bus,device_t child,const char * locator,struct sbuf * sb)6046 pci_get_device_path_method(device_t bus, device_t child, const char *locator,
6047 struct sbuf *sb)
6048 {
6049 device_t parent = device_get_parent(bus);
6050 int rv;
6051
6052 if (strcmp(locator, BUS_LOCATOR_UEFI) == 0) {
6053 rv = bus_generic_get_device_path(parent, bus, locator, sb);
6054 if (rv == 0) {
6055 sbuf_printf(sb, "/Pci(0x%x,0x%x)", pci_get_slot(child),
6056 pci_get_function(child));
6057 }
6058 return (0);
6059 }
6060 return (bus_generic_get_device_path(bus, child, locator, sb));
6061 }
6062
6063 int
pci_assign_interrupt_method(device_t dev,device_t child)6064 pci_assign_interrupt_method(device_t dev, device_t child)
6065 {
6066 struct pci_devinfo *dinfo = device_get_ivars(child);
6067 pcicfgregs *cfg = &dinfo->cfg;
6068
6069 return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child,
6070 cfg->intpin));
6071 }
6072
6073 static void
pci_lookup(void * arg,const char * name,device_t * dev)6074 pci_lookup(void *arg, const char *name, device_t *dev)
6075 {
6076 long val;
6077 char *end;
6078 int domain, bus, slot, func;
6079
6080 if (*dev != NULL)
6081 return;
6082
6083 /*
6084 * Accept pciconf-style selectors of either pciD:B:S:F or
6085 * pciB:S:F. In the latter case, the domain is assumed to
6086 * be zero.
6087 */
6088 if (strncmp(name, "pci", 3) != 0)
6089 return;
6090 val = strtol(name + 3, &end, 10);
6091 if (val < 0 || val > INT_MAX || *end != ':')
6092 return;
6093 domain = val;
6094 val = strtol(end + 1, &end, 10);
6095 if (val < 0 || val > INT_MAX || *end != ':')
6096 return;
6097 bus = val;
6098 val = strtol(end + 1, &end, 10);
6099 if (val < 0 || val > INT_MAX)
6100 return;
6101 slot = val;
6102 if (*end == ':') {
6103 val = strtol(end + 1, &end, 10);
6104 if (val < 0 || val > INT_MAX || *end != '\0')
6105 return;
6106 func = val;
6107 } else if (*end == '\0') {
6108 func = slot;
6109 slot = bus;
6110 bus = domain;
6111 domain = 0;
6112 } else
6113 return;
6114
6115 if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX ||
6116 func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX))
6117 return;
6118
6119 *dev = pci_find_dbsf(domain, bus, slot, func);
6120 }
6121
6122 static int
pci_modevent(module_t mod,int what,void * arg)6123 pci_modevent(module_t mod, int what, void *arg)
6124 {
6125 static struct cdev *pci_cdev;
6126 static eventhandler_tag tag;
6127
6128 switch (what) {
6129 case MOD_LOAD:
6130 STAILQ_INIT(&pci_devq);
6131 pci_generation = 0;
6132 pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644,
6133 "pci");
6134 pci_load_vendor_data();
6135 tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL,
6136 1000);
6137 break;
6138
6139 case MOD_UNLOAD:
6140 if (tag != NULL)
6141 EVENTHANDLER_DEREGISTER(dev_lookup, tag);
6142 destroy_dev(pci_cdev);
6143 break;
6144 }
6145
6146 return (0);
6147 }
6148
6149 static void
pci_cfg_restore_pcie(device_t dev,struct pci_devinfo * dinfo)6150 pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo)
6151 {
6152 #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2)
6153 struct pcicfg_pcie *cfg;
6154 int version, pos;
6155
6156 cfg = &dinfo->cfg.pcie;
6157 pos = cfg->pcie_location;
6158
6159 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6160
6161 WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl);
6162
6163 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6164 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6165 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6166 WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl);
6167
6168 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6169 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6170 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6171 WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl);
6172
6173 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6174 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6175 WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl);
6176
6177 if (version > 1) {
6178 WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2);
6179 WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2);
6180 WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2);
6181 }
6182 #undef WREG
6183 }
6184
6185 static void
pci_cfg_restore_pcix(device_t dev,struct pci_devinfo * dinfo)6186 pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo)
6187 {
6188 pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND,
6189 dinfo->cfg.pcix.pcix_command, 2);
6190 }
6191
6192 void
pci_cfg_restore(device_t dev,struct pci_devinfo * dinfo)6193 pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
6194 {
6195
6196 /*
6197 * Restore the device to full power mode. We must do this
6198 * before we restore the registers because moving from D3 to
6199 * D0 will cause the chip's BARs and some other registers to
6200 * be reset to some unknown power on reset values. Cut down
6201 * the noise on boot by doing nothing if we are already in
6202 * state D0.
6203 */
6204 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0)
6205 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6206 pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1);
6207 pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1);
6208 pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1);
6209 pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1);
6210 pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
6211 pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
6212 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6213 case PCIM_HDRTYPE_NORMAL:
6214 pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1);
6215 pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1);
6216 break;
6217 case PCIM_HDRTYPE_BRIDGE:
6218 pci_write_config(dev, PCIR_SECLAT_1,
6219 dinfo->cfg.bridge.br_seclat, 1);
6220 pci_write_config(dev, PCIR_SUBBUS_1,
6221 dinfo->cfg.bridge.br_subbus, 1);
6222 pci_write_config(dev, PCIR_SECBUS_1,
6223 dinfo->cfg.bridge.br_secbus, 1);
6224 pci_write_config(dev, PCIR_PRIBUS_1,
6225 dinfo->cfg.bridge.br_pribus, 1);
6226 pci_write_config(dev, PCIR_BRIDGECTL_1,
6227 dinfo->cfg.bridge.br_control, 2);
6228 break;
6229 case PCIM_HDRTYPE_CARDBUS:
6230 pci_write_config(dev, PCIR_SECLAT_2,
6231 dinfo->cfg.bridge.br_seclat, 1);
6232 pci_write_config(dev, PCIR_SUBBUS_2,
6233 dinfo->cfg.bridge.br_subbus, 1);
6234 pci_write_config(dev, PCIR_SECBUS_2,
6235 dinfo->cfg.bridge.br_secbus, 1);
6236 pci_write_config(dev, PCIR_PRIBUS_2,
6237 dinfo->cfg.bridge.br_pribus, 1);
6238 pci_write_config(dev, PCIR_BRIDGECTL_2,
6239 dinfo->cfg.bridge.br_control, 2);
6240 break;
6241 }
6242 pci_restore_bars(dev);
6243
6244 if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE)
6245 pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2);
6246
6247 /*
6248 * Restore extended capabilities for PCI-Express and PCI-X
6249 */
6250 if (dinfo->cfg.pcie.pcie_location != 0)
6251 pci_cfg_restore_pcie(dev, dinfo);
6252 if (dinfo->cfg.pcix.pcix_location != 0)
6253 pci_cfg_restore_pcix(dev, dinfo);
6254
6255 /* Restore MSI and MSI-X configurations if they are present. */
6256 if (dinfo->cfg.msi.msi_location != 0)
6257 pci_resume_msi(dev);
6258 if (dinfo->cfg.msix.msix_location != 0)
6259 pci_resume_msix(dev);
6260
6261 #ifdef PCI_IOV
6262 if (dinfo->cfg.iov != NULL)
6263 pci_iov_cfg_restore(dev, dinfo);
6264 #endif
6265 }
6266
6267 static void
pci_cfg_save_pcie(device_t dev,struct pci_devinfo * dinfo)6268 pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo)
6269 {
6270 #define RREG(n) pci_read_config(dev, pos + (n), 2)
6271 struct pcicfg_pcie *cfg;
6272 int version, pos;
6273
6274 cfg = &dinfo->cfg.pcie;
6275 pos = cfg->pcie_location;
6276
6277 cfg->pcie_flags = RREG(PCIER_FLAGS);
6278
6279 version = cfg->pcie_flags & PCIEM_FLAGS_VERSION;
6280
6281 cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL);
6282
6283 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6284 cfg->pcie_type == PCIEM_TYPE_ENDPOINT ||
6285 cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT)
6286 cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL);
6287
6288 if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6289 (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT &&
6290 (cfg->pcie_flags & PCIEM_FLAGS_SLOT))))
6291 cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL);
6292
6293 if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT ||
6294 cfg->pcie_type == PCIEM_TYPE_ROOT_EC)
6295 cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL);
6296
6297 if (version > 1) {
6298 cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2);
6299 cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2);
6300 cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2);
6301 }
6302 #undef RREG
6303 }
6304
6305 static void
pci_cfg_save_pcix(device_t dev,struct pci_devinfo * dinfo)6306 pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo)
6307 {
6308 dinfo->cfg.pcix.pcix_command = pci_read_config(dev,
6309 dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2);
6310 }
6311
6312 void
pci_cfg_save(device_t dev,struct pci_devinfo * dinfo,int setstate)6313 pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
6314 {
6315 uint32_t cls;
6316 int ps;
6317
6318 /*
6319 * Some drivers apparently write to these registers w/o updating our
6320 * cached copy. No harm happens if we update the copy, so do so here
6321 * so we can restore them. The COMMAND register is modified by the
6322 * bus w/o updating the cache. This should represent the normally
6323 * writable portion of the 'defined' part of type 0/1/2 headers.
6324 */
6325 dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2);
6326 dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2);
6327 dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2);
6328 dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1);
6329 dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1);
6330 dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
6331 dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
6332 dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1);
6333 dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1);
6334 dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1);
6335 dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1);
6336 switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) {
6337 case PCIM_HDRTYPE_NORMAL:
6338 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2);
6339 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2);
6340 dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1);
6341 dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1);
6342 break;
6343 case PCIM_HDRTYPE_BRIDGE:
6344 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6345 PCIR_SECLAT_1, 1);
6346 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6347 PCIR_SUBBUS_1, 1);
6348 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6349 PCIR_SECBUS_1, 1);
6350 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6351 PCIR_PRIBUS_1, 1);
6352 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6353 PCIR_BRIDGECTL_1, 2);
6354 break;
6355 case PCIM_HDRTYPE_CARDBUS:
6356 dinfo->cfg.bridge.br_seclat = pci_read_config(dev,
6357 PCIR_SECLAT_2, 1);
6358 dinfo->cfg.bridge.br_subbus = pci_read_config(dev,
6359 PCIR_SUBBUS_2, 1);
6360 dinfo->cfg.bridge.br_secbus = pci_read_config(dev,
6361 PCIR_SECBUS_2, 1);
6362 dinfo->cfg.bridge.br_pribus = pci_read_config(dev,
6363 PCIR_PRIBUS_2, 1);
6364 dinfo->cfg.bridge.br_control = pci_read_config(dev,
6365 PCIR_BRIDGECTL_2, 2);
6366 dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2);
6367 dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2);
6368 break;
6369 }
6370
6371 if (dinfo->cfg.pcie.pcie_location != 0)
6372 pci_cfg_save_pcie(dev, dinfo);
6373
6374 if (dinfo->cfg.pcix.pcix_location != 0)
6375 pci_cfg_save_pcix(dev, dinfo);
6376
6377 #ifdef PCI_IOV
6378 if (dinfo->cfg.iov != NULL)
6379 pci_iov_cfg_save(dev, dinfo);
6380 #endif
6381
6382 /*
6383 * don't set the state for display devices, base peripherals and
6384 * memory devices since bad things happen when they are powered down.
6385 * We should (a) have drivers that can easily detach and (b) use
6386 * generic drivers for these devices so that some device actually
6387 * attaches. We need to make sure that when we implement (a) we don't
6388 * power the device down on a reattach.
6389 */
6390 cls = pci_get_class(dev);
6391 if (!setstate)
6392 return;
6393 switch (pci_do_power_nodriver)
6394 {
6395 case 0: /* NO powerdown at all */
6396 return;
6397 case 1: /* Conservative about what to power down */
6398 if (cls == PCIC_STORAGE)
6399 return;
6400 /*FALLTHROUGH*/
6401 case 2: /* Aggressive about what to power down */
6402 if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY ||
6403 cls == PCIC_BASEPERIPH)
6404 return;
6405 /*FALLTHROUGH*/
6406 case 3: /* Power down everything */
6407 break;
6408 }
6409 /*
6410 * PCI spec says we can only go into D3 state from D0 state.
6411 * Transition from D[12] into D0 before going to D3 state.
6412 */
6413 ps = pci_get_powerstate(dev);
6414 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6415 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6416 if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
6417 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6418 }
6419
6420 /* Wrapper APIs suitable for device driver use. */
6421 void
pci_save_state(device_t dev)6422 pci_save_state(device_t dev)
6423 {
6424 struct pci_devinfo *dinfo;
6425
6426 dinfo = device_get_ivars(dev);
6427 pci_cfg_save(dev, dinfo, 0);
6428 }
6429
6430 void
pci_restore_state(device_t dev)6431 pci_restore_state(device_t dev)
6432 {
6433 struct pci_devinfo *dinfo;
6434
6435 dinfo = device_get_ivars(dev);
6436 pci_cfg_restore(dev, dinfo);
6437 }
6438
6439 static int
pci_get_id_method(device_t dev,device_t child,enum pci_id_type type,uintptr_t * id)6440 pci_get_id_method(device_t dev, device_t child, enum pci_id_type type,
6441 uintptr_t *id)
6442 {
6443
6444 return (PCIB_GET_ID(device_get_parent(dev), child, type, id));
6445 }
6446
6447 /* Find the upstream port of a given PCI device in a root complex. */
6448 device_t
pci_find_pcie_root_port(device_t dev)6449 pci_find_pcie_root_port(device_t dev)
6450 {
6451 struct pci_devinfo *dinfo;
6452 devclass_t pci_class;
6453 device_t pcib, bus;
6454
6455 pci_class = devclass_find("pci");
6456 KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class,
6457 ("%s: non-pci device %s", __func__, device_get_nameunit(dev)));
6458
6459 /*
6460 * Walk the bridge hierarchy until we find a PCI-e root
6461 * port or a non-PCI device.
6462 */
6463 for (;;) {
6464 bus = device_get_parent(dev);
6465 KASSERT(bus != NULL, ("%s: null parent of %s", __func__,
6466 device_get_nameunit(dev)));
6467
6468 pcib = device_get_parent(bus);
6469 KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__,
6470 device_get_nameunit(bus)));
6471
6472 /*
6473 * pcib's parent must be a PCI bus for this to be a
6474 * PCI-PCI bridge.
6475 */
6476 if (device_get_devclass(device_get_parent(pcib)) != pci_class)
6477 return (NULL);
6478
6479 dinfo = device_get_ivars(pcib);
6480 if (dinfo->cfg.pcie.pcie_location != 0 &&
6481 dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT)
6482 return (pcib);
6483
6484 dev = pcib;
6485 }
6486 }
6487
6488 /*
6489 * Wait for pending transactions to complete on a PCI-express function.
6490 *
6491 * The maximum delay is specified in milliseconds in max_delay. Note
6492 * that this function may sleep.
6493 *
6494 * Returns true if the function is idle and false if the timeout is
6495 * exceeded. If dev is not a PCI-express function, this returns true.
6496 */
6497 bool
pcie_wait_for_pending_transactions(device_t dev,u_int max_delay)6498 pcie_wait_for_pending_transactions(device_t dev, u_int max_delay)
6499 {
6500 struct pci_devinfo *dinfo = device_get_ivars(dev);
6501 uint16_t sta;
6502 int cap;
6503
6504 cap = dinfo->cfg.pcie.pcie_location;
6505 if (cap == 0)
6506 return (true);
6507
6508 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6509 while (sta & PCIEM_STA_TRANSACTION_PND) {
6510 if (max_delay == 0)
6511 return (false);
6512
6513 /* Poll once every 100 milliseconds up to the timeout. */
6514 if (max_delay > 100) {
6515 pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK);
6516 max_delay -= 100;
6517 } else {
6518 pause_sbt("pcietp", max_delay * SBT_1MS, 0,
6519 C_HARDCLOCK);
6520 max_delay = 0;
6521 }
6522 sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2);
6523 }
6524
6525 return (true);
6526 }
6527
6528 /*
6529 * Determine the maximum Completion Timeout in microseconds.
6530 *
6531 * For non-PCI-express functions this returns 0.
6532 */
6533 int
pcie_get_max_completion_timeout(device_t dev)6534 pcie_get_max_completion_timeout(device_t dev)
6535 {
6536 struct pci_devinfo *dinfo = device_get_ivars(dev);
6537 int cap;
6538
6539 cap = dinfo->cfg.pcie.pcie_location;
6540 if (cap == 0)
6541 return (0);
6542
6543 /*
6544 * Functions using the 1.x spec use the default timeout range of
6545 * 50 microseconds to 50 milliseconds. Functions that do not
6546 * support programmable timeouts also use this range.
6547 */
6548 if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 ||
6549 (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) &
6550 PCIEM_CAP2_COMP_TIMO_RANGES) == 0)
6551 return (50 * 1000);
6552
6553 switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) &
6554 PCIEM_CTL2_COMP_TIMO_VAL) {
6555 case PCIEM_CTL2_COMP_TIMO_100US:
6556 return (100);
6557 case PCIEM_CTL2_COMP_TIMO_10MS:
6558 return (10 * 1000);
6559 case PCIEM_CTL2_COMP_TIMO_55MS:
6560 return (55 * 1000);
6561 case PCIEM_CTL2_COMP_TIMO_210MS:
6562 return (210 * 1000);
6563 case PCIEM_CTL2_COMP_TIMO_900MS:
6564 return (900 * 1000);
6565 case PCIEM_CTL2_COMP_TIMO_3500MS:
6566 return (3500 * 1000);
6567 case PCIEM_CTL2_COMP_TIMO_13S:
6568 return (13 * 1000 * 1000);
6569 case PCIEM_CTL2_COMP_TIMO_64S:
6570 return (64 * 1000 * 1000);
6571 default:
6572 return (50 * 1000);
6573 }
6574 }
6575
6576 void
pcie_apei_error(device_t dev,int sev,uint8_t * aerp)6577 pcie_apei_error(device_t dev, int sev, uint8_t *aerp)
6578 {
6579 struct pci_devinfo *dinfo = device_get_ivars(dev);
6580 const char *s;
6581 int aer;
6582 uint32_t r, r1;
6583 uint16_t rs;
6584
6585 if (sev == PCIEM_STA_CORRECTABLE_ERROR)
6586 s = "Correctable";
6587 else if (sev == PCIEM_STA_NON_FATAL_ERROR)
6588 s = "Uncorrectable (Non-Fatal)";
6589 else
6590 s = "Uncorrectable (Fatal)";
6591 device_printf(dev, "%s PCIe error reported by APEI\n", s);
6592 if (aerp) {
6593 if (sev == PCIEM_STA_CORRECTABLE_ERROR) {
6594 r = le32dec(aerp + PCIR_AER_COR_STATUS);
6595 r1 = le32dec(aerp + PCIR_AER_COR_MASK);
6596 } else {
6597 r = le32dec(aerp + PCIR_AER_UC_STATUS);
6598 r1 = le32dec(aerp + PCIR_AER_UC_MASK);
6599 }
6600 device_printf(dev, "status 0x%08x mask 0x%08x", r, r1);
6601 if (sev != PCIEM_STA_CORRECTABLE_ERROR) {
6602 r = le32dec(aerp + PCIR_AER_UC_SEVERITY);
6603 rs = le16dec(aerp + PCIR_AER_CAP_CONTROL);
6604 printf(" severity 0x%08x first %d\n",
6605 r, rs & 0x1f);
6606 } else
6607 printf("\n");
6608 }
6609
6610 /* As kind of recovery just report and clear the error statuses. */
6611 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
6612 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6613 if (r != 0) {
6614 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
6615 device_printf(dev, "Clearing UC AER errors 0x%08x\n", r);
6616 }
6617
6618 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6619 if (r != 0) {
6620 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
6621 device_printf(dev, "Clearing COR AER errors 0x%08x\n", r);
6622 }
6623 }
6624 if (dinfo->cfg.pcie.pcie_location != 0) {
6625 rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
6626 PCIER_DEVICE_STA, 2);
6627 if ((rs & (PCIEM_STA_CORRECTABLE_ERROR |
6628 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
6629 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
6630 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
6631 PCIER_DEVICE_STA, rs, 2);
6632 device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs);
6633 }
6634 }
6635 }
6636
6637 /*
6638 * Perform a Function Level Reset (FLR) on a device.
6639 *
6640 * This function first waits for any pending transactions to complete
6641 * within the timeout specified by max_delay. If transactions are
6642 * still pending, the function will return false without attempting a
6643 * reset.
6644 *
6645 * If dev is not a PCI-express function or does not support FLR, this
6646 * function returns false.
6647 *
6648 * Note that no registers are saved or restored. The caller is
6649 * responsible for saving and restoring any registers including
6650 * PCI-standard registers via pci_save_state() and
6651 * pci_restore_state().
6652 */
6653 bool
pcie_flr(device_t dev,u_int max_delay,bool force)6654 pcie_flr(device_t dev, u_int max_delay, bool force)
6655 {
6656 struct pci_devinfo *dinfo = device_get_ivars(dev);
6657 uint16_t cmd, ctl;
6658 int compl_delay;
6659 int cap;
6660
6661 cap = dinfo->cfg.pcie.pcie_location;
6662 if (cap == 0)
6663 return (false);
6664
6665 if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR))
6666 return (false);
6667
6668 /*
6669 * Disable busmastering to prevent generation of new
6670 * transactions while waiting for the device to go idle. If
6671 * the idle timeout fails, the command register is restored
6672 * which will re-enable busmastering.
6673 */
6674 cmd = pci_read_config(dev, PCIR_COMMAND, 2);
6675 pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2);
6676 if (!pcie_wait_for_pending_transactions(dev, max_delay)) {
6677 if (!force) {
6678 pci_write_config(dev, PCIR_COMMAND, cmd, 2);
6679 return (false);
6680 }
6681 pci_printf(&dinfo->cfg,
6682 "Resetting with transactions pending after %d ms\n",
6683 max_delay);
6684
6685 /*
6686 * Extend the post-FLR delay to cover the maximum
6687 * Completion Timeout delay of anything in flight
6688 * during the FLR delay. Enforce a minimum delay of
6689 * at least 10ms.
6690 */
6691 compl_delay = pcie_get_max_completion_timeout(dev) / 1000;
6692 if (compl_delay < 10)
6693 compl_delay = 10;
6694 } else
6695 compl_delay = 0;
6696
6697 /* Initiate the reset. */
6698 ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2);
6699 pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl |
6700 PCIEM_CTL_INITIATE_FLR, 2);
6701
6702 /* Wait for 100ms. */
6703 pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK);
6704
6705 if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) &
6706 PCIEM_STA_TRANSACTION_PND)
6707 pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n");
6708 return (true);
6709 }
6710
6711 /*
6712 * Attempt a power-management reset by cycling the device in/out of D3
6713 * state. PCI spec says we can only go into D3 state from D0 state.
6714 * Transition from D[12] into D0 before going to D3 state.
6715 */
6716 int
pci_power_reset(device_t dev)6717 pci_power_reset(device_t dev)
6718 {
6719 int ps;
6720
6721 ps = pci_get_powerstate(dev);
6722 if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
6723 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
6724 pci_set_powerstate(dev, PCI_POWERSTATE_D3);
6725 pci_set_powerstate(dev, ps);
6726 return (0);
6727 }
6728
6729 /*
6730 * Try link drop and retrain of the downstream port of upstream
6731 * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must
6732 * cause Conventional Hot reset of the device in the slot.
6733 * Alternative, for PCIe, could be the secondary bus reset initiatied
6734 * on the upstream switch PCIR_BRIDGECTL_1, bit 6.
6735 */
6736 int
pcie_link_reset(device_t port,int pcie_location)6737 pcie_link_reset(device_t port, int pcie_location)
6738 {
6739 uint16_t v;
6740
6741 v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2);
6742 v |= PCIEM_LINK_CTL_LINK_DIS;
6743 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
6744 pause_sbt("pcier1", mstosbt(20), 0, 0);
6745 v &= ~PCIEM_LINK_CTL_LINK_DIS;
6746 v |= PCIEM_LINK_CTL_RETRAIN_LINK;
6747 pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2);
6748 pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */
6749 v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2);
6750 return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0);
6751 }
6752
6753 static int
pci_reset_post(device_t dev,device_t child)6754 pci_reset_post(device_t dev, device_t child)
6755 {
6756
6757 if (dev == device_get_parent(child))
6758 pci_restore_state(child);
6759 return (0);
6760 }
6761
6762 static int
pci_reset_prepare(device_t dev,device_t child)6763 pci_reset_prepare(device_t dev, device_t child)
6764 {
6765
6766 if (dev == device_get_parent(child))
6767 pci_save_state(child);
6768 return (0);
6769 }
6770
6771 static int
pci_reset_child(device_t dev,device_t child,int flags)6772 pci_reset_child(device_t dev, device_t child, int flags)
6773 {
6774 int error;
6775
6776 if (dev == NULL || device_get_parent(child) != dev)
6777 return (0);
6778 if ((flags & DEVF_RESET_DETACH) != 0) {
6779 error = device_get_state(child) == DS_ATTACHED ?
6780 device_detach(child) : 0;
6781 } else {
6782 error = BUS_SUSPEND_CHILD(dev, child);
6783 }
6784 if (error == 0) {
6785 if (!pcie_flr(child, 1000, false)) {
6786 error = BUS_RESET_PREPARE(dev, child);
6787 if (error == 0)
6788 pci_power_reset(child);
6789 BUS_RESET_POST(dev, child);
6790 }
6791 if ((flags & DEVF_RESET_DETACH) != 0)
6792 device_probe_and_attach(child);
6793 else
6794 BUS_RESUME_CHILD(dev, child);
6795 }
6796 return (error);
6797 }
6798
6799 const struct pci_device_table *
pci_match_device(device_t child,const struct pci_device_table * id,size_t nelt)6800 pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt)
6801 {
6802 bool match;
6803 uint16_t vendor, device, subvendor, subdevice, class, subclass, revid;
6804
6805 vendor = pci_get_vendor(child);
6806 device = pci_get_device(child);
6807 subvendor = pci_get_subvendor(child);
6808 subdevice = pci_get_subdevice(child);
6809 class = pci_get_class(child);
6810 subclass = pci_get_subclass(child);
6811 revid = pci_get_revid(child);
6812 while (nelt-- > 0) {
6813 match = true;
6814 if (id->match_flag_vendor)
6815 match &= vendor == id->vendor;
6816 if (id->match_flag_device)
6817 match &= device == id->device;
6818 if (id->match_flag_subvendor)
6819 match &= subvendor == id->subvendor;
6820 if (id->match_flag_subdevice)
6821 match &= subdevice == id->subdevice;
6822 if (id->match_flag_class)
6823 match &= class == id->class_id;
6824 if (id->match_flag_subclass)
6825 match &= subclass == id->subclass;
6826 if (id->match_flag_revid)
6827 match &= revid == id->revid;
6828 if (match)
6829 return (id);
6830 id++;
6831 }
6832 return (NULL);
6833 }
6834
6835 static void
pci_print_faulted_dev_name(const struct pci_devinfo * dinfo)6836 pci_print_faulted_dev_name(const struct pci_devinfo *dinfo)
6837 {
6838 const char *dev_name;
6839 device_t dev;
6840
6841 dev = dinfo->cfg.dev;
6842 printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus,
6843 dinfo->cfg.slot, dinfo->cfg.func);
6844 dev_name = device_get_name(dev);
6845 if (dev_name != NULL)
6846 printf(" (%s%d)", dev_name, device_get_unit(dev));
6847 }
6848
6849 void
pci_print_faulted_dev(void)6850 pci_print_faulted_dev(void)
6851 {
6852 struct pci_devinfo *dinfo;
6853 device_t dev;
6854 int aer, i;
6855 uint32_t r1, r2;
6856 uint16_t status;
6857
6858 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
6859 dev = dinfo->cfg.dev;
6860 status = pci_read_config(dev, PCIR_STATUS, 2);
6861 status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
6862 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
6863 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
6864 if (status != 0) {
6865 pci_print_faulted_dev_name(dinfo);
6866 printf(" error 0x%04x\n", status);
6867 }
6868 if (dinfo->cfg.pcie.pcie_location != 0) {
6869 status = pci_read_config(dev,
6870 dinfo->cfg.pcie.pcie_location +
6871 PCIER_DEVICE_STA, 2);
6872 if ((status & (PCIEM_STA_CORRECTABLE_ERROR |
6873 PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR |
6874 PCIEM_STA_UNSUPPORTED_REQ)) != 0) {
6875 pci_print_faulted_dev_name(dinfo);
6876 printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n",
6877 pci_read_config(dev,
6878 dinfo->cfg.pcie.pcie_location +
6879 PCIER_DEVICE_CTL, 2),
6880 status);
6881 }
6882 }
6883 if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) {
6884 r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6885 r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6886 if (r1 != 0 || r2 != 0) {
6887 pci_print_faulted_dev_name(dinfo);
6888 printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n"
6889 " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n",
6890 r1, pci_read_config(dev, aer +
6891 PCIR_AER_UC_MASK, 4),
6892 pci_read_config(dev, aer +
6893 PCIR_AER_UC_SEVERITY, 4),
6894 r2, pci_read_config(dev, aer +
6895 PCIR_AER_COR_MASK, 4),
6896 pci_read_config(dev, aer +
6897 PCIR_AER_CAP_CONTROL, 4));
6898 for (i = 0; i < 4; i++) {
6899 r1 = pci_read_config(dev, aer +
6900 PCIR_AER_HEADER_LOG + i * 4, 4);
6901 printf(" HL%d: 0x%08x\n", i, r1);
6902 }
6903 }
6904 }
6905 }
6906 }
6907
6908 #ifdef DDB
DB_SHOW_COMMAND_FLAGS(pcierr,pci_print_faulted_dev_db,DB_CMD_MEMSAFE)6909 DB_SHOW_COMMAND_FLAGS(pcierr, pci_print_faulted_dev_db, DB_CMD_MEMSAFE)
6910 {
6911
6912 pci_print_faulted_dev();
6913 }
6914
6915 static void
db_clear_pcie_errors(const struct pci_devinfo * dinfo)6916 db_clear_pcie_errors(const struct pci_devinfo *dinfo)
6917 {
6918 device_t dev;
6919 int aer;
6920 uint32_t r;
6921
6922 dev = dinfo->cfg.dev;
6923 r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location +
6924 PCIER_DEVICE_STA, 2);
6925 pci_write_config(dev, dinfo->cfg.pcie.pcie_location +
6926 PCIER_DEVICE_STA, r, 2);
6927
6928 if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0)
6929 return;
6930 r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4);
6931 if (r != 0)
6932 pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4);
6933 r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4);
6934 if (r != 0)
6935 pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4);
6936 }
6937
DB_COMMAND_FLAGS(pci_clearerr,db_pci_clearerr,DB_CMD_MEMSAFE)6938 DB_COMMAND_FLAGS(pci_clearerr, db_pci_clearerr, DB_CMD_MEMSAFE)
6939 {
6940 struct pci_devinfo *dinfo;
6941 device_t dev;
6942 uint16_t status, status1;
6943
6944 STAILQ_FOREACH(dinfo, &pci_devq, pci_links) {
6945 dev = dinfo->cfg.dev;
6946 status1 = status = pci_read_config(dev, PCIR_STATUS, 2);
6947 status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT |
6948 PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT |
6949 PCIM_STATUS_SERR | PCIM_STATUS_PERR;
6950 if (status1 != 0) {
6951 status &= ~status1;
6952 pci_write_config(dev, PCIR_STATUS, status, 2);
6953 }
6954 if (dinfo->cfg.pcie.pcie_location != 0)
6955 db_clear_pcie_errors(dinfo);
6956 }
6957 }
6958 #endif
6959