1 /*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013-2016 Mellanox Technologies, Ltd.
6 * All rights reserved.
7 * Copyright (c) 2020-2022 The FreeBSD Foundation
8 *
9 * Portions of this software were developed by Björn Zeeb
10 * under sponsorship from the FreeBSD Foundation.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice unmodified, this list of conditions, and the following
17 * disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33 #ifndef _LINUXKPI_LINUX_PCI_H_
34 #define _LINUXKPI_LINUX_PCI_H_
35
36 #define CONFIG_PCI_MSI
37
38 #include <linux/types.h>
39
40 #include <sys/param.h>
41 #include <sys/bus.h>
42 #include <sys/module.h>
43 #include <sys/nv.h>
44 #include <sys/pciio.h>
45 #include <dev/pci/pcivar.h>
46 #include <dev/pci/pcireg.h>
47 #include <dev/pci/pci_private.h>
48
49 #include <machine/resource.h>
50
51 #include <linux/list.h>
52 #include <linux/dmapool.h>
53 #include <linux/dma-mapping.h>
54 #include <linux/compiler.h>
55 #include <linux/errno.h>
56 #include <asm/atomic.h>
57 #include <asm/memtype.h>
58 #include <linux/device.h>
59 #include <linux/pci_ids.h>
60 #include <linux/pm.h>
61
62 struct pci_device_id {
63 uint32_t vendor;
64 uint32_t device;
65 uint32_t subvendor;
66 uint32_t subdevice;
67 uint32_t class;
68 uint32_t class_mask;
69 uintptr_t driver_data;
70 };
71
72 /* Linux has an empty element at the end of the ID table -> nitems() - 1. */
73 #define MODULE_DEVICE_TABLE(_bus, _table) \
74 \
75 static device_method_t _ ## _bus ## _ ## _table ## _methods[] = { \
76 DEVMETHOD_END \
77 }; \
78 \
79 static driver_t _ ## _bus ## _ ## _table ## _driver = { \
80 "lkpi_" #_bus #_table, \
81 _ ## _bus ## _ ## _table ## _methods, \
82 0 \
83 }; \
84 \
85 DRIVER_MODULE(lkpi_ ## _table, pci, _ ## _bus ## _ ## _table ## _driver,\
86 0, 0); \
87 \
88 MODULE_PNP_INFO("U32:vendor;U32:device;V32:subvendor;V32:subdevice", \
89 _bus, lkpi_ ## _table, _table, nitems(_table) - 1)
90
91 #define PCI_ANY_ID -1U
92
93 #define PCI_DEVFN(slot, func) ((((slot) & 0x1f) << 3) | ((func) & 0x07))
94 #define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f)
95 #define PCI_FUNC(devfn) ((devfn) & 0x07)
96 #define PCI_BUS_NUM(devfn) (((devfn) >> 8) & 0xff)
97 #define PCI_DEVID(bus, devfn) ((((uint16_t)(bus)) << 8) | (devfn))
98
99 #define PCI_VDEVICE(_vendor, _device) \
100 .vendor = PCI_VENDOR_ID_##_vendor, .device = (_device), \
101 .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID
102 #define PCI_DEVICE(_vendor, _device) \
103 .vendor = (_vendor), .device = (_device), \
104 .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID
105
106 #define to_pci_dev(n) container_of(n, struct pci_dev, dev)
107
108 #define PCI_STD_NUM_BARS 6
109 #define PCI_BASE_ADDRESS_0 PCIR_BARS
110 #define PCI_BASE_ADDRESS_MEM_TYPE_64 PCIM_BAR_MEM_64
111 #define PCI_VENDOR_ID PCIR_VENDOR
112 #define PCI_DEVICE_ID PCIR_DEVICE
113 #define PCI_COMMAND PCIR_COMMAND
114 #define PCI_COMMAND_INTX_DISABLE PCIM_CMD_INTxDIS
115 #define PCI_COMMAND_MEMORY PCIM_CMD_MEMEN
116 #define PCI_EXP_DEVCTL PCIER_DEVICE_CTL /* Device Control */
117 #define PCI_EXP_LNKCTL PCIER_LINK_CTL /* Link Control */
118 #define PCI_EXP_LNKCTL_ASPM_L0S PCIEM_LINK_CTL_ASPMC_L0S
119 #define PCI_EXP_LNKCTL_ASPM_L1 PCIEM_LINK_CTL_ASPMC_L1
120 #define PCI_EXP_LNKCTL_ASPMC PCIEM_LINK_CTL_ASPMC
121 #define PCI_EXP_LNKCTL_CLKREQ_EN PCIEM_LINK_CTL_ECPM /* Enable clock PM */
122 #define PCI_EXP_LNKCTL_HAWD PCIEM_LINK_CTL_HAWD
123 #define PCI_EXP_FLAGS_TYPE PCIEM_FLAGS_TYPE /* Device/Port type */
124 #define PCI_EXP_DEVCAP PCIER_DEVICE_CAP /* Device capabilities */
125 #define PCI_EXP_DEVSTA PCIER_DEVICE_STA /* Device Status */
126 #define PCI_EXP_LNKCAP PCIER_LINK_CAP /* Link Capabilities */
127 #define PCI_EXP_LNKSTA PCIER_LINK_STA /* Link Status */
128 #define PCI_EXP_SLTCAP PCIER_SLOT_CAP /* Slot Capabilities */
129 #define PCI_EXP_SLTCTL PCIER_SLOT_CTL /* Slot Control */
130 #define PCI_EXP_SLTSTA PCIER_SLOT_STA /* Slot Status */
131 #define PCI_EXP_RTCTL PCIER_ROOT_CTL /* Root Control */
132 #define PCI_EXP_RTCAP PCIER_ROOT_CAP /* Root Capabilities */
133 #define PCI_EXP_RTSTA PCIER_ROOT_STA /* Root Status */
134 #define PCI_EXP_DEVCAP2 PCIER_DEVICE_CAP2 /* Device Capabilities 2 */
135 #define PCI_EXP_DEVCTL2 PCIER_DEVICE_CTL2 /* Device Control 2 */
136 #define PCI_EXP_DEVCTL2_LTR_EN PCIEM_CTL2_LTR_ENABLE
137 #define PCI_EXP_DEVCTL2_COMP_TMOUT_DIS PCIEM_CTL2_COMP_TIMO_DISABLE
138 #define PCI_EXP_LNKCAP2 PCIER_LINK_CAP2 /* Link Capabilities 2 */
139 #define PCI_EXP_LNKCTL2 PCIER_LINK_CTL2 /* Link Control 2 */
140 #define PCI_EXP_LNKSTA2 PCIER_LINK_STA2 /* Link Status 2 */
141 #define PCI_EXP_FLAGS PCIER_FLAGS /* Capabilities register */
142 #define PCI_EXP_FLAGS_VERS PCIEM_FLAGS_VERSION /* Capability version */
143 #define PCI_EXP_TYPE_ROOT_PORT PCIEM_TYPE_ROOT_PORT /* Root Port */
144 #define PCI_EXP_TYPE_ENDPOINT PCIEM_TYPE_ENDPOINT /* Express Endpoint */
145 #define PCI_EXP_TYPE_LEG_END PCIEM_TYPE_LEGACY_ENDPOINT /* Legacy Endpoint */
146 #define PCI_EXP_TYPE_DOWNSTREAM PCIEM_TYPE_DOWNSTREAM_PORT /* Downstream Port */
147 #define PCI_EXP_FLAGS_SLOT PCIEM_FLAGS_SLOT /* Slot implemented */
148 #define PCI_EXP_TYPE_RC_EC PCIEM_TYPE_ROOT_EC /* Root Complex Event Collector */
149 #define PCI_EXP_LNKSTA_CLS PCIEM_LINK_STA_SPEED
150 #define PCI_EXP_LNKSTA_CLS_8_0GB 0x0003 /* Current Link Speed 8.0GT/s */
151 #define PCI_EXP_LNKCAP_SLS_2_5GB 0x01 /* Supported Link Speed 2.5GT/s */
152 #define PCI_EXP_LNKCAP_SLS_5_0GB 0x02 /* Supported Link Speed 5.0GT/s */
153 #define PCI_EXP_LNKCAP_SLS_8_0GB 0x03 /* Supported Link Speed 8.0GT/s */
154 #define PCI_EXP_LNKCAP_SLS_16_0GB 0x04 /* Supported Link Speed 16.0GT/s */
155 #define PCI_EXP_LNKCAP_SLS_32_0GB 0x05 /* Supported Link Speed 32.0GT/s */
156 #define PCI_EXP_LNKCAP_SLS_64_0GB 0x06 /* Supported Link Speed 64.0GT/s */
157 #define PCI_EXP_LNKCAP_MLW 0x03f0 /* Maximum Link Width */
158 #define PCI_EXP_LNKCAP2_SLS_2_5GB 0x02 /* Supported Link Speed 2.5GT/s */
159 #define PCI_EXP_LNKCAP2_SLS_5_0GB 0x04 /* Supported Link Speed 5.0GT/s */
160 #define PCI_EXP_LNKCAP2_SLS_8_0GB 0x08 /* Supported Link Speed 8.0GT/s */
161 #define PCI_EXP_LNKCAP2_SLS_16_0GB 0x10 /* Supported Link Speed 16.0GT/s */
162 #define PCI_EXP_LNKCAP2_SLS_32_0GB 0x20 /* Supported Link Speed 32.0GT/s */
163 #define PCI_EXP_LNKCAP2_SLS_64_0GB 0x40 /* Supported Link Speed 64.0GT/s */
164 #define PCI_EXP_LNKCTL2_TLS 0x000f
165 #define PCI_EXP_LNKCTL2_TLS_2_5GT 0x0001 /* Supported Speed 2.5GT/s */
166 #define PCI_EXP_LNKCTL2_TLS_5_0GT 0x0002 /* Supported Speed 5GT/s */
167 #define PCI_EXP_LNKCTL2_TLS_8_0GT 0x0003 /* Supported Speed 8GT/s */
168 #define PCI_EXP_LNKCTL2_TLS_16_0GT 0x0004 /* Supported Speed 16GT/s */
169 #define PCI_EXP_LNKCTL2_TLS_32_0GT 0x0005 /* Supported Speed 32GT/s */
170 #define PCI_EXP_LNKCTL2_TLS_64_0GT 0x0006 /* Supported Speed 64GT/s */
171 #define PCI_EXP_LNKCTL2_ENTER_COMP 0x0010 /* Enter Compliance */
172 #define PCI_EXP_LNKCTL2_TX_MARGIN 0x0380 /* Transmit Margin */
173
174 #define PCI_MSI_ADDRESS_LO PCIR_MSI_ADDR
175 #define PCI_MSI_ADDRESS_HI PCIR_MSI_ADDR_HIGH
176 #define PCI_MSI_FLAGS PCIR_MSI_CTRL
177 #define PCI_MSI_FLAGS_ENABLE PCIM_MSICTRL_MSI_ENABLE
178 #define PCI_MSIX_FLAGS PCIR_MSIX_CTRL
179 #define PCI_MSIX_FLAGS_ENABLE PCIM_MSIXCTRL_MSIX_ENABLE
180
181 #define PCI_EXP_LNKCAP_CLKPM 0x00040000
182 #define PCI_EXP_DEVSTA_TRPND 0x0020
183
184 #define IORESOURCE_MEM (1 << SYS_RES_MEMORY)
185 #define IORESOURCE_IO (1 << SYS_RES_IOPORT)
186 #define IORESOURCE_IRQ (1 << SYS_RES_IRQ)
187
188 enum pci_bus_speed {
189 PCI_SPEED_UNKNOWN = -1,
190 PCIE_SPEED_2_5GT,
191 PCIE_SPEED_5_0GT,
192 PCIE_SPEED_8_0GT,
193 PCIE_SPEED_16_0GT,
194 PCIE_SPEED_32_0GT,
195 PCIE_SPEED_64_0GT,
196 };
197
198 enum pcie_link_width {
199 PCIE_LNK_WIDTH_RESRV = 0x00,
200 PCIE_LNK_X1 = 0x01,
201 PCIE_LNK_X2 = 0x02,
202 PCIE_LNK_X4 = 0x04,
203 PCIE_LNK_X8 = 0x08,
204 PCIE_LNK_X12 = 0x0c,
205 PCIE_LNK_X16 = 0x10,
206 PCIE_LNK_X32 = 0x20,
207 PCIE_LNK_WIDTH_UNKNOWN = 0xff,
208 };
209
210 #define PCIE_LINK_STATE_L0S 0x00000001
211 #define PCIE_LINK_STATE_L1 0x00000002
212 #define PCIE_LINK_STATE_CLKPM 0x00000004
213
214 typedef int pci_power_t;
215
216 #define PCI_D0 PCI_POWERSTATE_D0
217 #define PCI_D1 PCI_POWERSTATE_D1
218 #define PCI_D2 PCI_POWERSTATE_D2
219 #define PCI_D3hot PCI_POWERSTATE_D3
220 #define PCI_D3cold 4
221
222 #define PCI_POWER_ERROR PCI_POWERSTATE_UNKNOWN
223
224 extern const char *pci_power_names[6];
225
226 #define PCI_ERR_ROOT_COMMAND PCIR_AER_ROOTERR_CMD
227 #define PCI_ERR_ROOT_ERR_SRC PCIR_AER_COR_SOURCE_ID
228
229 #define PCI_EXT_CAP_ID_ERR PCIZ_AER
230 #define PCI_EXT_CAP_ID_L1SS PCIZ_L1PM
231
232 #define PCI_L1SS_CTL1 0x8
233 #define PCI_L1SS_CTL1_L1SS_MASK 0xf
234
235 #define PCI_IRQ_LEGACY 0x01
236 #define PCI_IRQ_MSI 0x02
237 #define PCI_IRQ_MSIX 0x04
238 #define PCI_IRQ_ALL_TYPES (PCI_IRQ_MSIX|PCI_IRQ_MSI|PCI_IRQ_LEGACY)
239
240 struct pci_dev;
241
242 struct pci_driver {
243 struct list_head node;
244 char *name;
245 const struct pci_device_id *id_table;
246 int (*probe)(struct pci_dev *dev, const struct pci_device_id *id);
247 void (*remove)(struct pci_dev *dev);
248 int (*suspend) (struct pci_dev *dev, pm_message_t state); /* Device suspended */
249 int (*resume) (struct pci_dev *dev); /* Device woken up */
250 void (*shutdown) (struct pci_dev *dev); /* Device shutdown */
251 driver_t bsddriver;
252 devclass_t bsdclass;
253 struct device_driver driver;
254 const struct pci_error_handlers *err_handler;
255 bool isdrm;
256 int bsd_probe_return;
257 int (*bsd_iov_init)(device_t dev, uint16_t num_vfs,
258 const nvlist_t *pf_config);
259 void (*bsd_iov_uninit)(device_t dev);
260 int (*bsd_iov_add_vf)(device_t dev, uint16_t vfnum,
261 const nvlist_t *vf_config);
262 };
263
264 struct pci_bus {
265 struct pci_dev *self;
266 /* struct pci_bus *parent */
267 int domain;
268 int number;
269 };
270
271 extern struct list_head pci_drivers;
272 extern struct list_head pci_devices;
273 extern spinlock_t pci_lock;
274
275 #define __devexit_p(x) x
276
277 #define module_pci_driver(_driver) \
278 \
279 static inline int \
280 _pci_init(void) \
281 { \
282 \
283 return (linux_pci_register_driver(&_driver)); \
284 } \
285 \
286 static inline void \
287 _pci_exit(void) \
288 { \
289 \
290 linux_pci_unregister_driver(&_driver); \
291 } \
292 \
293 module_init(_pci_init); \
294 module_exit(_pci_exit)
295
296 struct msi_msg {
297 uint32_t data;
298 };
299
300 struct pci_msi_desc {
301 struct {
302 bool is_64;
303 } msi_attrib;
304 };
305
306 struct msi_desc {
307 struct msi_msg msg;
308 struct pci_msi_desc pci;
309 };
310
311 struct msix_entry {
312 int entry;
313 int vector;
314 };
315
316 /*
317 * If we find drivers accessing this from multiple KPIs we may have to
318 * refcount objects of this structure.
319 */
320 struct resource;
321 struct pci_mmio_region {
322 TAILQ_ENTRY(pci_mmio_region) next;
323 struct resource *res;
324 int rid;
325 int type;
326 };
327
328 struct pci_dev {
329 struct device dev;
330 struct list_head links;
331 struct pci_driver *pdrv;
332 struct pci_bus *bus;
333 struct pci_dev *root;
334 pci_power_t current_state;
335 uint16_t device;
336 uint16_t vendor;
337 uint16_t subsystem_vendor;
338 uint16_t subsystem_device;
339 unsigned int irq;
340 unsigned int devfn;
341 uint32_t class;
342 uint8_t revision;
343 uint8_t msi_cap;
344 uint8_t msix_cap;
345 bool managed; /* devres "pcim_*(). */
346 bool want_iomap_res;
347 bool msi_enabled;
348 bool msix_enabled;
349 phys_addr_t rom;
350 size_t romlen;
351 struct msi_desc **msi_desc;
352 char *path_name;
353 spinlock_t pcie_cap_lock;
354
355 TAILQ_HEAD(, pci_mmio_region) mmio;
356 };
357
358 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name);
359 int pci_alloc_irq_vectors(struct pci_dev *pdev, int minv, int maxv,
360 unsigned int flags);
361 bool pci_device_is_present(struct pci_dev *pdev);
362
363 int linuxkpi_pcim_enable_device(struct pci_dev *pdev);
364 void __iomem **linuxkpi_pcim_iomap_table(struct pci_dev *pdev);
365 void *linuxkpi_pci_iomap(struct pci_dev *pdev, int mmio_bar, int mmio_size);
366 void linuxkpi_pci_iounmap(struct pci_dev *pdev, void *res);
367 int linuxkpi_pcim_iomap_regions(struct pci_dev *pdev, uint32_t mask,
368 const char *name);
369 int linuxkpi_pci_request_regions(struct pci_dev *pdev, const char *res_name);
370 void linuxkpi_pci_release_region(struct pci_dev *pdev, int bar);
371 void linuxkpi_pci_release_regions(struct pci_dev *pdev);
372 int linuxkpi_pci_enable_msix(struct pci_dev *pdev, struct msix_entry *entries,
373 int nreq);
374
375 /* Internal helper function(s). */
376 struct pci_dev *lkpinew_pci_dev(device_t);
377 void lkpi_pci_devres_release(struct device *, void *);
378 struct pci_dev *lkpi_pci_get_device(uint16_t, uint16_t, struct pci_dev *);
379 struct msi_desc *lkpi_pci_msi_desc_alloc(int);
380 struct device *lkpi_pci_find_irq_dev(unsigned int irq);
381 int _lkpi_pci_enable_msi_range(struct pci_dev *pdev, int minvec, int maxvec);
382
383 static inline bool
dev_is_pci(struct device * dev)384 dev_is_pci(struct device *dev)
385 {
386
387 return (device_get_devclass(dev->bsddev) == devclass_find("pci"));
388 }
389
390 static inline uint16_t
pci_dev_id(struct pci_dev * pdev)391 pci_dev_id(struct pci_dev *pdev)
392 {
393 return (PCI_DEVID(pdev->bus->number, pdev->devfn));
394 }
395
396 static inline int
pci_resource_type(struct pci_dev * pdev,int bar)397 pci_resource_type(struct pci_dev *pdev, int bar)
398 {
399 struct pci_map *pm;
400
401 pm = pci_find_bar(pdev->dev.bsddev, PCIR_BAR(bar));
402 if (!pm)
403 return (-1);
404
405 if (PCI_BAR_IO(pm->pm_value))
406 return (SYS_RES_IOPORT);
407 else
408 return (SYS_RES_MEMORY);
409 }
410
411 /*
412 * All drivers just seem to want to inspect the type not flags.
413 */
414 static inline int
pci_resource_flags(struct pci_dev * pdev,int bar)415 pci_resource_flags(struct pci_dev *pdev, int bar)
416 {
417 int type;
418
419 type = pci_resource_type(pdev, bar);
420 if (type < 0)
421 return (0);
422 return (1 << type);
423 }
424
425 static inline const char *
pci_name(struct pci_dev * d)426 pci_name(struct pci_dev *d)
427 {
428 return d->path_name;
429 }
430
431 static inline void *
pci_get_drvdata(struct pci_dev * pdev)432 pci_get_drvdata(struct pci_dev *pdev)
433 {
434
435 return dev_get_drvdata(&pdev->dev);
436 }
437
438 static inline void
pci_set_drvdata(struct pci_dev * pdev,void * data)439 pci_set_drvdata(struct pci_dev *pdev, void *data)
440 {
441
442 dev_set_drvdata(&pdev->dev, data);
443 }
444
445 static inline struct pci_dev *
pci_dev_get(struct pci_dev * pdev)446 pci_dev_get(struct pci_dev *pdev)
447 {
448
449 if (pdev != NULL)
450 get_device(&pdev->dev);
451 return (pdev);
452 }
453
454 static __inline void
pci_dev_put(struct pci_dev * pdev)455 pci_dev_put(struct pci_dev *pdev)
456 {
457
458 if (pdev != NULL)
459 put_device(&pdev->dev);
460 }
461
462 static inline int
pci_enable_device(struct pci_dev * pdev)463 pci_enable_device(struct pci_dev *pdev)
464 {
465
466 pci_enable_io(pdev->dev.bsddev, SYS_RES_IOPORT);
467 pci_enable_io(pdev->dev.bsddev, SYS_RES_MEMORY);
468 return (0);
469 }
470
471 static inline void
pci_disable_device(struct pci_dev * pdev)472 pci_disable_device(struct pci_dev *pdev)
473 {
474
475 pci_disable_busmaster(pdev->dev.bsddev);
476 }
477
478 static inline int
pci_set_master(struct pci_dev * pdev)479 pci_set_master(struct pci_dev *pdev)
480 {
481
482 pci_enable_busmaster(pdev->dev.bsddev);
483 return (0);
484 }
485
486 static inline int
pci_set_power_state(struct pci_dev * pdev,int state)487 pci_set_power_state(struct pci_dev *pdev, int state)
488 {
489
490 pci_set_powerstate(pdev->dev.bsddev, state);
491 return (0);
492 }
493
494 static inline int
pci_clear_master(struct pci_dev * pdev)495 pci_clear_master(struct pci_dev *pdev)
496 {
497
498 pci_disable_busmaster(pdev->dev.bsddev);
499 return (0);
500 }
501
502 static inline bool
pci_is_root_bus(struct pci_bus * pbus)503 pci_is_root_bus(struct pci_bus *pbus)
504 {
505
506 return (pbus->self == NULL);
507 }
508
509 static inline struct pci_dev *
pci_upstream_bridge(struct pci_dev * pdev)510 pci_upstream_bridge(struct pci_dev *pdev)
511 {
512
513 if (pci_is_root_bus(pdev->bus))
514 return (NULL);
515
516 /*
517 * If we do not have a (proper) "upstream bridge" set, e.g., we point
518 * to ourselves, try to handle this case on the fly like we do
519 * for pcie_find_root_port().
520 */
521 if (pdev == pdev->bus->self) {
522 device_t bridge;
523
524 bridge = device_get_parent(pdev->dev.bsddev);
525 if (bridge == NULL)
526 goto done;
527 bridge = device_get_parent(bridge);
528 if (bridge == NULL)
529 goto done;
530 if (device_get_devclass(device_get_parent(bridge)) !=
531 devclass_find("pci"))
532 goto done;
533
534 /*
535 * "bridge" is a PCI-to-PCI bridge. Create a Linux pci_dev
536 * for it so it can be returned.
537 */
538 pdev->bus->self = lkpinew_pci_dev(bridge);
539 }
540 done:
541 return (pdev->bus->self);
542 }
543
544 #define pci_release_region(pdev, bar) linuxkpi_pci_release_region(pdev, bar)
545 #define pci_release_regions(pdev) linuxkpi_pci_release_regions(pdev)
546 #define pci_request_regions(pdev, res_name) \
547 linuxkpi_pci_request_regions(pdev, res_name)
548
549 static inline void
lkpi_pci_disable_msix(struct pci_dev * pdev)550 lkpi_pci_disable_msix(struct pci_dev *pdev)
551 {
552
553 pci_release_msi(pdev->dev.bsddev);
554
555 /*
556 * The MSIX IRQ numbers associated with this PCI device are no
557 * longer valid and might be re-assigned. Make sure
558 * lkpi_pci_find_irq_dev() does no longer see them by
559 * resetting their references to zero:
560 */
561 pdev->dev.irq_start = 0;
562 pdev->dev.irq_end = 0;
563 pdev->msix_enabled = false;
564 }
565 /* Only for consistency. No conflict on that one. */
566 #define pci_disable_msix(pdev) lkpi_pci_disable_msix(pdev)
567
568 static inline void
lkpi_pci_disable_msi(struct pci_dev * pdev)569 lkpi_pci_disable_msi(struct pci_dev *pdev)
570 {
571
572 pci_release_msi(pdev->dev.bsddev);
573
574 pdev->dev.irq_start = 0;
575 pdev->dev.irq_end = 0;
576 pdev->irq = pdev->dev.irq;
577 pdev->msi_enabled = false;
578 }
579 #define pci_disable_msi(pdev) lkpi_pci_disable_msi(pdev)
580 #define pci_free_irq_vectors(pdev) lkpi_pci_disable_msi(pdev)
581
582 unsigned long pci_resource_start(struct pci_dev *pdev, int bar);
583 unsigned long pci_resource_len(struct pci_dev *pdev, int bar);
584
585 static inline bus_addr_t
pci_bus_address(struct pci_dev * pdev,int bar)586 pci_bus_address(struct pci_dev *pdev, int bar)
587 {
588
589 return (pci_resource_start(pdev, bar));
590 }
591
592 #define PCI_CAP_ID_EXP PCIY_EXPRESS
593 #define PCI_CAP_ID_PCIX PCIY_PCIX
594 #define PCI_CAP_ID_AGP PCIY_AGP
595 #define PCI_CAP_ID_PM PCIY_PMG
596
597 #define PCI_EXP_DEVCTL PCIER_DEVICE_CTL
598 #define PCI_EXP_DEVCTL_PAYLOAD PCIEM_CTL_MAX_PAYLOAD
599 #define PCI_EXP_DEVCTL_READRQ PCIEM_CTL_MAX_READ_REQUEST
600 #define PCI_EXP_LNKCTL PCIER_LINK_CTL
601 #define PCI_EXP_LNKSTA PCIER_LINK_STA
602
603 static inline int
pci_find_capability(struct pci_dev * pdev,int capid)604 pci_find_capability(struct pci_dev *pdev, int capid)
605 {
606 int reg;
607
608 if (pci_find_cap(pdev->dev.bsddev, capid, ®))
609 return (0);
610 return (reg);
611 }
612
pci_pcie_cap(struct pci_dev * dev)613 static inline int pci_pcie_cap(struct pci_dev *dev)
614 {
615 return pci_find_capability(dev, PCI_CAP_ID_EXP);
616 }
617
618 static inline int
pci_find_ext_capability(struct pci_dev * pdev,int capid)619 pci_find_ext_capability(struct pci_dev *pdev, int capid)
620 {
621 int reg;
622
623 if (pci_find_extcap(pdev->dev.bsddev, capid, ®))
624 return (0);
625 return (reg);
626 }
627
628 #define PCIM_PCAP_PME_SHIFT 11
629 static __inline bool
pci_pme_capable(struct pci_dev * pdev,uint32_t flag)630 pci_pme_capable(struct pci_dev *pdev, uint32_t flag)
631 {
632 struct pci_devinfo *dinfo;
633 pcicfgregs *cfg;
634
635 if (flag > (PCIM_PCAP_D3PME_COLD >> PCIM_PCAP_PME_SHIFT))
636 return (false);
637
638 dinfo = device_get_ivars(pdev->dev.bsddev);
639 cfg = &dinfo->cfg;
640
641 if (cfg->pp.pp_cap == 0)
642 return (false);
643
644 if ((cfg->pp.pp_cap & (1 << (PCIM_PCAP_PME_SHIFT + flag))) != 0)
645 return (true);
646
647 return (false);
648 }
649
650 static inline int
pci_disable_link_state(struct pci_dev * pdev,uint32_t flags)651 pci_disable_link_state(struct pci_dev *pdev, uint32_t flags)
652 {
653
654 if (!pci_enable_aspm)
655 return (-EPERM);
656
657 return (-ENXIO);
658 }
659
660 static inline int
pci_read_config_byte(const struct pci_dev * pdev,int where,u8 * val)661 pci_read_config_byte(const struct pci_dev *pdev, int where, u8 *val)
662 {
663
664 *val = (u8)pci_read_config(pdev->dev.bsddev, where, 1);
665 return (0);
666 }
667
668 static inline int
pci_read_config_word(const struct pci_dev * pdev,int where,u16 * val)669 pci_read_config_word(const struct pci_dev *pdev, int where, u16 *val)
670 {
671
672 *val = (u16)pci_read_config(pdev->dev.bsddev, where, 2);
673 return (0);
674 }
675
676 static inline int
pci_read_config_dword(const struct pci_dev * pdev,int where,u32 * val)677 pci_read_config_dword(const struct pci_dev *pdev, int where, u32 *val)
678 {
679
680 *val = (u32)pci_read_config(pdev->dev.bsddev, where, 4);
681 return (0);
682 }
683
684 static inline int
pci_write_config_byte(const struct pci_dev * pdev,int where,u8 val)685 pci_write_config_byte(const struct pci_dev *pdev, int where, u8 val)
686 {
687
688 pci_write_config(pdev->dev.bsddev, where, val, 1);
689 return (0);
690 }
691
692 static inline int
pci_write_config_word(const struct pci_dev * pdev,int where,u16 val)693 pci_write_config_word(const struct pci_dev *pdev, int where, u16 val)
694 {
695
696 pci_write_config(pdev->dev.bsddev, where, val, 2);
697 return (0);
698 }
699
700 static inline int
pci_write_config_dword(const struct pci_dev * pdev,int where,u32 val)701 pci_write_config_dword(const struct pci_dev *pdev, int where, u32 val)
702 {
703
704 pci_write_config(pdev->dev.bsddev, where, val, 4);
705 return (0);
706 }
707
708 int linux_pci_register_driver(struct pci_driver *pdrv);
709 int linux_pci_register_drm_driver(struct pci_driver *pdrv);
710 void linux_pci_unregister_driver(struct pci_driver *pdrv);
711 void linux_pci_unregister_drm_driver(struct pci_driver *pdrv);
712
713 #define pci_register_driver(pdrv) linux_pci_register_driver(pdrv)
714 #define pci_unregister_driver(pdrv) linux_pci_unregister_driver(pdrv)
715
716 /*
717 * Enable msix, positive errors indicate actual number of available
718 * vectors. Negative errors are failures.
719 *
720 * NB: define added to prevent this definition of pci_enable_msix from
721 * clashing with the native FreeBSD version.
722 */
723 #define pci_enable_msix(...) linuxkpi_pci_enable_msix(__VA_ARGS__)
724
725 #define pci_enable_msix_range(...) \
726 linux_pci_enable_msix_range(__VA_ARGS__)
727
728 static inline int
pci_enable_msix_range(struct pci_dev * dev,struct msix_entry * entries,int minvec,int maxvec)729 pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
730 int minvec, int maxvec)
731 {
732 int nvec = maxvec;
733 int rc;
734
735 if (maxvec < minvec)
736 return (-ERANGE);
737
738 do {
739 rc = pci_enable_msix(dev, entries, nvec);
740 if (rc < 0) {
741 return (rc);
742 } else if (rc > 0) {
743 if (rc < minvec)
744 return (-ENOSPC);
745 nvec = rc;
746 }
747 } while (rc);
748 return (nvec);
749 }
750
751 #define pci_enable_msi(pdev) \
752 linux_pci_enable_msi(pdev)
753
754 static inline int
pci_enable_msi(struct pci_dev * pdev)755 pci_enable_msi(struct pci_dev *pdev)
756 {
757
758 return (_lkpi_pci_enable_msi_range(pdev, 1, 1));
759 }
760
761 static inline int
pci_channel_offline(struct pci_dev * pdev)762 pci_channel_offline(struct pci_dev *pdev)
763 {
764
765 return (pci_read_config(pdev->dev.bsddev, PCIR_VENDOR, 2) == PCIV_INVALID);
766 }
767
pci_enable_sriov(struct pci_dev * dev,int nr_virtfn)768 static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
769 {
770 return -ENODEV;
771 }
772
pci_disable_sriov(struct pci_dev * dev)773 static inline void pci_disable_sriov(struct pci_dev *dev)
774 {
775 }
776
777 #define pci_iomap(pdev, mmio_bar, mmio_size) \
778 linuxkpi_pci_iomap(pdev, mmio_bar, mmio_size)
779 #define pci_iounmap(pdev, res) linuxkpi_pci_iounmap(pdev, res)
780
781 static inline void
lkpi_pci_save_state(struct pci_dev * pdev)782 lkpi_pci_save_state(struct pci_dev *pdev)
783 {
784
785 pci_save_state(pdev->dev.bsddev);
786 }
787
788 static inline void
lkpi_pci_restore_state(struct pci_dev * pdev)789 lkpi_pci_restore_state(struct pci_dev *pdev)
790 {
791
792 pci_restore_state(pdev->dev.bsddev);
793 }
794
795 #define pci_save_state(dev) lkpi_pci_save_state(dev)
796 #define pci_restore_state(dev) lkpi_pci_restore_state(dev)
797
798 static inline int
pci_reset_function(struct pci_dev * pdev)799 pci_reset_function(struct pci_dev *pdev)
800 {
801
802 return (-ENOSYS);
803 }
804
805 #define DEFINE_PCI_DEVICE_TABLE(_table) \
806 const struct pci_device_id _table[] __devinitdata
807
808 /* XXX This should not be necessary. */
809 #define pcix_set_mmrbc(d, v) 0
810 #define pcix_get_max_mmrbc(d) 0
811 #define pcie_set_readrq(d, v) pci_set_max_read_req((d)->dev.bsddev, (v))
812
813 #define PCI_DMA_BIDIRECTIONAL 0
814 #define PCI_DMA_TODEVICE 1
815 #define PCI_DMA_FROMDEVICE 2
816 #define PCI_DMA_NONE 3
817
818 #define pci_pool dma_pool
819 #define pci_pool_destroy(...) dma_pool_destroy(__VA_ARGS__)
820 #define pci_pool_alloc(...) dma_pool_alloc(__VA_ARGS__)
821 #define pci_pool_free(...) dma_pool_free(__VA_ARGS__)
822 #define pci_pool_create(_name, _pdev, _size, _align, _alloc) \
823 dma_pool_create(_name, &(_pdev)->dev, _size, _align, _alloc)
824 #define pci_free_consistent(_hwdev, _size, _vaddr, _dma_handle) \
825 dma_free_coherent((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \
826 _size, _vaddr, _dma_handle)
827 #define pci_map_sg(_hwdev, _sg, _nents, _dir) \
828 dma_map_sg((_hwdev) == NULL ? NULL : &(_hwdev->dev), \
829 _sg, _nents, (enum dma_data_direction)_dir)
830 #define pci_map_single(_hwdev, _ptr, _size, _dir) \
831 dma_map_single((_hwdev) == NULL ? NULL : &(_hwdev->dev), \
832 (_ptr), (_size), (enum dma_data_direction)_dir)
833 #define pci_unmap_single(_hwdev, _addr, _size, _dir) \
834 dma_unmap_single((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \
835 _addr, _size, (enum dma_data_direction)_dir)
836 #define pci_unmap_sg(_hwdev, _sg, _nents, _dir) \
837 dma_unmap_sg((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \
838 _sg, _nents, (enum dma_data_direction)_dir)
839 #define pci_map_page(_hwdev, _page, _offset, _size, _dir) \
840 dma_map_page((_hwdev) == NULL ? NULL : &(_hwdev)->dev, _page,\
841 _offset, _size, (enum dma_data_direction)_dir)
842 #define pci_unmap_page(_hwdev, _dma_address, _size, _dir) \
843 dma_unmap_page((_hwdev) == NULL ? NULL : &(_hwdev)->dev, \
844 _dma_address, _size, (enum dma_data_direction)_dir)
845 #define pci_set_dma_mask(_pdev, mask) dma_set_mask(&(_pdev)->dev, (mask))
846 #define pci_dma_mapping_error(_pdev, _dma_addr) \
847 dma_mapping_error(&(_pdev)->dev, _dma_addr)
848 #define pci_set_consistent_dma_mask(_pdev, _mask) \
849 dma_set_coherent_mask(&(_pdev)->dev, (_mask))
850 #define DECLARE_PCI_UNMAP_ADDR(x) DEFINE_DMA_UNMAP_ADDR(x);
851 #define DECLARE_PCI_UNMAP_LEN(x) DEFINE_DMA_UNMAP_LEN(x);
852 #define pci_unmap_addr dma_unmap_addr
853 #define pci_unmap_addr_set dma_unmap_addr_set
854 #define pci_unmap_len dma_unmap_len
855 #define pci_unmap_len_set dma_unmap_len_set
856
857 typedef unsigned int __bitwise pci_channel_state_t;
858 typedef unsigned int __bitwise pci_ers_result_t;
859
860 enum pci_channel_state {
861 pci_channel_io_normal = 1,
862 pci_channel_io_frozen = 2,
863 pci_channel_io_perm_failure = 3,
864 };
865
866 enum pci_ers_result {
867 PCI_ERS_RESULT_NONE = 1,
868 PCI_ERS_RESULT_CAN_RECOVER = 2,
869 PCI_ERS_RESULT_NEED_RESET = 3,
870 PCI_ERS_RESULT_DISCONNECT = 4,
871 PCI_ERS_RESULT_RECOVERED = 5,
872 };
873
874 /* PCI bus error event callbacks */
875 struct pci_error_handlers {
876 pci_ers_result_t (*error_detected)(struct pci_dev *dev,
877 enum pci_channel_state error);
878 pci_ers_result_t (*mmio_enabled)(struct pci_dev *dev);
879 pci_ers_result_t (*link_reset)(struct pci_dev *dev);
880 pci_ers_result_t (*slot_reset)(struct pci_dev *dev);
881 void (*resume)(struct pci_dev *dev);
882 };
883
884 /* FreeBSD does not support SRIOV - yet */
pci_physfn(struct pci_dev * dev)885 static inline struct pci_dev *pci_physfn(struct pci_dev *dev)
886 {
887 return dev;
888 }
889
pci_is_pcie(struct pci_dev * dev)890 static inline bool pci_is_pcie(struct pci_dev *dev)
891 {
892 return !!pci_pcie_cap(dev);
893 }
894
pcie_flags_reg(struct pci_dev * dev)895 static inline u16 pcie_flags_reg(struct pci_dev *dev)
896 {
897 int pos;
898 u16 reg16;
899
900 pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
901 if (!pos)
902 return 0;
903
904 pci_read_config_word(dev, pos + PCI_EXP_FLAGS, ®16);
905
906 return reg16;
907 }
908
pci_pcie_type(struct pci_dev * dev)909 static inline int pci_pcie_type(struct pci_dev *dev)
910 {
911 return (pcie_flags_reg(dev) & PCI_EXP_FLAGS_TYPE) >> 4;
912 }
913
pcie_cap_version(struct pci_dev * dev)914 static inline int pcie_cap_version(struct pci_dev *dev)
915 {
916 return pcie_flags_reg(dev) & PCI_EXP_FLAGS_VERS;
917 }
918
pcie_cap_has_lnkctl(struct pci_dev * dev)919 static inline bool pcie_cap_has_lnkctl(struct pci_dev *dev)
920 {
921 int type = pci_pcie_type(dev);
922
923 return pcie_cap_version(dev) > 1 ||
924 type == PCI_EXP_TYPE_ROOT_PORT ||
925 type == PCI_EXP_TYPE_ENDPOINT ||
926 type == PCI_EXP_TYPE_LEG_END;
927 }
928
pcie_cap_has_devctl(const struct pci_dev * dev)929 static inline bool pcie_cap_has_devctl(const struct pci_dev *dev)
930 {
931 return true;
932 }
933
pcie_cap_has_sltctl(struct pci_dev * dev)934 static inline bool pcie_cap_has_sltctl(struct pci_dev *dev)
935 {
936 int type = pci_pcie_type(dev);
937
938 return pcie_cap_version(dev) > 1 || type == PCI_EXP_TYPE_ROOT_PORT ||
939 (type == PCI_EXP_TYPE_DOWNSTREAM &&
940 pcie_flags_reg(dev) & PCI_EXP_FLAGS_SLOT);
941 }
942
pcie_cap_has_rtctl(struct pci_dev * dev)943 static inline bool pcie_cap_has_rtctl(struct pci_dev *dev)
944 {
945 int type = pci_pcie_type(dev);
946
947 return pcie_cap_version(dev) > 1 || type == PCI_EXP_TYPE_ROOT_PORT ||
948 type == PCI_EXP_TYPE_RC_EC;
949 }
950
pcie_capability_reg_implemented(struct pci_dev * dev,int pos)951 static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos)
952 {
953 if (!pci_is_pcie(dev))
954 return false;
955
956 switch (pos) {
957 case PCI_EXP_FLAGS_TYPE:
958 return true;
959 case PCI_EXP_DEVCAP:
960 case PCI_EXP_DEVCTL:
961 case PCI_EXP_DEVSTA:
962 return pcie_cap_has_devctl(dev);
963 case PCI_EXP_LNKCAP:
964 case PCI_EXP_LNKCTL:
965 case PCI_EXP_LNKSTA:
966 return pcie_cap_has_lnkctl(dev);
967 case PCI_EXP_SLTCAP:
968 case PCI_EXP_SLTCTL:
969 case PCI_EXP_SLTSTA:
970 return pcie_cap_has_sltctl(dev);
971 case PCI_EXP_RTCTL:
972 case PCI_EXP_RTCAP:
973 case PCI_EXP_RTSTA:
974 return pcie_cap_has_rtctl(dev);
975 case PCI_EXP_DEVCAP2:
976 case PCI_EXP_DEVCTL2:
977 case PCI_EXP_LNKCAP2:
978 case PCI_EXP_LNKCTL2:
979 case PCI_EXP_LNKSTA2:
980 return pcie_cap_version(dev) > 1;
981 default:
982 return false;
983 }
984 }
985
986 static inline int
pcie_capability_read_dword(struct pci_dev * dev,int pos,u32 * dst)987 pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *dst)
988 {
989 *dst = 0;
990 if (pos & 3)
991 return -EINVAL;
992
993 if (!pcie_capability_reg_implemented(dev, pos))
994 return -EINVAL;
995
996 return pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, dst);
997 }
998
999 static inline int
pcie_capability_read_word(struct pci_dev * dev,int pos,u16 * dst)1000 pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *dst)
1001 {
1002 *dst = 0;
1003 if (pos & 3)
1004 return -EINVAL;
1005
1006 if (!pcie_capability_reg_implemented(dev, pos))
1007 return -EINVAL;
1008
1009 return pci_read_config_word(dev, pci_pcie_cap(dev) + pos, dst);
1010 }
1011
1012 static inline int
pcie_capability_write_word(struct pci_dev * dev,int pos,u16 val)1013 pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val)
1014 {
1015 if (pos & 1)
1016 return -EINVAL;
1017
1018 if (!pcie_capability_reg_implemented(dev, pos))
1019 return 0;
1020
1021 return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val);
1022 }
1023
1024 static inline int
pcie_capability_clear_and_set_word(struct pci_dev * dev,int pos,uint16_t clear,uint16_t set)1025 pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos,
1026 uint16_t clear, uint16_t set)
1027 {
1028 int error;
1029 uint16_t v;
1030
1031 if (pos == PCI_EXP_LNKCTL || pos == PCI_EXP_RTCTL)
1032 spin_lock(&dev->pcie_cap_lock);
1033
1034 error = pcie_capability_read_word(dev, pos, &v);
1035 if (error == 0) {
1036 v &= ~clear;
1037 v |= set;
1038 error = pcie_capability_write_word(dev, pos, v);
1039 }
1040
1041 if (pos == PCI_EXP_LNKCTL || pos == PCI_EXP_RTCTL)
1042 spin_unlock(&dev->pcie_cap_lock);
1043
1044 return (error);
1045 }
1046
1047 static inline int
pcie_capability_set_word(struct pci_dev * dev,int pos,uint16_t val)1048 pcie_capability_set_word(struct pci_dev *dev, int pos, uint16_t val)
1049 {
1050 return (pcie_capability_clear_and_set_word(dev, pos, 0, val));
1051 }
1052
1053 static inline int
pcie_capability_clear_word(struct pci_dev * dev,int pos,uint16_t val)1054 pcie_capability_clear_word(struct pci_dev *dev, int pos, uint16_t val)
1055 {
1056 return (pcie_capability_clear_and_set_word(dev, pos, val, 0));
1057 }
1058
pcie_get_minimum_link(struct pci_dev * dev,enum pci_bus_speed * speed,enum pcie_link_width * width)1059 static inline int pcie_get_minimum_link(struct pci_dev *dev,
1060 enum pci_bus_speed *speed, enum pcie_link_width *width)
1061 {
1062 *speed = PCI_SPEED_UNKNOWN;
1063 *width = PCIE_LNK_WIDTH_UNKNOWN;
1064 return (0);
1065 }
1066
1067 static inline int
pci_num_vf(struct pci_dev * dev)1068 pci_num_vf(struct pci_dev *dev)
1069 {
1070 return (0);
1071 }
1072
1073 static inline enum pci_bus_speed
pcie_get_speed_cap(struct pci_dev * dev)1074 pcie_get_speed_cap(struct pci_dev *dev)
1075 {
1076 device_t root;
1077 uint32_t lnkcap, lnkcap2;
1078 int error, pos;
1079
1080 root = device_get_parent(dev->dev.bsddev);
1081 if (root == NULL)
1082 return (PCI_SPEED_UNKNOWN);
1083 root = device_get_parent(root);
1084 if (root == NULL)
1085 return (PCI_SPEED_UNKNOWN);
1086 root = device_get_parent(root);
1087 if (root == NULL)
1088 return (PCI_SPEED_UNKNOWN);
1089
1090 if (pci_get_vendor(root) == PCI_VENDOR_ID_VIA ||
1091 pci_get_vendor(root) == PCI_VENDOR_ID_SERVERWORKS)
1092 return (PCI_SPEED_UNKNOWN);
1093
1094 if ((error = pci_find_cap(root, PCIY_EXPRESS, &pos)) != 0)
1095 return (PCI_SPEED_UNKNOWN);
1096
1097 lnkcap2 = pci_read_config(root, pos + PCIER_LINK_CAP2, 4);
1098
1099 if (lnkcap2) { /* PCIe r3.0-compliant */
1100 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB)
1101 return (PCIE_SPEED_2_5GT);
1102 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB)
1103 return (PCIE_SPEED_5_0GT);
1104 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB)
1105 return (PCIE_SPEED_8_0GT);
1106 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_16_0GB)
1107 return (PCIE_SPEED_16_0GT);
1108 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_32_0GB)
1109 return (PCIE_SPEED_32_0GT);
1110 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_64_0GB)
1111 return (PCIE_SPEED_64_0GT);
1112 } else { /* pre-r3.0 */
1113 lnkcap = pci_read_config(root, pos + PCIER_LINK_CAP, 4);
1114 if (lnkcap & PCI_EXP_LNKCAP_SLS_2_5GB)
1115 return (PCIE_SPEED_2_5GT);
1116 if (lnkcap & PCI_EXP_LNKCAP_SLS_5_0GB)
1117 return (PCIE_SPEED_5_0GT);
1118 if (lnkcap & PCI_EXP_LNKCAP_SLS_8_0GB)
1119 return (PCIE_SPEED_8_0GT);
1120 if (lnkcap & PCI_EXP_LNKCAP_SLS_16_0GB)
1121 return (PCIE_SPEED_16_0GT);
1122 if (lnkcap & PCI_EXP_LNKCAP_SLS_32_0GB)
1123 return (PCIE_SPEED_32_0GT);
1124 if (lnkcap & PCI_EXP_LNKCAP_SLS_64_0GB)
1125 return (PCIE_SPEED_64_0GT);
1126 }
1127 return (PCI_SPEED_UNKNOWN);
1128 }
1129
1130 static inline enum pcie_link_width
pcie_get_width_cap(struct pci_dev * dev)1131 pcie_get_width_cap(struct pci_dev *dev)
1132 {
1133 uint32_t lnkcap;
1134
1135 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
1136 if (lnkcap)
1137 return ((lnkcap & PCI_EXP_LNKCAP_MLW) >> 4);
1138
1139 return (PCIE_LNK_WIDTH_UNKNOWN);
1140 }
1141
1142 static inline int
pcie_get_mps(struct pci_dev * dev)1143 pcie_get_mps(struct pci_dev *dev)
1144 {
1145 return (pci_get_max_payload(dev->dev.bsddev));
1146 }
1147
1148 static inline uint32_t
PCIE_SPEED2MBS_ENC(enum pci_bus_speed spd)1149 PCIE_SPEED2MBS_ENC(enum pci_bus_speed spd)
1150 {
1151
1152 switch(spd) {
1153 case PCIE_SPEED_64_0GT:
1154 return (64000 * 128 / 130);
1155 case PCIE_SPEED_32_0GT:
1156 return (32000 * 128 / 130);
1157 case PCIE_SPEED_16_0GT:
1158 return (16000 * 128 / 130);
1159 case PCIE_SPEED_8_0GT:
1160 return (8000 * 128 / 130);
1161 case PCIE_SPEED_5_0GT:
1162 return (5000 * 8 / 10);
1163 case PCIE_SPEED_2_5GT:
1164 return (2500 * 8 / 10);
1165 default:
1166 return (0);
1167 }
1168 }
1169
1170 static inline uint32_t
pcie_bandwidth_available(struct pci_dev * pdev,struct pci_dev ** limiting,enum pci_bus_speed * speed,enum pcie_link_width * width)1171 pcie_bandwidth_available(struct pci_dev *pdev,
1172 struct pci_dev **limiting,
1173 enum pci_bus_speed *speed,
1174 enum pcie_link_width *width)
1175 {
1176 enum pci_bus_speed nspeed = pcie_get_speed_cap(pdev);
1177 enum pcie_link_width nwidth = pcie_get_width_cap(pdev);
1178
1179 if (speed)
1180 *speed = nspeed;
1181 if (width)
1182 *width = nwidth;
1183
1184 return (nwidth * PCIE_SPEED2MBS_ENC(nspeed));
1185 }
1186
1187 static inline bool
pcie_aspm_enabled(struct pci_dev * pdev)1188 pcie_aspm_enabled(struct pci_dev *pdev)
1189 {
1190 return (false);
1191 }
1192
1193 static inline struct pci_dev *
pcie_find_root_port(struct pci_dev * pdev)1194 pcie_find_root_port(struct pci_dev *pdev)
1195 {
1196 device_t root;
1197
1198 if (pdev->root != NULL)
1199 return (pdev->root);
1200
1201 root = pci_find_pcie_root_port(pdev->dev.bsddev);
1202 if (root == NULL)
1203 return (NULL);
1204
1205 pdev->root = lkpinew_pci_dev(root);
1206 return (pdev->root);
1207 }
1208
1209 /* This is needed when people rip out the device "HotPlug". */
1210 static inline void
pci_lock_rescan_remove(void)1211 pci_lock_rescan_remove(void)
1212 {
1213 }
1214
1215 static inline void
pci_unlock_rescan_remove(void)1216 pci_unlock_rescan_remove(void)
1217 {
1218 }
1219
1220 static __inline void
pci_stop_and_remove_bus_device(struct pci_dev * pdev)1221 pci_stop_and_remove_bus_device(struct pci_dev *pdev)
1222 {
1223 }
1224
1225 static inline int
pci_rescan_bus(struct pci_bus * pbus)1226 pci_rescan_bus(struct pci_bus *pbus)
1227 {
1228 device_t *devlist, parent;
1229 int devcount, error;
1230
1231 if (!device_is_attached(pbus->self->dev.bsddev))
1232 return (0);
1233 /* pci_rescan_method() will work on the pcib (parent). */
1234 error = BUS_RESCAN(pbus->self->dev.bsddev);
1235 if (error != 0)
1236 return (0);
1237
1238 parent = device_get_parent(pbus->self->dev.bsddev);
1239 error = device_get_children(parent, &devlist, &devcount);
1240 if (error != 0)
1241 return (0);
1242 if (devcount != 0)
1243 free(devlist, M_TEMP);
1244
1245 return (devcount);
1246 }
1247
1248 /*
1249 * The following functions can be used to attach/detach the LinuxKPI's
1250 * PCI device runtime. The pci_driver and pci_device_id pointer is
1251 * allowed to be NULL. Other pointers must be all valid.
1252 * The pci_dev structure should be zero-initialized before passed
1253 * to the linux_pci_attach_device function.
1254 */
1255 extern int linux_pci_attach_device(device_t, struct pci_driver *,
1256 const struct pci_device_id *, struct pci_dev *);
1257 extern int linux_pci_detach_device(struct pci_dev *);
1258
1259 static inline int
pci_dev_present(const struct pci_device_id * cur)1260 pci_dev_present(const struct pci_device_id *cur)
1261 {
1262 while (cur != NULL && (cur->vendor || cur->device)) {
1263 if (pci_find_device(cur->vendor, cur->device) != NULL) {
1264 return (1);
1265 }
1266 cur++;
1267 }
1268 return (0);
1269 }
1270
1271 struct pci_dev *lkpi_pci_get_domain_bus_and_slot(int domain,
1272 unsigned int bus, unsigned int devfn);
1273 #define pci_get_domain_bus_and_slot(domain, bus, devfn) \
1274 lkpi_pci_get_domain_bus_and_slot(domain, bus, devfn)
1275
1276 static inline int
pci_domain_nr(struct pci_bus * pbus)1277 pci_domain_nr(struct pci_bus *pbus)
1278 {
1279
1280 return (pbus->domain);
1281 }
1282
1283 static inline int
pci_bus_read_config(struct pci_bus * bus,unsigned int devfn,int pos,uint32_t * val,int len)1284 pci_bus_read_config(struct pci_bus *bus, unsigned int devfn,
1285 int pos, uint32_t *val, int len)
1286 {
1287
1288 *val = pci_read_config(bus->self->dev.bsddev, pos, len);
1289 return (0);
1290 }
1291
1292 static inline int
pci_bus_read_config_word(struct pci_bus * bus,unsigned int devfn,int pos,u16 * val)1293 pci_bus_read_config_word(struct pci_bus *bus, unsigned int devfn, int pos, u16 *val)
1294 {
1295 uint32_t tmp;
1296 int ret;
1297
1298 ret = pci_bus_read_config(bus, devfn, pos, &tmp, 2);
1299 *val = (u16)tmp;
1300 return (ret);
1301 }
1302
1303 static inline int
pci_bus_read_config_byte(struct pci_bus * bus,unsigned int devfn,int pos,u8 * val)1304 pci_bus_read_config_byte(struct pci_bus *bus, unsigned int devfn, int pos, u8 *val)
1305 {
1306 uint32_t tmp;
1307 int ret;
1308
1309 ret = pci_bus_read_config(bus, devfn, pos, &tmp, 1);
1310 *val = (u8)tmp;
1311 return (ret);
1312 }
1313
1314 static inline int
pci_bus_write_config(struct pci_bus * bus,unsigned int devfn,int pos,uint32_t val,int size)1315 pci_bus_write_config(struct pci_bus *bus, unsigned int devfn, int pos,
1316 uint32_t val, int size)
1317 {
1318
1319 pci_write_config(bus->self->dev.bsddev, pos, val, size);
1320 return (0);
1321 }
1322
1323 static inline int
pci_bus_write_config_byte(struct pci_bus * bus,unsigned int devfn,int pos,uint8_t val)1324 pci_bus_write_config_byte(struct pci_bus *bus, unsigned int devfn, int pos,
1325 uint8_t val)
1326 {
1327 return (pci_bus_write_config(bus, devfn, pos, val, 1));
1328 }
1329
1330 static inline int
pci_bus_write_config_word(struct pci_bus * bus,unsigned int devfn,int pos,uint16_t val)1331 pci_bus_write_config_word(struct pci_bus *bus, unsigned int devfn, int pos,
1332 uint16_t val)
1333 {
1334 return (pci_bus_write_config(bus, devfn, pos, val, 2));
1335 }
1336
1337 struct pci_dev *lkpi_pci_get_class(unsigned int class, struct pci_dev *from);
1338 #define pci_get_class(class, from) lkpi_pci_get_class(class, from)
1339
1340 /* -------------------------------------------------------------------------- */
1341
1342 #define pcim_enable_device(pdev) linuxkpi_pcim_enable_device(pdev)
1343 #define pcim_iomap_table(pdev) linuxkpi_pcim_iomap_table(pdev)
1344 #define pcim_iomap_regions(pdev, mask, name) \
1345 linuxkpi_pcim_iomap_regions(pdev, mask, name)
1346
1347 static inline int
pcim_iomap_regions_request_all(struct pci_dev * pdev,uint32_t mask,char * name)1348 pcim_iomap_regions_request_all(struct pci_dev *pdev, uint32_t mask, char *name)
1349 {
1350 uint32_t requests, req_mask;
1351 int bar, error;
1352
1353 /* Request all the BARs ("regions") we do not iomap. */
1354 req_mask = ((1 << (PCIR_MAX_BAR_0 + 1)) - 1) & ~mask;
1355 for (bar = requests = 0; requests != req_mask; bar++) {
1356 if ((req_mask & (1 << bar)) == 0)
1357 continue;
1358 error = pci_request_region(pdev, bar, name);
1359 if (error != 0 && error != -ENODEV)
1360 goto err;
1361 requests |= (1 << bar);
1362 }
1363
1364 error = pcim_iomap_regions(pdev, mask, name);
1365 if (error != 0)
1366 goto err;
1367
1368 return (0);
1369
1370 err:
1371 for (bar = PCIR_MAX_BAR_0; bar >= 0; bar--) {
1372 if ((requests & (1 << bar)) != 0)
1373 pci_release_region(pdev, bar);
1374 }
1375
1376 return (-EINVAL);
1377 }
1378
1379 /*
1380 * We cannot simply re-define pci_get_device() as we would normally do
1381 * and then hide it in linux_pci.c as too many semi-native drivers still
1382 * include linux/pci.h and run into the conflict with native PCI. Linux drivers
1383 * using pci_get_device() need to be changed to call linuxkpi_pci_get_device().
1384 */
1385 static inline struct pci_dev *
linuxkpi_pci_get_device(uint16_t vendor,uint16_t device,struct pci_dev * odev)1386 linuxkpi_pci_get_device(uint16_t vendor, uint16_t device, struct pci_dev *odev)
1387 {
1388
1389 return (lkpi_pci_get_device(vendor, device, odev));
1390 }
1391
1392 /* This is a FreeBSD extension so we can use bus_*(). */
1393 static inline void
linuxkpi_pcim_want_to_use_bus_functions(struct pci_dev * pdev)1394 linuxkpi_pcim_want_to_use_bus_functions(struct pci_dev *pdev)
1395 {
1396 pdev->want_iomap_res = true;
1397 }
1398
1399 static inline bool
pci_is_thunderbolt_attached(struct pci_dev * pdev)1400 pci_is_thunderbolt_attached(struct pci_dev *pdev)
1401 {
1402
1403 return (false);
1404 }
1405
1406 static inline void *
pci_platform_rom(struct pci_dev * pdev,size_t * size)1407 pci_platform_rom(struct pci_dev *pdev, size_t *size)
1408 {
1409
1410 return (NULL);
1411 }
1412
1413 static inline void
pci_ignore_hotplug(struct pci_dev * pdev)1414 pci_ignore_hotplug(struct pci_dev *pdev)
1415 {
1416 }
1417
1418 static inline const char *
pci_power_name(pci_power_t state)1419 pci_power_name(pci_power_t state)
1420 {
1421 int pstate = state + 1;
1422
1423 if (pstate >= 0 && pstate < nitems(pci_power_names))
1424 return (pci_power_names[pstate]);
1425 else
1426 return (pci_power_names[0]);
1427 }
1428
1429 static inline int
pcie_get_readrq(struct pci_dev * dev)1430 pcie_get_readrq(struct pci_dev *dev)
1431 {
1432 u16 ctl;
1433
1434 if (pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl))
1435 return (-EINVAL);
1436
1437 return (128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12));
1438 }
1439
1440 static inline bool
pci_is_enabled(struct pci_dev * pdev)1441 pci_is_enabled(struct pci_dev *pdev)
1442 {
1443
1444 return ((pci_read_config(pdev->dev.bsddev, PCIR_COMMAND, 2) &
1445 PCIM_CMD_BUSMASTEREN) != 0);
1446 }
1447
1448 static inline int
pci_wait_for_pending_transaction(struct pci_dev * pdev)1449 pci_wait_for_pending_transaction(struct pci_dev *pdev)
1450 {
1451
1452 return (0);
1453 }
1454
1455 static inline int
pci_assign_resource(struct pci_dev * pdev,int bar)1456 pci_assign_resource(struct pci_dev *pdev, int bar)
1457 {
1458
1459 return (0);
1460 }
1461
1462 static inline int
pci_irq_vector(struct pci_dev * pdev,unsigned int vector)1463 pci_irq_vector(struct pci_dev *pdev, unsigned int vector)
1464 {
1465
1466 if (!pdev->msix_enabled && !pdev->msi_enabled) {
1467 if (vector != 0)
1468 return (-EINVAL);
1469 return (pdev->irq);
1470 }
1471
1472 if (pdev->msix_enabled || pdev->msi_enabled) {
1473 if ((pdev->dev.irq_start + vector) >= pdev->dev.irq_end)
1474 return (-EINVAL);
1475 return (pdev->dev.irq_start + vector);
1476 }
1477
1478 return (-ENXIO);
1479 }
1480
1481 #endif /* _LINUXKPI_LINUX_PCI_H_ */
1482