1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 * Copyright 2024 Oxide Computer Company
26 */
27
28 /*
29 * PCIe Initialization
30 * -------------------
31 *
32 * The PCIe subsystem is split about and initializes itself in a couple of
33 * different places. This is due to the platform-specific nature of initializing
34 * resources and the nature of the SPARC PROM and how that influenced the
35 * subsystem. Note that traditional PCI (mostly seen these days in Virtual
36 * Machines) follows most of the same basic path outlined here, but skips a
37 * large chunk of PCIe-specific initialization.
38 *
39 * First, there is an initial device discovery phase that is taken care of by
40 * the platform. This is where we discover the set of devices that are present
41 * at system power on. These devices may or may not be hot-pluggable. In
42 * particular, this happens in a platform-specific way right now. In general, we
43 * expect most discovery to be driven by scanning each bus, device, and
44 * function, and seeing what actually exists and responds to configuration space
45 * reads. This is driven via pci_boot.c on x86. This may be seeded by something
46 * like device tree, a PROM, supplemented with ACPI, or by knowledge that the
47 * underlying platform has.
48 *
49 * As a part of this discovery process, the full set of resources that exist in
50 * the system for PCIe are:
51 *
52 * o PCI buses
53 * o Prefetchable Memory
54 * o Non-prefetchable memory
55 * o I/O ports
56 *
57 * This process is driven by a platform's PCI platform Resource Discovery (PRD)
58 * module. The PRD definitions can be found in <sys/plat/pci_prd.h> and are used
59 * to discover these resources, which will be converted into the initial set of
60 * the standard properties in the system: 'regs', 'available', 'ranges', etc.
61 * Currently it is up to platform-specific code (which should ideally be
62 * consolidated at some point) to set up all these properties.
63 *
64 * As a part of the discovery process, the platform code will create a device
65 * node (dev_info_t) for each discovered function and will create a PCIe nexus
66 * for each overall root complex that exists in the system. Most root complexes
67 * will have multiple root ports, each of which is the foundation of an
68 * independent PCIe bus due to the point-to-point nature of PCIe. When a root
69 * complex is found, a nexus driver such as npe (Nexus for PCIe Express) is
70 * attached. In the case of a non-PCIe-capable system this is where the older
71 * pci nexus driver would be used instead.
72 *
73 * To track data about a given device on a bus, a 'pcie_bus_t' structure is
74 * created for and assigned to every PCIe-based dev_info_t. This can be used to
75 * find the root port and get basic information about the device, its faults,
76 * and related information. This contains pointers to the corresponding root
77 * port as well.
78 *
79 * A root complex has its pcie_bus_t initialized as part of the device discovery
80 * process. That is, because we're trying to bootstrap the actual tree and most
81 * platforms don't have a representation for this that's explicitly
82 * discoverable, this is created manually. See callers of pcie_rc_init_bus().
83 *
84 * For other devices, bridges, and switches, the process is split into two.
85 * There is an initial pcie_bus_t that is created which will exist before we go
86 * through the actual driver attachment process. For example, on x86 this is
87 * done as part of the device and function discovery. The second pass of
88 * initialization is done only after the nexus driver actually is attached and
89 * it goes through and finishes processing all of its children.
90 *
91 * Child Initialization
92 * --------------------
93 *
94 * Generally speaking, the platform will first enumerate all PCIe devices that
95 * are in the sytem before it actually creates a device tree. This is part of
96 * the bus/device/function scanning that is performed and from that dev_info_t
97 * nodes are created for each discovered device and are inserted into the
98 * broader device tree. Later in boot, the actual device tree is walked and the
99 * nodes go through the standard dev_info_t initialization process (DS_PROTO,
100 * DS_LINKED, DS_BOUND, etc.).
101 *
102 * PCIe-specific initialization can roughly be broken into the following pieces:
103 *
104 * 1. Platform initial discovery and resource assignment
105 * 2. The pcie_bus_t initialization
106 * 3. Nexus driver child initialization
107 * 4. Fabric initialization
108 * 5. Device driver-specific initialization
109 *
110 * The first part of this (1) and (2) are discussed in the previous section.
111 * Part (1) in particular is a combination of the PRD (platform resource
112 * discovery) and general device initialization. After this, because we have a
113 * device tree, most of the standard nexus initialization happens.
114 *
115 * (5) is somewhat simple, so let's get into it before we discuss (3) and (4).
116 * This is the last thing that is called and that happens after all of the
117 * others are done. This is the logic that occurs in a driver's attach(9E) entry
118 * point. This is always device-specific and generally speaking should not be
119 * manipulating standard PCIe registers directly on their own. For example, the
120 * MSI/MSI-X, AER, Serial Number, etc. capabilities will be automatically dealt
121 * with by the framework in (3) and (4) below. In many cases, particularly
122 * things that are part of (4), adjusting them in the individual driver is not
123 * safe.
124 *
125 * Finally, let's talk about (3) and (4) as these are related. The NDI provides
126 * for a standard hook for a nexus to initialize its children. In our platforms,
127 * there are basically two possible PCIe nexus drivers: there is the generic
128 * pcieb -- PCIe bridge -- driver which is used for standard root ports,
129 * switches, etc. Then there is the platform-specific primary nexus driver,
130 * which is being slowly consolidated into a single one where it makes sense. An
131 * example of this is npe.
132 *
133 * Each of these has a child initialization function which is called from their
134 * DDI_CTLOPS_INITCHILD operation on the bus_ctl function pointer. This goes
135 * through and initializes a large number of different pieces of PCIe-based
136 * settings through the common pcie_initchild() function. This takes care of
137 * things like:
138 *
139 * o Advanced Error Reporting
140 * o Alternative Routing
141 * o Capturing information around link speed, width, serial numbers, etc.
142 * o Setting common properties around aborts
143 *
144 * There are a few caveats with this that need to be kept in mind:
145 *
146 * o A dev_info_t indicates a specific function. This means that a
147 * multi-function device will not all be initialized at the same time and
148 * there is no guarantee that all children will be initialized before one of
149 * them is attached.
150 * o A child is only initialized if we have found a driver that matches an
151 * alias in the dev_info_t's compatible array property. While a lot of
152 * multi-function devices are often multiple instances of the same thing
153 * (e.g. a multi-port NIC with a function / NIC), this is not always the
154 * case and one cannot make any assumptions here.
155 *
156 * This in turn leads to the next form of initialization that takes place in the
157 * case of (4). This is where we take care of things that need to be consistent
158 * across either entire devices or more generally across an entire root port and
159 * all of its children. There are a few different examples of this:
160 *
161 * o Setting the maximum packet size
162 * o Determining the tag width
163 *
164 * Note that features which are only based on function 0, such as ASPM (Active
165 * State Power Management), hardware autonomous width disable, etc. ultimately
166 * do not go through this path today. There are some implications here in that
167 * today several of these things are captured on functions which may not have
168 * any control here. This is an area of needed improvement.
169 *
170 * The settings in (4) are initialized in a common way, via
171 * pcie_fabric_setup(). This is called into from two different parts of
172 * the stack:
173 *
174 * 1. When we attach a root port, which is driven by pcieb.
175 * 2. When we have a hotplug event that adds a device.
176 *
177 * In general here we are going to use the term 'fabric' to refer to everything
178 * that is downstream of a root port. This corresponds to what the PCIe
179 * specification calls a 'hierarchy domain'. Strictly speaking, this is fine
180 * until peer-to-peer requests begin to happen that cause you to need to forward
181 * things across root ports. At that point the scope of the fabric increases and
182 * these settings become more complicated. We currently optimize for the much
183 * more common case, which is that each root port is effectively independent
184 * from a PCIe transaction routing perspective.
185 *
186 * Put differently, we use the term 'fabric' to refer to a set of PCIe devices
187 * that can route transactions to one another, which is generally constrained to
188 * everything under a root port and that root ports are independent. If this
189 * constraint changes, then all one needs to do is replace the discussion of the
190 * root port below with the broader root complex and system.
191 *
192 * A challenge with these settings is that once they're set and devices are
193 * actively making requests, we cannot really change them without resetting the
194 * links and cancelling all outstanding transactions via device resets. Because
195 * this is not something that we want to do, we instead look at how and when we
196 * set this to constrain what's going on.
197 *
198 * Because of this we basically say that if a given fabric has more than one
199 * hot-plug capable device that's encountered, then we have to use safe defaults
200 * (which we can allow an operator to tune eventually via pcieadm). If we have a
201 * mix of non-hotpluggable slots with downstream endpoints present and
202 * hot-pluggable slots, then we're in this case. If we don't have hot-pluggable
203 * slots, then we can have an arbitrarily complex setup. Let's look at a few of
204 * these visually:
205 *
206 * In the following diagrams, RP stands for Root Port, EP stands for Endpoint.
207 * If something is hot-pluggable, then we label it with (HP).
208 *
209 * (1) RP --> EP
210 * (2) RP --> Switch --> EP
211 * +--> EP
212 * +--> EP
213 *
214 * (3) RP --> Switch --> EP
215 * +--> EP
216 * +--> Switch --> EP
217 * +--> EP
218 * +--> EP
219 *
220 *
221 * (4) RP (HP) --> EP
222 * (5) RP (HP) --> Switch --> EP
223 * +--> EP
224 * +--> EP
225 *
226 * (6) RP --> Switch (HP) --> EP
227 * (7) RP (HP) --> Switch (HP) --> EP
228 *
229 * If we look at all of these, these are all cases where it's safe for us to set
230 * things based on all devices. (1), (2), and (3) are straightforward because
231 * they have no hot-pluggable elements. This means that nothing should come/go
232 * on the system and we can set up fabric-wide properties as part of the root
233 * port.
234 *
235 * Case (4) is the most standard one that we encounter for hot-plug. Here you
236 * have a root port directly connected to an endpoint. The most common example
237 * would be an NVMe device plugged into a root port. Case (5) is interesting to
238 * highlight. While there is a switch and multiple endpoints there, they are
239 * showing up as a unit. This ends up being a weirder variant of (4), but it is
240 * safe for us to set advanced properties because we can figure out what the
241 * total set should be.
242 *
243 * Now, the more interesting bits here are (6) and (7). The reason that (6)
244 * works is that ultimately there is only a single down-stream port here that is
245 * hot-pluggable and all non-hotpluggable ports do not have a device present,
246 * which suggests that they will never have a device present. (7) also could be
247 * made to work by making the observation that if there's truly only one
248 * endpoint in a fabric, it doesn't matter how many switches there are that are
249 * hot-pluggable. This would only hold if we can assume for some reason that no
250 * other endpoints could be added.
251 *
252 * In turn, let's look at several cases that we believe aren't safe:
253 *
254 * (8) RP --> Switch --> EP
255 * +--> EP
256 * (HP) +--> EP
257 *
258 * (9) RP --> Switch (HP) +--> EP
259 * (HP) +--> EP
260 *
261 * (10) RP (HP) --> Switch (HP) +--> EP
262 * (HP) +--> EP
263 *
264 * All of these are situations where it's much more explicitly unsafe. Let's
265 * take (8). The problem here is that the devices on the non-hotpluggable
266 * downstream switches are always there and we should assume all device drivers
267 * will be active and performing I/O when the hot-pluggable slot changes. If the
268 * hot-pluggable slot has a lower max payload size, then we're mostly out of
269 * luck. The case of (9) is very similar to (8), just that we have more hot-plug
270 * capable slots.
271 *
272 * Finally (10) is a case of multiple instances of hotplug. (9) and (10) are the
273 * more general case of (6) and (7). While we can try to detect (6) and (7) more
274 * generally or try to make it safe, we're going to start with a simpler form of
275 * detection for this, which roughly follows the following rules:
276 *
277 * o If there are no hot-pluggable slots in an entire fabric, then we can set
278 * all fabric properties based on device capabilities.
279 * o If we encounter a hot-pluggable slot, we can only set fabric properties
280 * based on device capabilities if:
281 *
282 * 1. The hotpluggable slot is a root port.
283 * 2. There are no other hotpluggable devices downstream of it.
284 *
285 * Otherwise, if neither of the above is true, then we must use the basic PCIe
286 * defaults for various fabric-wide properties (discussed below). Even in these
287 * more complicated cases, device-specific properties such as the configuration
288 * of AERs, ASPM, etc. are still handled in the general pcie_init_bus() and
289 * related discussed earlier here.
290 *
291 * Because the only fabrics that we'll change are those that correspond to root
292 * ports, we will only call into the actual fabric feature setup when one of
293 * those changes. This has the side effect of simplifying locking. When we make
294 * changes here we need to be able to hold the entire device tree under the root
295 * port (including the root port and its parent). This is much harder to do
296 * safely when starting in the middle of the tree.
297 *
298 * Handling of Specific Properties
299 * -------------------------------
300 *
301 * This section goes into the rationale behind how we initialize and program
302 * various parts of the PCIe stack.
303 *
304 * 5-, 8-, 10- AND 14-BIT TAGS
305 *
306 * Tags are part of PCIe transactions and when combined with a device identifier
307 * are used to uniquely identify a transaction. In PCIe parlance, a Requester
308 * (someone who initiates a PCIe request) sets a unique tag in the request and
309 * the Completer (someone who processes and responds to a PCIe request) echoes
310 * the tag back. This means that a requester generally is responsible for
311 * ensuring that they don't reuse a tag between transactions.
312 *
313 * Thus the number of tags that a device has relates to the number of
314 * outstanding transactions that it can have, which are usually tied to the
315 * number of outstanding DMA transfers. The size of these transactions is also
316 * then scoped by the handling of the Maximum Packet Payload.
317 *
318 * In PCIe 1.0, devices default to a 5-bit tag. There was also an option to
319 * support an 8-bit tag. The 8-bit extended tag did not distinguish between a
320 * Requester or Completer. There was a bit to indicate device support of 8-bit
321 * tags in the Device Capabilities Register of the PCIe Capability and a
322 * separate bit to enable it in the Device Control Register of the PCIe
323 * Capability.
324 *
325 * In PCIe 4.0, support for a 10-bit tag was added. The specification broke
326 * apart the support bit into multiple pieces. In particular, in the Device
327 * Capabilities 2 register of the PCIe Capability there is a separate bit to
328 * indicate whether the device supports 10-bit completions and 10-bit requests.
329 * All PCIe 4.0 compliant devices are required to support 10-bit tags if they
330 * operate at 16.0 GT/s speed (a PCIe Gen 4 compliant device does not have to
331 * operate at Gen 4 speeds).
332 *
333 * This allows a device to support 10-bit completions but not 10-bit requests.
334 * A device that supports 10-bit requests is required to support 10-bit
335 * completions. There is no ability to enable or disable 10-bit completion
336 * support in the Device Capabilities 2 register. There is only a bit to enable
337 * 10-bit requests. This distinction makes our life easier as this means that as
338 * long as the entire fabric supports 10-bit completions, it doesn't matter if
339 * not all devices support 10-bit requests and we can enable them as required.
340 * More on this in a bit.
341 *
342 * In PCIe 6.0, another set of bits was added for 14-bit tags. These follow the
343 * same pattern as the 10-bit tags. The biggest difference is that the
344 * capabilities and control for these are found in the Device Capabilities 3
345 * and Device Control 3 register of the Device 3 Extended Capability. Similar to
346 * what we see with 10-bit tags, requesters are required to support the
347 * completer capability. The only control bit is for whether or not they enable
348 * a 14-bit requester.
349 *
350 * PCIe switches which sit between root ports and endpoints and show up to
351 * software as a set of bridges. Bridges generally don't have to know about tags
352 * as they are usually neither requesters or completers (unless directly talking
353 * to the bridge instance). That is they are generally required to forward
354 * packets without modifying them. This works until we deal with switch error
355 * handling. At that point, the switch may try to interpret the transaction and
356 * if it doesn't understand the tagging scheme in use, return the transaction to
357 * with the wrong tag and also an incorrectly diagnosed error (usually a
358 * malformed TLP).
359 *
360 * With all this, we construct a somewhat simple policy of how and when we
361 * enable extended tags:
362 *
363 * o If we have a complex hotplug-capable fabric (based on the discussion
364 * earlier in fabric-specific settings), then we cannot enable any of the
365 * 8-bit, 10-bit, and 14-bit tagging features. This is due to the issues
366 * with intermediate PCIe switches and related.
367 *
368 * o If every device supports 8-bit capable tags, then we will go through and
369 * enable those everywhere.
370 *
371 * o If every device supports 10-bit capable completions, then we will enable
372 * 10-bit requester on every device that supports it.
373 *
374 * o If every device supports 14-bit capable completions, then we will enable
375 * 14-bit requesters on every device that supports it.
376 *
377 * This is the simpler end of the policy and one that is relatively easy to
378 * implement. While we could attempt to relax the constraint that every device
379 * in the fabric implement these features by making assumptions about peer-to-
380 * peer requests (that is devices at the same layer in the tree won't talk to
381 * one another), that is a lot of complexity. For now, we leave such an
382 * implementation to those who need it in the future.
383 *
384 * MAX PAYLOAD SIZE
385 *
386 * When performing transactions on the PCIe bus, a given transaction has a
387 * maximum allowed size. This size is called the MPS or 'Maximum Payload Size'.
388 * A given device reports its maximum supported size in the Device Capabilities
389 * register of the PCIe Capability. It is then set in the Device Control
390 * register.
391 *
392 * One of the challenges with this value is that different functions of a device
393 * have independent values, but strictly speaking are required to actually have
394 * the same value programmed in all of them lest device behavior goes awry. When
395 * a device has the ARI (alternative routing ID) capability enabled, then only
396 * function 0 controls the actual payload size.
397 *
398 * The settings for this need to be consistent throughout the fabric. A
399 * Transmitter is not allowed to create a TLP that exceeds its maximum packet
400 * size and a Receiver is not allowed to receive a packet that exceeds its
401 * maximum packet size. In all of these cases, this would result in something
402 * like a malformed TLP error.
403 *
404 * Effectively, this means that everything on a given fabric must have the same
405 * value programmed in its Device Control register for this value. While in the
406 * case of tags, switches generally weren't completers or requesters, here every
407 * device along the path is subject to this. This makes the actual value that we
408 * set throughout the fabric even more important and the constraints of hotplug
409 * even worse to deal with.
410 *
411 * Because a hotplug device can be inserted with any packet size, if we hit
412 * anything other than the simple hotplug cases discussed in the fabric-specific
413 * settings section, then we must use the smallest size of 128 byte payloads.
414 * This is because a device could be plugged in that supports something smaller
415 * than we had otherwise set. If there are other active devices, those could not
416 * be changed without quiescing the entire fabric. As such our algorithm is as
417 * follows:
418 *
419 * 1. Scan the entire fabric, keeping track of the smallest seen MPS in the
420 * Device Capabilities Register.
421 * 2. If we have a complex fabric, program each Device Control register with
422 * a 128 byte maximum payload size, otherwise, program it with the
423 * discovered value.
424 *
425 *
426 * MAX READ REQUEST SIZE
427 *
428 * The maximum read request size (mrrs) is a much more confusing thing when
429 * compared to the maximum payload size counterpart. The maximum payload size
430 * (MPS) above is what restricts the actual size of a TLP. The mrrs value
431 * is used to control part of the behavior of Memory Read Request, which is not
432 * strictly speaking subject to the MPS. A PCIe device is allowed to respond to
433 * a Memory Read Request with less bytes than were actually requested in a
434 * single completion. In general, the default size that a root complex and its
435 * root port will reply to are based around the length of a cache line.
436 *
437 * What this ultimately controls is the number of requests that the Requester
438 * has to make and trades off bandwidth, bus sharing, and related here. For
439 * example, if the maximum read request size is 4 KiB, then the requester would
440 * only issue a single read request asking for 4 KiB. It would still receive
441 * these as multiple packets in units of the MPS. If however, the maximum read
442 * request was only say 512 B, then it would need to make 8 separate requests,
443 * potentially increasing latency. On the other hand, if systems are relying on
444 * total requests for QoS, then it's important to set it to something that's
445 * closer to the actual MPS.
446 *
447 * Traditionally, the OS has not been the most straightforward about this. It's
448 * important to remember that setting this up is also somewhat in the realm of
449 * system firmware. Due to the PCI Firmware specification, the firmware may have
450 * set up a value for not just the MRRS but also the MPS. As such, our logic
451 * basically left the MRRS alone and used whatever the device had there as long
452 * as we weren't shrinking the device's MPS. If we were, then we'd set it to the
453 * MPS. If the device was a root port, then it was just left at a system wide
454 * and PCIe default of 512 bytes.
455 *
456 * If we survey firmware (which isn't easy due to its nature), we have seen most
457 * cases where the firmware just doesn't do anything and leaves it to the
458 * device's default, which is basically just the PCIe default, unless it has a
459 * specific knowledge of something like say wanting to do something for an NVMe
460 * device. The same is generally true of other systems, leaving it at its
461 * default unless otherwise set by a device driver.
462 *
463 * Because this value doesn't really have the same constraints as other fabric
464 * properties, this becomes much simpler and we instead opt to set it as part of
465 * the device node initialization. In addition, there are no real rules about
466 * different functions having different values here as it doesn't really impact
467 * the TLP processing the same way that the MPS does.
468 *
469 * While we should add a fuller way of setting this and allowing operator
470 * override of the MRRS based on things like device class, etc. that is driven
471 * by pcieadm, that is left to the future. For now we opt to that all devices
472 * are kept at their default (512 bytes or whatever firmware left behind) and we
473 * ensure that root ports always have the mrrs set to 512.
474 */
475
476 #include <sys/sysmacros.h>
477 #include <sys/types.h>
478 #include <sys/kmem.h>
479 #include <sys/modctl.h>
480 #include <sys/ddi.h>
481 #include <sys/sunddi.h>
482 #include <sys/sunndi.h>
483 #include <sys/fm/protocol.h>
484 #include <sys/fm/util.h>
485 #include <sys/promif.h>
486 #include <sys/disp.h>
487 #include <sys/stat.h>
488 #include <sys/file.h>
489 #include <sys/pci_cap.h>
490 #include <sys/pci_impl.h>
491 #include <sys/pcie_impl.h>
492 #include <sys/hotplug/pci/pcie_hp.h>
493 #include <sys/hotplug/pci/pciehpc.h>
494 #include <sys/hotplug/pci/pcishpc.h>
495 #include <sys/hotplug/pci/pcicfg.h>
496 #include <sys/pci_cfgacc.h>
497 #include <sys/sysevent.h>
498 #include <sys/sysevent/eventdefs.h>
499 #include <sys/sysevent/pcie.h>
500
501 /* Local functions prototypes */
502 static void pcie_init_pfd(dev_info_t *);
503 static void pcie_fini_pfd(dev_info_t *);
504
505 #ifdef DEBUG
506 uint_t pcie_debug_flags = 0;
507 static void pcie_print_bus(pcie_bus_t *bus_p);
508 void pcie_dbg(char *fmt, ...);
509 #endif /* DEBUG */
510
511 /* Variable to control default PCI-Express config settings */
512 ushort_t pcie_command_default =
513 PCI_COMM_SERR_ENABLE |
514 PCI_COMM_WAIT_CYC_ENAB |
515 PCI_COMM_PARITY_DETECT |
516 PCI_COMM_ME |
517 PCI_COMM_MAE |
518 PCI_COMM_IO;
519
520 /* xxx_fw are bits that are controlled by FW and should not be modified */
521 ushort_t pcie_command_default_fw =
522 PCI_COMM_SPEC_CYC |
523 PCI_COMM_MEMWR_INVAL |
524 PCI_COMM_PALETTE_SNOOP |
525 PCI_COMM_WAIT_CYC_ENAB |
526 0xF800; /* Reserved Bits */
527
528 ushort_t pcie_bdg_command_default_fw =
529 PCI_BCNF_BCNTRL_ISA_ENABLE |
530 PCI_BCNF_BCNTRL_VGA_ENABLE |
531 0xF000; /* Reserved Bits */
532
533 /* PCI-Express Base error defaults */
534 ushort_t pcie_base_err_default =
535 PCIE_DEVCTL_CE_REPORTING_EN |
536 PCIE_DEVCTL_NFE_REPORTING_EN |
537 PCIE_DEVCTL_FE_REPORTING_EN |
538 PCIE_DEVCTL_UR_REPORTING_EN;
539
540 /* PCI-Express Device Control Register */
541 uint16_t pcie_devctl_default = PCIE_DEVCTL_RO_EN |
542 PCIE_DEVCTL_MAX_READ_REQ_512;
543
544 /* PCI-Express AER Root Control Register */
545 #define PCIE_ROOT_SYS_ERR (PCIE_ROOTCTL_SYS_ERR_ON_CE_EN | \
546 PCIE_ROOTCTL_SYS_ERR_ON_NFE_EN | \
547 PCIE_ROOTCTL_SYS_ERR_ON_FE_EN)
548
549 ushort_t pcie_root_ctrl_default =
550 PCIE_ROOTCTL_SYS_ERR_ON_CE_EN |
551 PCIE_ROOTCTL_SYS_ERR_ON_NFE_EN |
552 PCIE_ROOTCTL_SYS_ERR_ON_FE_EN;
553
554 /* PCI-Express Root Error Command Register */
555 ushort_t pcie_root_error_cmd_default =
556 PCIE_AER_RE_CMD_CE_REP_EN |
557 PCIE_AER_RE_CMD_NFE_REP_EN |
558 PCIE_AER_RE_CMD_FE_REP_EN;
559
560 /* ECRC settings in the PCIe AER Control Register */
561 uint32_t pcie_ecrc_value =
562 PCIE_AER_CTL_ECRC_GEN_ENA |
563 PCIE_AER_CTL_ECRC_CHECK_ENA;
564
565 /*
566 * If a particular platform wants to disable certain errors such as UR/MA,
567 * instead of using #defines have the platform's PCIe Root Complex driver set
568 * these masks using the pcie_get_XXX_mask and pcie_set_XXX_mask functions. For
569 * x86 the closest thing to a PCIe root complex driver is NPE. For SPARC the
570 * closest PCIe root complex driver is PX.
571 *
572 * pcie_serr_disable_flag : disable SERR only (in RCR and command reg) x86
573 * systems may want to disable SERR in general. For root ports, enabling SERR
574 * causes NMIs which are not handled and results in a watchdog timeout error.
575 */
576 uint32_t pcie_aer_uce_mask = 0; /* AER UE Mask */
577 uint32_t pcie_aer_ce_mask = 0; /* AER CE Mask */
578 uint32_t pcie_aer_suce_mask = 0; /* AER Secondary UE Mask */
579 uint32_t pcie_serr_disable_flag = 0; /* Disable SERR */
580
581 /* Default severities needed for eversholt. Error handling doesn't care */
582 uint32_t pcie_aer_uce_severity = PCIE_AER_UCE_MTLP | PCIE_AER_UCE_RO | \
583 PCIE_AER_UCE_FCP | PCIE_AER_UCE_SD | PCIE_AER_UCE_DLP | \
584 PCIE_AER_UCE_TRAINING;
585 uint32_t pcie_aer_suce_severity = PCIE_AER_SUCE_SERR_ASSERT | \
586 PCIE_AER_SUCE_UC_ADDR_ERR | PCIE_AER_SUCE_UC_ATTR_ERR | \
587 PCIE_AER_SUCE_USC_MSG_DATA_ERR;
588
589 int pcie_disable_ari = 0;
590
591 /*
592 * On some platforms, such as the AMD B450 chipset, we've seen an odd
593 * relationship between enabling link bandwidth notifications and AERs about
594 * ECRC errors. This provides a mechanism to disable it.
595 */
596 int pcie_disable_lbw = 0;
597
598 /*
599 * Amount of time to wait for an in-progress retraining. The default is to try
600 * 500 times in 10ms chunks, thus a total of 5s.
601 */
602 uint32_t pcie_link_retrain_count = 500;
603 uint32_t pcie_link_retrain_delay_ms = 10;
604
605 taskq_t *pcie_link_tq;
606 kmutex_t pcie_link_tq_mutex;
607
608 static int pcie_link_bw_intr(dev_info_t *);
609 static void pcie_capture_speeds(dev_info_t *);
610
611 dev_info_t *pcie_get_rc_dip(dev_info_t *dip);
612
613 /*
614 * modload support
615 */
616
617 static struct modlmisc modlmisc = {
618 &mod_miscops, /* Type of module */
619 "PCI Express Framework Module"
620 };
621
622 static struct modlinkage modlinkage = {
623 MODREV_1,
624 (void *)&modlmisc,
625 NULL
626 };
627
628 /*
629 * Global Variables needed for a non-atomic version of ddi_fm_ereport_post.
630 * Currently used to send the pci.fabric ereports whose payload depends on the
631 * type of PCI device it is being sent for.
632 */
633 char *pcie_nv_buf;
634 nv_alloc_t *pcie_nvap;
635 nvlist_t *pcie_nvl;
636
637 int
_init(void)638 _init(void)
639 {
640 int rval;
641
642 pcie_nv_buf = kmem_alloc(ERPT_DATA_SZ, KM_SLEEP);
643 pcie_nvap = fm_nva_xcreate(pcie_nv_buf, ERPT_DATA_SZ);
644 pcie_nvl = fm_nvlist_create(pcie_nvap);
645 mutex_init(&pcie_link_tq_mutex, NULL, MUTEX_DRIVER, NULL);
646
647 if ((rval = mod_install(&modlinkage)) != 0) {
648 mutex_destroy(&pcie_link_tq_mutex);
649 fm_nvlist_destroy(pcie_nvl, FM_NVA_RETAIN);
650 fm_nva_xdestroy(pcie_nvap);
651 kmem_free(pcie_nv_buf, ERPT_DATA_SZ);
652 }
653 return (rval);
654 }
655
656 int
_fini()657 _fini()
658 {
659 int rval;
660
661 if ((rval = mod_remove(&modlinkage)) == 0) {
662 if (pcie_link_tq != NULL) {
663 taskq_destroy(pcie_link_tq);
664 }
665 mutex_destroy(&pcie_link_tq_mutex);
666 fm_nvlist_destroy(pcie_nvl, FM_NVA_RETAIN);
667 fm_nva_xdestroy(pcie_nvap);
668 kmem_free(pcie_nv_buf, ERPT_DATA_SZ);
669 }
670 return (rval);
671 }
672
673 int
_info(struct modinfo * modinfop)674 _info(struct modinfo *modinfop)
675 {
676 return (mod_info(&modlinkage, modinfop));
677 }
678
679 /* ARGSUSED */
680 int
pcie_init(dev_info_t * dip,caddr_t arg)681 pcie_init(dev_info_t *dip, caddr_t arg)
682 {
683 int ret = DDI_SUCCESS;
684
685 /*
686 * Our _init function is too early to create a taskq. Create the pcie
687 * link management taskq here now instead.
688 */
689 mutex_enter(&pcie_link_tq_mutex);
690 if (pcie_link_tq == NULL) {
691 pcie_link_tq = taskq_create("pcie_link", 1, minclsyspri, 0, 0,
692 0);
693 }
694 mutex_exit(&pcie_link_tq_mutex);
695
696
697 /*
698 * Create a "devctl" minor node to support DEVCTL_DEVICE_*
699 * and DEVCTL_BUS_* ioctls to this bus.
700 */
701 if ((ret = ddi_create_minor_node(dip, "devctl", S_IFCHR,
702 PCI_MINOR_NUM(ddi_get_instance(dip), PCI_DEVCTL_MINOR),
703 DDI_NT_NEXUS, 0)) != DDI_SUCCESS) {
704 PCIE_DBG("Failed to create devctl minor node for %s%d\n",
705 ddi_driver_name(dip), ddi_get_instance(dip));
706
707 return (ret);
708 }
709
710 if ((ret = pcie_hp_init(dip, arg)) != DDI_SUCCESS) {
711 /*
712 * On some x86 platforms, we observed unexpected hotplug
713 * initialization failures in recent years. The known cause
714 * is a hardware issue: while the problem PCI bridges have
715 * the Hotplug Capable registers set, the machine actually
716 * does not implement the expected ACPI object.
717 *
718 * We don't want to stop PCI driver attach and system boot
719 * just because of this hotplug initialization failure.
720 * Continue with a debug message printed.
721 */
722 PCIE_DBG("%s%d: Failed setting hotplug framework\n",
723 ddi_driver_name(dip), ddi_get_instance(dip));
724
725 #if defined(__sparc)
726 ddi_remove_minor_node(dip, "devctl");
727
728 return (ret);
729 #endif /* defined(__sparc) */
730 }
731
732 return (DDI_SUCCESS);
733 }
734
735 /* ARGSUSED */
736 int
pcie_uninit(dev_info_t * dip)737 pcie_uninit(dev_info_t *dip)
738 {
739 int ret = DDI_SUCCESS;
740
741 if (pcie_ari_is_enabled(dip) == PCIE_ARI_FORW_ENABLED)
742 (void) pcie_ari_disable(dip);
743
744 if ((ret = pcie_hp_uninit(dip)) != DDI_SUCCESS) {
745 PCIE_DBG("Failed to uninitialize hotplug for %s%d\n",
746 ddi_driver_name(dip), ddi_get_instance(dip));
747
748 return (ret);
749 }
750
751 if (pcie_link_bw_supported(dip)) {
752 (void) pcie_link_bw_disable(dip);
753 }
754
755 ddi_remove_minor_node(dip, "devctl");
756
757 return (ret);
758 }
759
760 /*
761 * PCIe module interface for enabling hotplug interrupt.
762 *
763 * It should be called after pcie_init() is done and bus driver's
764 * interrupt handlers have being attached.
765 */
766 int
pcie_hpintr_enable(dev_info_t * dip)767 pcie_hpintr_enable(dev_info_t *dip)
768 {
769 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
770 pcie_hp_ctrl_t *ctrl_p = PCIE_GET_HP_CTRL(dip);
771
772 if (PCIE_IS_PCIE_HOTPLUG_ENABLED(bus_p)) {
773 (void) (ctrl_p->hc_ops.enable_hpc_intr)(ctrl_p);
774 } else if (PCIE_IS_PCI_HOTPLUG_ENABLED(bus_p)) {
775 (void) pcishpc_enable_irqs(ctrl_p);
776 }
777 return (DDI_SUCCESS);
778 }
779
780 /*
781 * PCIe module interface for disabling hotplug interrupt.
782 *
783 * It should be called before pcie_uninit() is called and bus driver's
784 * interrupt handlers is dettached.
785 */
786 int
pcie_hpintr_disable(dev_info_t * dip)787 pcie_hpintr_disable(dev_info_t *dip)
788 {
789 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
790 pcie_hp_ctrl_t *ctrl_p = PCIE_GET_HP_CTRL(dip);
791
792 if (PCIE_IS_PCIE_HOTPLUG_ENABLED(bus_p)) {
793 (void) (ctrl_p->hc_ops.disable_hpc_intr)(ctrl_p);
794 } else if (PCIE_IS_PCI_HOTPLUG_ENABLED(bus_p)) {
795 (void) pcishpc_disable_irqs(ctrl_p);
796 }
797 return (DDI_SUCCESS);
798 }
799
800 /* ARGSUSED */
801 int
pcie_intr(dev_info_t * dip)802 pcie_intr(dev_info_t *dip)
803 {
804 int hp, lbw;
805
806 hp = pcie_hp_intr(dip);
807 lbw = pcie_link_bw_intr(dip);
808
809 if (hp == DDI_INTR_CLAIMED || lbw == DDI_INTR_CLAIMED) {
810 return (DDI_INTR_CLAIMED);
811 }
812
813 return (DDI_INTR_UNCLAIMED);
814 }
815
816 /* ARGSUSED */
817 int
pcie_open(dev_info_t * dip,dev_t * devp,int flags,int otyp,cred_t * credp)818 pcie_open(dev_info_t *dip, dev_t *devp, int flags, int otyp, cred_t *credp)
819 {
820 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
821
822 /*
823 * Make sure the open is for the right file type.
824 */
825 if (otyp != OTYP_CHR)
826 return (EINVAL);
827
828 /*
829 * Handle the open by tracking the device state.
830 */
831 if ((bus_p->bus_soft_state == PCI_SOFT_STATE_OPEN_EXCL) ||
832 ((flags & FEXCL) &&
833 (bus_p->bus_soft_state != PCI_SOFT_STATE_CLOSED))) {
834 return (EBUSY);
835 }
836
837 if (flags & FEXCL)
838 bus_p->bus_soft_state = PCI_SOFT_STATE_OPEN_EXCL;
839 else
840 bus_p->bus_soft_state = PCI_SOFT_STATE_OPEN;
841
842 return (0);
843 }
844
845 /* ARGSUSED */
846 int
pcie_close(dev_info_t * dip,dev_t dev,int flags,int otyp,cred_t * credp)847 pcie_close(dev_info_t *dip, dev_t dev, int flags, int otyp, cred_t *credp)
848 {
849 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
850
851 if (otyp != OTYP_CHR)
852 return (EINVAL);
853
854 bus_p->bus_soft_state = PCI_SOFT_STATE_CLOSED;
855
856 return (0);
857 }
858
859 /* ARGSUSED */
860 int
pcie_ioctl(dev_info_t * dip,dev_t dev,int cmd,intptr_t arg,int mode,cred_t * credp,int * rvalp)861 pcie_ioctl(dev_info_t *dip, dev_t dev, int cmd, intptr_t arg, int mode,
862 cred_t *credp, int *rvalp)
863 {
864 struct devctl_iocdata *dcp;
865 uint_t bus_state;
866 int rv = DDI_SUCCESS;
867
868 /*
869 * We can use the generic implementation for devctl ioctl
870 */
871 switch (cmd) {
872 case DEVCTL_DEVICE_GETSTATE:
873 case DEVCTL_DEVICE_ONLINE:
874 case DEVCTL_DEVICE_OFFLINE:
875 case DEVCTL_BUS_GETSTATE:
876 return (ndi_devctl_ioctl(dip, cmd, arg, mode, 0));
877 default:
878 break;
879 }
880
881 /*
882 * read devctl ioctl data
883 */
884 if (ndi_dc_allochdl((void *)arg, &dcp) != NDI_SUCCESS)
885 return (EFAULT);
886
887 switch (cmd) {
888 case DEVCTL_BUS_QUIESCE:
889 if (ndi_get_bus_state(dip, &bus_state) == NDI_SUCCESS)
890 if (bus_state == BUS_QUIESCED)
891 break;
892 (void) ndi_set_bus_state(dip, BUS_QUIESCED);
893 break;
894 case DEVCTL_BUS_UNQUIESCE:
895 if (ndi_get_bus_state(dip, &bus_state) == NDI_SUCCESS)
896 if (bus_state == BUS_ACTIVE)
897 break;
898 (void) ndi_set_bus_state(dip, BUS_ACTIVE);
899 break;
900 case DEVCTL_BUS_RESET:
901 case DEVCTL_BUS_RESETALL:
902 case DEVCTL_DEVICE_RESET:
903 rv = ENOTSUP;
904 break;
905 default:
906 rv = ENOTTY;
907 }
908
909 ndi_dc_freehdl(dcp);
910 return (rv);
911 }
912
913 /* ARGSUSED */
914 int
pcie_prop_op(dev_t dev,dev_info_t * dip,ddi_prop_op_t prop_op,int flags,char * name,caddr_t valuep,int * lengthp)915 pcie_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op,
916 int flags, char *name, caddr_t valuep, int *lengthp)
917 {
918 if (dev == DDI_DEV_T_ANY)
919 goto skip;
920
921 if (PCIE_IS_HOTPLUG_CAPABLE(dip) &&
922 strcmp(name, "pci-occupant") == 0) {
923 int pci_dev = PCI_MINOR_NUM_TO_PCI_DEVNUM(getminor(dev));
924
925 pcie_hp_create_occupant_props(dip, dev, pci_dev);
926 }
927
928 skip:
929 return (ddi_prop_op(dev, dip, prop_op, flags, name, valuep, lengthp));
930 }
931
932 int
pcie_init_cfghdl(dev_info_t * cdip)933 pcie_init_cfghdl(dev_info_t *cdip)
934 {
935 pcie_bus_t *bus_p;
936 ddi_acc_handle_t eh = NULL;
937
938 bus_p = PCIE_DIP2BUS(cdip);
939 if (bus_p == NULL)
940 return (DDI_FAILURE);
941
942 /* Create an config access special to error handling */
943 if (pci_config_setup(cdip, &eh) != DDI_SUCCESS) {
944 cmn_err(CE_WARN, "Cannot setup config access"
945 " for BDF 0x%x\n", bus_p->bus_bdf);
946 return (DDI_FAILURE);
947 }
948
949 bus_p->bus_cfg_hdl = eh;
950 return (DDI_SUCCESS);
951 }
952
953 void
pcie_fini_cfghdl(dev_info_t * cdip)954 pcie_fini_cfghdl(dev_info_t *cdip)
955 {
956 pcie_bus_t *bus_p = PCIE_DIP2BUS(cdip);
957
958 pci_config_teardown(&bus_p->bus_cfg_hdl);
959 }
960
961 void
pcie_determine_serial(dev_info_t * dip)962 pcie_determine_serial(dev_info_t *dip)
963 {
964 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
965 ddi_acc_handle_t h;
966 uint16_t cap;
967 uchar_t serial[8];
968 uint32_t low, high;
969
970 if (!PCIE_IS_PCIE(bus_p))
971 return;
972
973 h = bus_p->bus_cfg_hdl;
974
975 if ((PCI_CAP_LOCATE(h, PCI_CAP_XCFG_SPC(PCIE_EXT_CAP_ID_SER), &cap)) ==
976 DDI_FAILURE)
977 return;
978
979 high = PCI_XCAP_GET32(h, 0, cap, PCIE_SER_SID_UPPER_DW);
980 low = PCI_XCAP_GET32(h, 0, cap, PCIE_SER_SID_LOWER_DW);
981
982 /*
983 * Here, we're trying to figure out if we had an invalid PCIe read. From
984 * looking at the contents of the value, it can be hard to tell the
985 * difference between a value that has all 1s correctly versus if we had
986 * an error. In this case, we only assume it's invalid if both register
987 * reads are invalid. We also only use 32-bit reads as we're not sure if
988 * all devices will support these as 64-bit reads, while we know that
989 * they'll support these as 32-bit reads.
990 */
991 if (high == PCI_EINVAL32 && low == PCI_EINVAL32)
992 return;
993
994 serial[0] = low & 0xff;
995 serial[1] = (low >> 8) & 0xff;
996 serial[2] = (low >> 16) & 0xff;
997 serial[3] = (low >> 24) & 0xff;
998 serial[4] = high & 0xff;
999 serial[5] = (high >> 8) & 0xff;
1000 serial[6] = (high >> 16) & 0xff;
1001 serial[7] = (high >> 24) & 0xff;
1002
1003 (void) ndi_prop_update_byte_array(DDI_DEV_T_NONE, dip, "pcie-serial",
1004 serial, sizeof (serial));
1005 }
1006
1007 static void
pcie_determine_aspm(dev_info_t * dip)1008 pcie_determine_aspm(dev_info_t *dip)
1009 {
1010 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
1011 uint32_t linkcap;
1012 uint16_t linkctl;
1013
1014 if (!PCIE_IS_PCIE(bus_p))
1015 return;
1016
1017 linkcap = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP);
1018 linkctl = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL);
1019
1020 switch (linkcap & PCIE_LINKCAP_ASPM_SUP_MASK) {
1021 case PCIE_LINKCAP_ASPM_SUP_L0S:
1022 (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
1023 "pcie-aspm-support", "l0s");
1024 break;
1025 case PCIE_LINKCAP_ASPM_SUP_L1:
1026 (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
1027 "pcie-aspm-support", "l1");
1028 break;
1029 case PCIE_LINKCAP_ASPM_SUP_L0S_L1:
1030 (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
1031 "pcie-aspm-support", "l0s,l1");
1032 break;
1033 default:
1034 return;
1035 }
1036
1037 switch (linkctl & PCIE_LINKCTL_ASPM_CTL_MASK) {
1038 case PCIE_LINKCTL_ASPM_CTL_DIS:
1039 (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
1040 "pcie-aspm-state", "disabled");
1041 break;
1042 case PCIE_LINKCTL_ASPM_CTL_L0S:
1043 (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
1044 "pcie-aspm-state", "l0s");
1045 break;
1046 case PCIE_LINKCTL_ASPM_CTL_L1:
1047 (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
1048 "pcie-aspm-state", "l1");
1049 break;
1050 case PCIE_LINKCTL_ASPM_CTL_L0S_L1:
1051 (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
1052 "pcie-aspm-state", "l0s,l1");
1053 break;
1054 }
1055 }
1056
1057 /*
1058 * PCI-Express child device initialization. Note, this only will be called on a
1059 * device or function if we actually attach a device driver to it.
1060 *
1061 * This function enables generic pci-express interrupts and error handling.
1062 * Note, tagging, the max packet size, and related are all set up before this
1063 * point and is performed in pcie_fabric_setup().
1064 *
1065 * @param pdip root dip (root nexus's dip)
1066 * @param cdip child's dip (device's dip)
1067 * @return DDI_SUCCESS or DDI_FAILURE
1068 */
1069 /* ARGSUSED */
1070 int
pcie_initchild(dev_info_t * cdip)1071 pcie_initchild(dev_info_t *cdip)
1072 {
1073 uint16_t tmp16, reg16;
1074 pcie_bus_t *bus_p;
1075 uint32_t devid, venid;
1076
1077 bus_p = PCIE_DIP2BUS(cdip);
1078 if (bus_p == NULL) {
1079 PCIE_DBG("%s: BUS not found.\n",
1080 ddi_driver_name(cdip));
1081
1082 return (DDI_FAILURE);
1083 }
1084
1085 if (pcie_init_cfghdl(cdip) != DDI_SUCCESS)
1086 return (DDI_FAILURE);
1087
1088 /*
1089 * Update pcie_bus_t with real Vendor Id Device Id.
1090 *
1091 * For assigned devices in IOV environment, the OBP will return
1092 * faked device id/vendor id on configration read and for both
1093 * properties in root domain. translate_devid() function will
1094 * update the properties with real device-id/vendor-id on such
1095 * platforms, so that we can utilize the properties here to get
1096 * real device-id/vendor-id and overwrite the faked ids.
1097 *
1098 * For unassigned devices or devices in non-IOV environment, the
1099 * operation below won't make a difference.
1100 *
1101 * The IOV implementation only supports assignment of PCIE
1102 * endpoint devices. Devices under pci-pci bridges don't need
1103 * operation like this.
1104 */
1105 devid = ddi_prop_get_int(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
1106 "device-id", -1);
1107 venid = ddi_prop_get_int(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
1108 "vendor-id", -1);
1109 bus_p->bus_dev_ven_id = (devid << 16) | (venid & 0xffff);
1110
1111 /* Clear the device's status register */
1112 reg16 = PCIE_GET(16, bus_p, PCI_CONF_STAT);
1113 PCIE_PUT(16, bus_p, PCI_CONF_STAT, reg16);
1114
1115 /* Setup the device's command register */
1116 reg16 = PCIE_GET(16, bus_p, PCI_CONF_COMM);
1117 tmp16 = (reg16 & pcie_command_default_fw) | pcie_command_default;
1118
1119 if (pcie_serr_disable_flag && PCIE_IS_PCIE(bus_p))
1120 tmp16 &= ~PCI_COMM_SERR_ENABLE;
1121
1122 PCIE_PUT(16, bus_p, PCI_CONF_COMM, tmp16);
1123 PCIE_DBG_CFG(cdip, bus_p, "COMMAND", 16, PCI_CONF_COMM, reg16);
1124
1125 /*
1126 * If the device has a bus control register then program it
1127 * based on the settings in the command register.
1128 */
1129 if (PCIE_IS_BDG(bus_p)) {
1130 /* Clear the device's secondary status register */
1131 reg16 = PCIE_GET(16, bus_p, PCI_BCNF_SEC_STATUS);
1132 PCIE_PUT(16, bus_p, PCI_BCNF_SEC_STATUS, reg16);
1133
1134 /* Setup the device's secondary command register */
1135 reg16 = PCIE_GET(16, bus_p, PCI_BCNF_BCNTRL);
1136 tmp16 = (reg16 & pcie_bdg_command_default_fw);
1137
1138 tmp16 |= PCI_BCNF_BCNTRL_SERR_ENABLE;
1139 /*
1140 * Workaround for this Nvidia bridge. Don't enable the SERR
1141 * enable bit in the bridge control register as it could lead to
1142 * bogus NMIs.
1143 */
1144 if (bus_p->bus_dev_ven_id == 0x037010DE)
1145 tmp16 &= ~PCI_BCNF_BCNTRL_SERR_ENABLE;
1146
1147 if (pcie_command_default & PCI_COMM_PARITY_DETECT)
1148 tmp16 |= PCI_BCNF_BCNTRL_PARITY_ENABLE;
1149
1150 /*
1151 * Enable Master Abort Mode only if URs have not been masked.
1152 * For PCI and PCIe-PCI bridges, enabling this bit causes a
1153 * Master Aborts/UR to be forwarded as a UR/TA or SERR. If this
1154 * bit is masked, posted requests are dropped and non-posted
1155 * requests are returned with -1.
1156 */
1157 if (pcie_aer_uce_mask & PCIE_AER_UCE_UR)
1158 tmp16 &= ~PCI_BCNF_BCNTRL_MAST_AB_MODE;
1159 else
1160 tmp16 |= PCI_BCNF_BCNTRL_MAST_AB_MODE;
1161 PCIE_PUT(16, bus_p, PCI_BCNF_BCNTRL, tmp16);
1162 PCIE_DBG_CFG(cdip, bus_p, "SEC CMD", 16, PCI_BCNF_BCNTRL,
1163 reg16);
1164 }
1165
1166 if (PCIE_IS_PCIE(bus_p)) {
1167 /* Setup PCIe device control register */
1168 reg16 = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL);
1169 /* note: MPS/MRRS are initialized in pcie_initchild_mps() */
1170 tmp16 = (reg16 & (PCIE_DEVCTL_MAX_READ_REQ_MASK |
1171 PCIE_DEVCTL_MAX_PAYLOAD_MASK)) |
1172 (pcie_devctl_default & ~(PCIE_DEVCTL_MAX_READ_REQ_MASK |
1173 PCIE_DEVCTL_MAX_PAYLOAD_MASK));
1174 PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL, tmp16);
1175 PCIE_DBG_CAP(cdip, bus_p, "DEVCTL", 16, PCIE_DEVCTL, reg16);
1176
1177 /* Enable PCIe errors */
1178 pcie_enable_errors(cdip);
1179
1180 pcie_determine_serial(cdip);
1181
1182 pcie_determine_aspm(cdip);
1183
1184 pcie_capture_speeds(cdip);
1185 }
1186
1187 bus_p->bus_ari = B_FALSE;
1188 if ((pcie_ari_is_enabled(ddi_get_parent(cdip))
1189 == PCIE_ARI_FORW_ENABLED) && (pcie_ari_device(cdip)
1190 == PCIE_ARI_DEVICE)) {
1191 bus_p->bus_ari = B_TRUE;
1192 }
1193
1194 return (DDI_SUCCESS);
1195 }
1196
1197 static void
pcie_init_pfd(dev_info_t * dip)1198 pcie_init_pfd(dev_info_t *dip)
1199 {
1200 pf_data_t *pfd_p = PCIE_ZALLOC(pf_data_t);
1201 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
1202
1203 PCIE_DIP2PFD(dip) = pfd_p;
1204
1205 pfd_p->pe_bus_p = bus_p;
1206 pfd_p->pe_severity_flags = 0;
1207 pfd_p->pe_severity_mask = 0;
1208 pfd_p->pe_orig_severity_flags = 0;
1209 pfd_p->pe_lock = B_FALSE;
1210 pfd_p->pe_valid = B_FALSE;
1211
1212 /* Allocate the root fault struct for both RC and RP */
1213 if (PCIE_IS_ROOT(bus_p)) {
1214 PCIE_ROOT_FAULT(pfd_p) = PCIE_ZALLOC(pf_root_fault_t);
1215 PCIE_ROOT_FAULT(pfd_p)->scan_bdf = PCIE_INVALID_BDF;
1216 PCIE_ROOT_EH_SRC(pfd_p) = PCIE_ZALLOC(pf_root_eh_src_t);
1217 }
1218
1219 PCI_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_err_regs_t);
1220 PFD_AFFECTED_DEV(pfd_p) = PCIE_ZALLOC(pf_affected_dev_t);
1221 PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf = PCIE_INVALID_BDF;
1222
1223 if (PCIE_IS_BDG(bus_p))
1224 PCI_BDG_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_bdg_err_regs_t);
1225
1226 if (PCIE_IS_PCIE(bus_p)) {
1227 PCIE_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_err_regs_t);
1228
1229 if (PCIE_IS_RP(bus_p))
1230 PCIE_RP_REG(pfd_p) =
1231 PCIE_ZALLOC(pf_pcie_rp_err_regs_t);
1232
1233 PCIE_ADV_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_adv_err_regs_t);
1234 PCIE_ADV_REG(pfd_p)->pcie_ue_tgt_bdf = PCIE_INVALID_BDF;
1235
1236 if (PCIE_IS_RP(bus_p)) {
1237 PCIE_ADV_RP_REG(pfd_p) =
1238 PCIE_ZALLOC(pf_pcie_adv_rp_err_regs_t);
1239 PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ce_src_id =
1240 PCIE_INVALID_BDF;
1241 PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ue_src_id =
1242 PCIE_INVALID_BDF;
1243 } else if (PCIE_IS_PCIE_BDG(bus_p)) {
1244 PCIE_ADV_BDG_REG(pfd_p) =
1245 PCIE_ZALLOC(pf_pcie_adv_bdg_err_regs_t);
1246 PCIE_ADV_BDG_REG(pfd_p)->pcie_sue_tgt_bdf =
1247 PCIE_INVALID_BDF;
1248 }
1249
1250 if (PCIE_IS_PCIE_BDG(bus_p) && PCIE_IS_PCIX(bus_p)) {
1251 PCIX_BDG_ERR_REG(pfd_p) =
1252 PCIE_ZALLOC(pf_pcix_bdg_err_regs_t);
1253
1254 if (PCIX_ECC_VERSION_CHECK(bus_p)) {
1255 PCIX_BDG_ECC_REG(pfd_p, 0) =
1256 PCIE_ZALLOC(pf_pcix_ecc_regs_t);
1257 PCIX_BDG_ECC_REG(pfd_p, 1) =
1258 PCIE_ZALLOC(pf_pcix_ecc_regs_t);
1259 }
1260 }
1261
1262 PCIE_SLOT_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_slot_regs_t);
1263 PCIE_SLOT_REG(pfd_p)->pcie_slot_regs_valid = B_FALSE;
1264 PCIE_SLOT_REG(pfd_p)->pcie_slot_cap = 0;
1265 PCIE_SLOT_REG(pfd_p)->pcie_slot_control = 0;
1266 PCIE_SLOT_REG(pfd_p)->pcie_slot_status = 0;
1267
1268 } else if (PCIE_IS_PCIX(bus_p)) {
1269 if (PCIE_IS_BDG(bus_p)) {
1270 PCIX_BDG_ERR_REG(pfd_p) =
1271 PCIE_ZALLOC(pf_pcix_bdg_err_regs_t);
1272
1273 if (PCIX_ECC_VERSION_CHECK(bus_p)) {
1274 PCIX_BDG_ECC_REG(pfd_p, 0) =
1275 PCIE_ZALLOC(pf_pcix_ecc_regs_t);
1276 PCIX_BDG_ECC_REG(pfd_p, 1) =
1277 PCIE_ZALLOC(pf_pcix_ecc_regs_t);
1278 }
1279 } else {
1280 PCIX_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pcix_err_regs_t);
1281
1282 if (PCIX_ECC_VERSION_CHECK(bus_p))
1283 PCIX_ECC_REG(pfd_p) =
1284 PCIE_ZALLOC(pf_pcix_ecc_regs_t);
1285 }
1286 }
1287 }
1288
1289 static void
pcie_fini_pfd(dev_info_t * dip)1290 pcie_fini_pfd(dev_info_t *dip)
1291 {
1292 pf_data_t *pfd_p = PCIE_DIP2PFD(dip);
1293 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
1294
1295 if (PCIE_IS_PCIE(bus_p)) {
1296 if (PCIE_IS_PCIE_BDG(bus_p) && PCIE_IS_PCIX(bus_p)) {
1297 if (PCIX_ECC_VERSION_CHECK(bus_p)) {
1298 kmem_free(PCIX_BDG_ECC_REG(pfd_p, 0),
1299 sizeof (pf_pcix_ecc_regs_t));
1300 kmem_free(PCIX_BDG_ECC_REG(pfd_p, 1),
1301 sizeof (pf_pcix_ecc_regs_t));
1302 }
1303
1304 kmem_free(PCIX_BDG_ERR_REG(pfd_p),
1305 sizeof (pf_pcix_bdg_err_regs_t));
1306 }
1307
1308 if (PCIE_IS_RP(bus_p))
1309 kmem_free(PCIE_ADV_RP_REG(pfd_p),
1310 sizeof (pf_pcie_adv_rp_err_regs_t));
1311 else if (PCIE_IS_PCIE_BDG(bus_p))
1312 kmem_free(PCIE_ADV_BDG_REG(pfd_p),
1313 sizeof (pf_pcie_adv_bdg_err_regs_t));
1314
1315 kmem_free(PCIE_ADV_REG(pfd_p),
1316 sizeof (pf_pcie_adv_err_regs_t));
1317
1318 if (PCIE_IS_RP(bus_p))
1319 kmem_free(PCIE_RP_REG(pfd_p),
1320 sizeof (pf_pcie_rp_err_regs_t));
1321
1322 kmem_free(PCIE_ERR_REG(pfd_p), sizeof (pf_pcie_err_regs_t));
1323 } else if (PCIE_IS_PCIX(bus_p)) {
1324 if (PCIE_IS_BDG(bus_p)) {
1325 if (PCIX_ECC_VERSION_CHECK(bus_p)) {
1326 kmem_free(PCIX_BDG_ECC_REG(pfd_p, 0),
1327 sizeof (pf_pcix_ecc_regs_t));
1328 kmem_free(PCIX_BDG_ECC_REG(pfd_p, 1),
1329 sizeof (pf_pcix_ecc_regs_t));
1330 }
1331
1332 kmem_free(PCIX_BDG_ERR_REG(pfd_p),
1333 sizeof (pf_pcix_bdg_err_regs_t));
1334 } else {
1335 if (PCIX_ECC_VERSION_CHECK(bus_p))
1336 kmem_free(PCIX_ECC_REG(pfd_p),
1337 sizeof (pf_pcix_ecc_regs_t));
1338
1339 kmem_free(PCIX_ERR_REG(pfd_p),
1340 sizeof (pf_pcix_err_regs_t));
1341 }
1342 }
1343
1344 if (PCIE_IS_BDG(bus_p))
1345 kmem_free(PCI_BDG_ERR_REG(pfd_p),
1346 sizeof (pf_pci_bdg_err_regs_t));
1347
1348 kmem_free(PFD_AFFECTED_DEV(pfd_p), sizeof (pf_affected_dev_t));
1349 kmem_free(PCI_ERR_REG(pfd_p), sizeof (pf_pci_err_regs_t));
1350
1351 if (PCIE_IS_ROOT(bus_p)) {
1352 kmem_free(PCIE_ROOT_FAULT(pfd_p), sizeof (pf_root_fault_t));
1353 kmem_free(PCIE_ROOT_EH_SRC(pfd_p), sizeof (pf_root_eh_src_t));
1354 }
1355
1356 kmem_free(PCIE_DIP2PFD(dip), sizeof (pf_data_t));
1357
1358 PCIE_DIP2PFD(dip) = NULL;
1359 }
1360
1361
1362 /*
1363 * Special functions to allocate pf_data_t's for PCIe root complexes.
1364 * Note: Root Complex not Root Port
1365 */
1366 void
pcie_rc_init_pfd(dev_info_t * dip,pf_data_t * pfd_p)1367 pcie_rc_init_pfd(dev_info_t *dip, pf_data_t *pfd_p)
1368 {
1369 pfd_p->pe_bus_p = PCIE_DIP2DOWNBUS(dip);
1370 pfd_p->pe_severity_flags = 0;
1371 pfd_p->pe_severity_mask = 0;
1372 pfd_p->pe_orig_severity_flags = 0;
1373 pfd_p->pe_lock = B_FALSE;
1374 pfd_p->pe_valid = B_FALSE;
1375
1376 PCIE_ROOT_FAULT(pfd_p) = PCIE_ZALLOC(pf_root_fault_t);
1377 PCIE_ROOT_FAULT(pfd_p)->scan_bdf = PCIE_INVALID_BDF;
1378 PCIE_ROOT_EH_SRC(pfd_p) = PCIE_ZALLOC(pf_root_eh_src_t);
1379 PCI_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_err_regs_t);
1380 PFD_AFFECTED_DEV(pfd_p) = PCIE_ZALLOC(pf_affected_dev_t);
1381 PFD_AFFECTED_DEV(pfd_p)->pe_affected_bdf = PCIE_INVALID_BDF;
1382 PCI_BDG_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pci_bdg_err_regs_t);
1383 PCIE_ERR_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_err_regs_t);
1384 PCIE_RP_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_rp_err_regs_t);
1385 PCIE_ADV_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_adv_err_regs_t);
1386 PCIE_ADV_RP_REG(pfd_p) = PCIE_ZALLOC(pf_pcie_adv_rp_err_regs_t);
1387 PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ce_src_id = PCIE_INVALID_BDF;
1388 PCIE_ADV_RP_REG(pfd_p)->pcie_rp_ue_src_id = PCIE_INVALID_BDF;
1389
1390 PCIE_ADV_REG(pfd_p)->pcie_ue_sev = pcie_aer_uce_severity;
1391 }
1392
1393 void
pcie_rc_fini_pfd(pf_data_t * pfd_p)1394 pcie_rc_fini_pfd(pf_data_t *pfd_p)
1395 {
1396 kmem_free(PCIE_ADV_RP_REG(pfd_p), sizeof (pf_pcie_adv_rp_err_regs_t));
1397 kmem_free(PCIE_ADV_REG(pfd_p), sizeof (pf_pcie_adv_err_regs_t));
1398 kmem_free(PCIE_RP_REG(pfd_p), sizeof (pf_pcie_rp_err_regs_t));
1399 kmem_free(PCIE_ERR_REG(pfd_p), sizeof (pf_pcie_err_regs_t));
1400 kmem_free(PCI_BDG_ERR_REG(pfd_p), sizeof (pf_pci_bdg_err_regs_t));
1401 kmem_free(PFD_AFFECTED_DEV(pfd_p), sizeof (pf_affected_dev_t));
1402 kmem_free(PCI_ERR_REG(pfd_p), sizeof (pf_pci_err_regs_t));
1403 kmem_free(PCIE_ROOT_FAULT(pfd_p), sizeof (pf_root_fault_t));
1404 kmem_free(PCIE_ROOT_EH_SRC(pfd_p), sizeof (pf_root_eh_src_t));
1405 }
1406
1407 /*
1408 * init pcie_bus_t for root complex
1409 *
1410 * Only a few of the fields in bus_t is valid for root complex.
1411 * The fields that are bracketed are initialized in this routine:
1412 *
1413 * dev_info_t * <bus_dip>
1414 * dev_info_t * bus_rp_dip
1415 * ddi_acc_handle_t bus_cfg_hdl
1416 * uint_t <bus_fm_flags>
1417 * pcie_req_id_t bus_bdf
1418 * pcie_req_id_t bus_rp_bdf
1419 * uint32_t bus_dev_ven_id
1420 * uint8_t bus_rev_id
1421 * uint8_t <bus_hdr_type>
1422 * uint16_t <bus_dev_type>
1423 * uint8_t bus_bdg_secbus
1424 * uint16_t bus_pcie_off
1425 * uint16_t <bus_aer_off>
1426 * uint16_t bus_pcix_off
1427 * uint16_t bus_ecc_ver
1428 * pci_bus_range_t bus_bus_range
1429 * ppb_ranges_t * bus_addr_ranges
1430 * int bus_addr_entries
1431 * pci_regspec_t * bus_assigned_addr
1432 * int bus_assigned_entries
1433 * pf_data_t * bus_pfd
1434 * pcie_domain_t * <bus_dom>
1435 * int bus_mps
1436 * uint64_t bus_cfgacc_base
1437 * void * bus_plat_private
1438 */
1439 void
pcie_rc_init_bus(dev_info_t * dip)1440 pcie_rc_init_bus(dev_info_t *dip)
1441 {
1442 pcie_bus_t *bus_p;
1443
1444 bus_p = (pcie_bus_t *)kmem_zalloc(sizeof (pcie_bus_t), KM_SLEEP);
1445 bus_p->bus_dip = dip;
1446 bus_p->bus_dev_type = PCIE_PCIECAP_DEV_TYPE_RC_PSEUDO;
1447 bus_p->bus_hdr_type = PCI_HEADER_ONE;
1448
1449 /* Fake that there are AER logs */
1450 bus_p->bus_aer_off = (uint16_t)-1;
1451
1452 /* Needed only for handle lookup */
1453 atomic_or_uint(&bus_p->bus_fm_flags, PF_FM_READY);
1454
1455 ndi_set_bus_private(dip, B_FALSE, DEVI_PORT_TYPE_PCI, bus_p);
1456
1457 PCIE_BUS2DOM(bus_p) = PCIE_ZALLOC(pcie_domain_t);
1458 }
1459
1460 void
pcie_rc_fini_bus(dev_info_t * dip)1461 pcie_rc_fini_bus(dev_info_t *dip)
1462 {
1463 pcie_bus_t *bus_p = PCIE_DIP2DOWNBUS(dip);
1464 ndi_set_bus_private(dip, B_FALSE, 0, NULL);
1465 kmem_free(PCIE_BUS2DOM(bus_p), sizeof (pcie_domain_t));
1466 kmem_free(bus_p, sizeof (pcie_bus_t));
1467 }
1468
1469 static int
pcie_width_to_int(pcie_link_width_t width)1470 pcie_width_to_int(pcie_link_width_t width)
1471 {
1472 switch (width) {
1473 case PCIE_LINK_WIDTH_X1:
1474 return (1);
1475 case PCIE_LINK_WIDTH_X2:
1476 return (2);
1477 case PCIE_LINK_WIDTH_X4:
1478 return (4);
1479 case PCIE_LINK_WIDTH_X8:
1480 return (8);
1481 case PCIE_LINK_WIDTH_X12:
1482 return (12);
1483 case PCIE_LINK_WIDTH_X16:
1484 return (16);
1485 case PCIE_LINK_WIDTH_X32:
1486 return (32);
1487 default:
1488 return (0);
1489 }
1490 }
1491
1492 /*
1493 * Return the speed in Transfers / second. This is a signed quantity to match
1494 * the ndi/ddi property interfaces.
1495 */
1496 static int64_t
pcie_speed_to_int(pcie_link_speed_t speed)1497 pcie_speed_to_int(pcie_link_speed_t speed)
1498 {
1499 switch (speed) {
1500 case PCIE_LINK_SPEED_2_5:
1501 return (2500000000LL);
1502 case PCIE_LINK_SPEED_5:
1503 return (5000000000LL);
1504 case PCIE_LINK_SPEED_8:
1505 return (8000000000LL);
1506 case PCIE_LINK_SPEED_16:
1507 return (16000000000LL);
1508 case PCIE_LINK_SPEED_32:
1509 return (32000000000LL);
1510 case PCIE_LINK_SPEED_64:
1511 return (64000000000LL);
1512 default:
1513 return (0);
1514 }
1515 }
1516
1517 /*
1518 * Translate the recorded speed information into devinfo properties.
1519 */
1520 static void
pcie_speeds_to_devinfo(dev_info_t * dip,pcie_bus_t * bus_p)1521 pcie_speeds_to_devinfo(dev_info_t *dip, pcie_bus_t *bus_p)
1522 {
1523 if (bus_p->bus_max_width != PCIE_LINK_WIDTH_UNKNOWN) {
1524 (void) ndi_prop_update_int(DDI_DEV_T_NONE, dip,
1525 "pcie-link-maximum-width",
1526 pcie_width_to_int(bus_p->bus_max_width));
1527 }
1528
1529 if (bus_p->bus_cur_width != PCIE_LINK_WIDTH_UNKNOWN) {
1530 (void) ndi_prop_update_int(DDI_DEV_T_NONE, dip,
1531 "pcie-link-current-width",
1532 pcie_width_to_int(bus_p->bus_cur_width));
1533 }
1534
1535 if (bus_p->bus_cur_speed != PCIE_LINK_SPEED_UNKNOWN) {
1536 (void) ndi_prop_update_int64(DDI_DEV_T_NONE, dip,
1537 "pcie-link-current-speed",
1538 pcie_speed_to_int(bus_p->bus_cur_speed));
1539 }
1540
1541 if (bus_p->bus_max_speed != PCIE_LINK_SPEED_UNKNOWN) {
1542 (void) ndi_prop_update_int64(DDI_DEV_T_NONE, dip,
1543 "pcie-link-maximum-speed",
1544 pcie_speed_to_int(bus_p->bus_max_speed));
1545 }
1546
1547 if (bus_p->bus_target_speed != PCIE_LINK_SPEED_UNKNOWN) {
1548 (void) ndi_prop_update_int64(DDI_DEV_T_NONE, dip,
1549 "pcie-link-target-speed",
1550 pcie_speed_to_int(bus_p->bus_target_speed));
1551 }
1552
1553 if ((bus_p->bus_speed_flags & PCIE_LINK_F_ADMIN_TARGET) != 0) {
1554 (void) ndi_prop_create_boolean(DDI_DEV_T_NONE, dip,
1555 "pcie-link-admin-target-speed");
1556 }
1557
1558 if (bus_p->bus_sup_speed != PCIE_LINK_SPEED_UNKNOWN) {
1559 int64_t speeds[PCIE_NSPEEDS];
1560 uint_t nspeeds = 0;
1561
1562 if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_2_5) {
1563 speeds[nspeeds++] =
1564 pcie_speed_to_int(PCIE_LINK_SPEED_2_5);
1565 }
1566
1567 if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_5) {
1568 speeds[nspeeds++] =
1569 pcie_speed_to_int(PCIE_LINK_SPEED_5);
1570 }
1571
1572 if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_8) {
1573 speeds[nspeeds++] =
1574 pcie_speed_to_int(PCIE_LINK_SPEED_8);
1575 }
1576
1577 if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_16) {
1578 speeds[nspeeds++] =
1579 pcie_speed_to_int(PCIE_LINK_SPEED_16);
1580 }
1581
1582 if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_32) {
1583 speeds[nspeeds++] =
1584 pcie_speed_to_int(PCIE_LINK_SPEED_32);
1585 }
1586
1587 if (bus_p->bus_sup_speed & PCIE_LINK_SPEED_64) {
1588 speeds[nspeeds++] =
1589 pcie_speed_to_int(PCIE_LINK_SPEED_64);
1590 }
1591
1592 (void) ndi_prop_update_int64_array(DDI_DEV_T_NONE, dip,
1593 "pcie-link-supported-speeds", speeds, nspeeds);
1594 }
1595 }
1596
1597 /*
1598 * We need to capture the supported, maximum, and current device speed and
1599 * width. The way that this has been done has changed over time.
1600 *
1601 * Prior to PCIe Gen 3, there were only current and supported speed fields.
1602 * These were found in the link status and link capabilities registers of the
1603 * PCI express capability. With the change to PCIe Gen 3, the information in the
1604 * link capabilities changed to the maximum value. The supported speeds vector
1605 * was moved to the link capabilities 2 register.
1606 *
1607 * Now, a device may not implement some of these registers. To determine whether
1608 * or not it's here, we have to do the following. First, we need to check the
1609 * revision of the PCI express capability. The link capabilities 2 register did
1610 * not exist prior to version 2 of this capability. If a modern device does not
1611 * implement it, it is supposed to return zero for the register.
1612 */
1613 static void
pcie_capture_speeds(dev_info_t * dip)1614 pcie_capture_speeds(dev_info_t *dip)
1615 {
1616 uint16_t vers, status;
1617 uint32_t cap, cap2, ctl2;
1618 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
1619 dev_info_t *rcdip;
1620
1621 if (!PCIE_IS_PCIE(bus_p))
1622 return;
1623
1624 rcdip = pcie_get_rc_dip(dip);
1625 if (bus_p->bus_cfg_hdl == NULL) {
1626 vers = pci_cfgacc_get16(rcdip, bus_p->bus_bdf,
1627 bus_p->bus_pcie_off + PCIE_PCIECAP);
1628 } else {
1629 vers = PCIE_CAP_GET(16, bus_p, PCIE_PCIECAP);
1630 }
1631 if (vers == PCI_EINVAL16)
1632 return;
1633 vers &= PCIE_PCIECAP_VER_MASK;
1634
1635 /*
1636 * Verify the capability's version.
1637 */
1638 switch (vers) {
1639 case PCIE_PCIECAP_VER_1_0:
1640 cap2 = 0;
1641 ctl2 = 0;
1642 break;
1643 case PCIE_PCIECAP_VER_2_0:
1644 if (bus_p->bus_cfg_hdl == NULL) {
1645 cap2 = pci_cfgacc_get32(rcdip, bus_p->bus_bdf,
1646 bus_p->bus_pcie_off + PCIE_LINKCAP2);
1647 ctl2 = pci_cfgacc_get16(rcdip, bus_p->bus_bdf,
1648 bus_p->bus_pcie_off + PCIE_LINKCTL2);
1649 } else {
1650 cap2 = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP2);
1651 ctl2 = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL2);
1652 }
1653 if (cap2 == PCI_EINVAL32)
1654 cap2 = 0;
1655 if (ctl2 == PCI_EINVAL16)
1656 ctl2 = 0;
1657 break;
1658 default:
1659 /* Don't try and handle an unknown version */
1660 return;
1661 }
1662
1663 if (bus_p->bus_cfg_hdl == NULL) {
1664 status = pci_cfgacc_get16(rcdip, bus_p->bus_bdf,
1665 bus_p->bus_pcie_off + PCIE_LINKSTS);
1666 cap = pci_cfgacc_get32(rcdip, bus_p->bus_bdf,
1667 bus_p->bus_pcie_off + PCIE_LINKCAP);
1668 } else {
1669 status = PCIE_CAP_GET(16, bus_p, PCIE_LINKSTS);
1670 cap = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP);
1671 }
1672 if (status == PCI_EINVAL16 || cap == PCI_EINVAL32)
1673 return;
1674
1675 mutex_enter(&bus_p->bus_speed_mutex);
1676
1677 switch (status & PCIE_LINKSTS_SPEED_MASK) {
1678 case PCIE_LINKSTS_SPEED_2_5:
1679 bus_p->bus_cur_speed = PCIE_LINK_SPEED_2_5;
1680 break;
1681 case PCIE_LINKSTS_SPEED_5:
1682 bus_p->bus_cur_speed = PCIE_LINK_SPEED_5;
1683 break;
1684 case PCIE_LINKSTS_SPEED_8:
1685 bus_p->bus_cur_speed = PCIE_LINK_SPEED_8;
1686 break;
1687 case PCIE_LINKSTS_SPEED_16:
1688 bus_p->bus_cur_speed = PCIE_LINK_SPEED_16;
1689 break;
1690 case PCIE_LINKSTS_SPEED_32:
1691 bus_p->bus_cur_speed = PCIE_LINK_SPEED_32;
1692 break;
1693 case PCIE_LINKSTS_SPEED_64:
1694 bus_p->bus_cur_speed = PCIE_LINK_SPEED_64;
1695 break;
1696 default:
1697 bus_p->bus_cur_speed = PCIE_LINK_SPEED_UNKNOWN;
1698 break;
1699 }
1700
1701 switch (status & PCIE_LINKSTS_NEG_WIDTH_MASK) {
1702 case PCIE_LINKSTS_NEG_WIDTH_X1:
1703 bus_p->bus_cur_width = PCIE_LINK_WIDTH_X1;
1704 break;
1705 case PCIE_LINKSTS_NEG_WIDTH_X2:
1706 bus_p->bus_cur_width = PCIE_LINK_WIDTH_X2;
1707 break;
1708 case PCIE_LINKSTS_NEG_WIDTH_X4:
1709 bus_p->bus_cur_width = PCIE_LINK_WIDTH_X4;
1710 break;
1711 case PCIE_LINKSTS_NEG_WIDTH_X8:
1712 bus_p->bus_cur_width = PCIE_LINK_WIDTH_X8;
1713 break;
1714 case PCIE_LINKSTS_NEG_WIDTH_X12:
1715 bus_p->bus_cur_width = PCIE_LINK_WIDTH_X12;
1716 break;
1717 case PCIE_LINKSTS_NEG_WIDTH_X16:
1718 bus_p->bus_cur_width = PCIE_LINK_WIDTH_X16;
1719 break;
1720 case PCIE_LINKSTS_NEG_WIDTH_X32:
1721 bus_p->bus_cur_width = PCIE_LINK_WIDTH_X32;
1722 break;
1723 default:
1724 bus_p->bus_cur_width = PCIE_LINK_WIDTH_UNKNOWN;
1725 break;
1726 }
1727
1728 switch (cap & PCIE_LINKCAP_MAX_WIDTH_MASK) {
1729 case PCIE_LINKCAP_MAX_WIDTH_X1:
1730 bus_p->bus_max_width = PCIE_LINK_WIDTH_X1;
1731 break;
1732 case PCIE_LINKCAP_MAX_WIDTH_X2:
1733 bus_p->bus_max_width = PCIE_LINK_WIDTH_X2;
1734 break;
1735 case PCIE_LINKCAP_MAX_WIDTH_X4:
1736 bus_p->bus_max_width = PCIE_LINK_WIDTH_X4;
1737 break;
1738 case PCIE_LINKCAP_MAX_WIDTH_X8:
1739 bus_p->bus_max_width = PCIE_LINK_WIDTH_X8;
1740 break;
1741 case PCIE_LINKCAP_MAX_WIDTH_X12:
1742 bus_p->bus_max_width = PCIE_LINK_WIDTH_X12;
1743 break;
1744 case PCIE_LINKCAP_MAX_WIDTH_X16:
1745 bus_p->bus_max_width = PCIE_LINK_WIDTH_X16;
1746 break;
1747 case PCIE_LINKCAP_MAX_WIDTH_X32:
1748 bus_p->bus_max_width = PCIE_LINK_WIDTH_X32;
1749 break;
1750 default:
1751 bus_p->bus_max_width = PCIE_LINK_WIDTH_UNKNOWN;
1752 break;
1753 }
1754
1755 /*
1756 * If we have the Link Capabilities 2, then we can get the supported
1757 * speeds from it and treat the bits in Link Capabilities 1 as the
1758 * maximum. If we don't, then we need to follow the Implementation Note
1759 * in the standard under Link Capabilities 2. Effectively, this means
1760 * that if the value of 10b is set in Link Capabilities register, that
1761 * it supports both 2.5 and 5 GT/s speeds.
1762 */
1763 if (cap2 != 0) {
1764 if (cap2 & PCIE_LINKCAP2_SPEED_2_5)
1765 bus_p->bus_sup_speed |= PCIE_LINK_SPEED_2_5;
1766 if (cap2 & PCIE_LINKCAP2_SPEED_5)
1767 bus_p->bus_sup_speed |= PCIE_LINK_SPEED_5;
1768 if (cap2 & PCIE_LINKCAP2_SPEED_8)
1769 bus_p->bus_sup_speed |= PCIE_LINK_SPEED_8;
1770 if (cap2 & PCIE_LINKCAP2_SPEED_16)
1771 bus_p->bus_sup_speed |= PCIE_LINK_SPEED_16;
1772 if (cap2 & PCIE_LINKCAP2_SPEED_32)
1773 bus_p->bus_sup_speed |= PCIE_LINK_SPEED_32;
1774 if (cap2 & PCIE_LINKCAP2_SPEED_64)
1775 bus_p->bus_sup_speed |= PCIE_LINK_SPEED_64;
1776
1777 switch (cap & PCIE_LINKCAP_MAX_SPEED_MASK) {
1778 case PCIE_LINKCAP_MAX_SPEED_2_5:
1779 bus_p->bus_max_speed = PCIE_LINK_SPEED_2_5;
1780 break;
1781 case PCIE_LINKCAP_MAX_SPEED_5:
1782 bus_p->bus_max_speed = PCIE_LINK_SPEED_5;
1783 break;
1784 case PCIE_LINKCAP_MAX_SPEED_8:
1785 bus_p->bus_max_speed = PCIE_LINK_SPEED_8;
1786 break;
1787 case PCIE_LINKCAP_MAX_SPEED_16:
1788 bus_p->bus_max_speed = PCIE_LINK_SPEED_16;
1789 break;
1790 case PCIE_LINKCAP_MAX_SPEED_32:
1791 bus_p->bus_max_speed = PCIE_LINK_SPEED_32;
1792 break;
1793 case PCIE_LINKCAP_MAX_SPEED_64:
1794 bus_p->bus_max_speed = PCIE_LINK_SPEED_64;
1795 break;
1796 default:
1797 bus_p->bus_max_speed = PCIE_LINK_SPEED_UNKNOWN;
1798 break;
1799 }
1800 } else {
1801 if (cap & PCIE_LINKCAP_MAX_SPEED_5) {
1802 bus_p->bus_max_speed = PCIE_LINK_SPEED_5;
1803 bus_p->bus_sup_speed = PCIE_LINK_SPEED_2_5 |
1804 PCIE_LINK_SPEED_5;
1805 } else if (cap & PCIE_LINKCAP_MAX_SPEED_2_5) {
1806 bus_p->bus_max_speed = PCIE_LINK_SPEED_2_5;
1807 bus_p->bus_sup_speed = PCIE_LINK_SPEED_2_5;
1808 }
1809 }
1810
1811 switch (ctl2 & PCIE_LINKCTL2_TARGET_SPEED_MASK) {
1812 case PCIE_LINKCTL2_TARGET_SPEED_2_5:
1813 bus_p->bus_target_speed = PCIE_LINK_SPEED_2_5;
1814 break;
1815 case PCIE_LINKCTL2_TARGET_SPEED_5:
1816 bus_p->bus_target_speed = PCIE_LINK_SPEED_5;
1817 break;
1818 case PCIE_LINKCTL2_TARGET_SPEED_8:
1819 bus_p->bus_target_speed = PCIE_LINK_SPEED_8;
1820 break;
1821 case PCIE_LINKCTL2_TARGET_SPEED_16:
1822 bus_p->bus_target_speed = PCIE_LINK_SPEED_16;
1823 break;
1824 case PCIE_LINKCTL2_TARGET_SPEED_32:
1825 bus_p->bus_target_speed = PCIE_LINK_SPEED_32;
1826 break;
1827 case PCIE_LINKCTL2_TARGET_SPEED_64:
1828 bus_p->bus_target_speed = PCIE_LINK_SPEED_64;
1829 break;
1830 default:
1831 bus_p->bus_target_speed = PCIE_LINK_SPEED_UNKNOWN;
1832 break;
1833 }
1834
1835 pcie_speeds_to_devinfo(dip, bus_p);
1836 mutex_exit(&bus_p->bus_speed_mutex);
1837 }
1838
1839 /*
1840 * partially init pcie_bus_t for device (dip,bdf) for accessing pci
1841 * config space
1842 *
1843 * This routine is invoked during boot, either after creating a devinfo node
1844 * (x86 case) or during px driver attach (sparc case); it is also invoked
1845 * in hotplug context after a devinfo node is created.
1846 *
1847 * The fields that are bracketed are initialized if flag PCIE_BUS_INITIAL
1848 * is set:
1849 *
1850 * dev_info_t * <bus_dip>
1851 * dev_info_t * <bus_rp_dip>
1852 * ddi_acc_handle_t bus_cfg_hdl
1853 * uint_t bus_fm_flags
1854 * pcie_req_id_t <bus_bdf>
1855 * pcie_req_id_t <bus_rp_bdf>
1856 * uint32_t <bus_dev_ven_id>
1857 * uint8_t <bus_rev_id>
1858 * uint8_t <bus_hdr_type>
1859 * uint16_t <bus_dev_type>
1860 * uint8_t <bus_bdg_secbus
1861 * uint16_t <bus_pcie_off>
1862 * uint16_t <bus_aer_off>
1863 * uint16_t <bus_pcix_off>
1864 * uint16_t <bus_ecc_ver>
1865 * pci_bus_range_t bus_bus_range
1866 * ppb_ranges_t * bus_addr_ranges
1867 * int bus_addr_entries
1868 * pci_regspec_t * bus_assigned_addr
1869 * int bus_assigned_entries
1870 * pf_data_t * bus_pfd
1871 * pcie_domain_t * bus_dom
1872 * int bus_mps
1873 * uint64_t bus_cfgacc_base
1874 * void * bus_plat_private
1875 *
1876 * The fields that are bracketed are initialized if flag PCIE_BUS_FINAL
1877 * is set:
1878 *
1879 * dev_info_t * bus_dip
1880 * dev_info_t * bus_rp_dip
1881 * ddi_acc_handle_t bus_cfg_hdl
1882 * uint_t bus_fm_flags
1883 * pcie_req_id_t bus_bdf
1884 * pcie_req_id_t bus_rp_bdf
1885 * uint32_t bus_dev_ven_id
1886 * uint8_t bus_rev_id
1887 * uint8_t bus_hdr_type
1888 * uint16_t bus_dev_type
1889 * uint8_t <bus_bdg_secbus>
1890 * uint16_t bus_pcie_off
1891 * uint16_t bus_aer_off
1892 * uint16_t bus_pcix_off
1893 * uint16_t bus_ecc_ver
1894 * pci_bus_range_t <bus_bus_range>
1895 * ppb_ranges_t * <bus_addr_ranges>
1896 * int <bus_addr_entries>
1897 * pci_regspec_t * <bus_assigned_addr>
1898 * int <bus_assigned_entries>
1899 * pf_data_t * <bus_pfd>
1900 * pcie_domain_t * bus_dom
1901 * int bus_mps
1902 * uint64_t bus_cfgacc_base
1903 * void * <bus_plat_private>
1904 */
1905
1906 pcie_bus_t *
pcie_init_bus(dev_info_t * dip,pcie_req_id_t bdf,uint8_t flags)1907 pcie_init_bus(dev_info_t *dip, pcie_req_id_t bdf, uint8_t flags)
1908 {
1909 uint16_t status, base, baseptr, num_cap;
1910 uint32_t capid;
1911 int range_size;
1912 pcie_bus_t *bus_p = NULL;
1913 dev_info_t *rcdip;
1914 dev_info_t *pdip;
1915 const char *errstr = NULL;
1916
1917 if (!(flags & PCIE_BUS_INITIAL))
1918 goto initial_done;
1919
1920 bus_p = kmem_zalloc(sizeof (pcie_bus_t), KM_SLEEP);
1921
1922 bus_p->bus_dip = dip;
1923 bus_p->bus_bdf = bdf;
1924
1925 rcdip = pcie_get_rc_dip(dip);
1926 ASSERT(rcdip != NULL);
1927
1928 /* Save the Vendor ID, Device ID and revision ID */
1929 bus_p->bus_dev_ven_id = pci_cfgacc_get32(rcdip, bdf, PCI_CONF_VENID);
1930 bus_p->bus_rev_id = pci_cfgacc_get8(rcdip, bdf, PCI_CONF_REVID);
1931 /* Save the Header Type */
1932 bus_p->bus_hdr_type = pci_cfgacc_get8(rcdip, bdf, PCI_CONF_HEADER);
1933 bus_p->bus_hdr_type &= PCI_HEADER_TYPE_M;
1934
1935 /*
1936 * Figure out the device type and all the relavant capability offsets
1937 */
1938 /* set default value */
1939 bus_p->bus_dev_type = PCIE_PCIECAP_DEV_TYPE_PCI_PSEUDO;
1940
1941 status = pci_cfgacc_get16(rcdip, bdf, PCI_CONF_STAT);
1942 if (status == PCI_CAP_EINVAL16 || !(status & PCI_STAT_CAP))
1943 goto caps_done; /* capability not supported */
1944
1945 /* Relevant conventional capabilities first */
1946
1947 /* Conventional caps: PCI_CAP_ID_PCI_E, PCI_CAP_ID_PCIX */
1948 num_cap = 2;
1949
1950 switch (bus_p->bus_hdr_type) {
1951 case PCI_HEADER_ZERO:
1952 baseptr = PCI_CONF_CAP_PTR;
1953 break;
1954 case PCI_HEADER_PPB:
1955 baseptr = PCI_BCNF_CAP_PTR;
1956 break;
1957 case PCI_HEADER_CARDBUS:
1958 baseptr = PCI_CBUS_CAP_PTR;
1959 break;
1960 default:
1961 cmn_err(CE_WARN, "%s: unexpected pci header type:%x",
1962 __func__, bus_p->bus_hdr_type);
1963 goto caps_done;
1964 }
1965
1966 base = baseptr;
1967 for (base = pci_cfgacc_get8(rcdip, bdf, base); base && num_cap;
1968 base = pci_cfgacc_get8(rcdip, bdf, base + PCI_CAP_NEXT_PTR)) {
1969 capid = pci_cfgacc_get8(rcdip, bdf, base);
1970 uint16_t pcap;
1971
1972 switch (capid) {
1973 case PCI_CAP_ID_PCI_E:
1974 bus_p->bus_pcie_off = base;
1975 pcap = pci_cfgacc_get16(rcdip, bdf, base +
1976 PCIE_PCIECAP);
1977 bus_p->bus_dev_type = pcap & PCIE_PCIECAP_DEV_TYPE_MASK;
1978 bus_p->bus_pcie_vers = pcap & PCIE_PCIECAP_VER_MASK;
1979
1980 /* Check and save PCIe hotplug capability information */
1981 if ((PCIE_IS_RP(bus_p) || PCIE_IS_SWD(bus_p)) &&
1982 (pci_cfgacc_get16(rcdip, bdf, base + PCIE_PCIECAP)
1983 & PCIE_PCIECAP_SLOT_IMPL) &&
1984 (pci_cfgacc_get32(rcdip, bdf, base + PCIE_SLOTCAP)
1985 & PCIE_SLOTCAP_HP_CAPABLE))
1986 bus_p->bus_hp_sup_modes |= PCIE_NATIVE_HP_MODE;
1987
1988 num_cap--;
1989 break;
1990 case PCI_CAP_ID_PCIX:
1991 bus_p->bus_pcix_off = base;
1992 if (PCIE_IS_BDG(bus_p))
1993 bus_p->bus_ecc_ver =
1994 pci_cfgacc_get16(rcdip, bdf, base +
1995 PCI_PCIX_SEC_STATUS) & PCI_PCIX_VER_MASK;
1996 else
1997 bus_p->bus_ecc_ver =
1998 pci_cfgacc_get16(rcdip, bdf, base +
1999 PCI_PCIX_COMMAND) & PCI_PCIX_VER_MASK;
2000 num_cap--;
2001 break;
2002 default:
2003 break;
2004 }
2005 }
2006
2007 /* Check and save PCI hotplug (SHPC) capability information */
2008 if (PCIE_IS_BDG(bus_p)) {
2009 base = baseptr;
2010 for (base = pci_cfgacc_get8(rcdip, bdf, base);
2011 base; base = pci_cfgacc_get8(rcdip, bdf,
2012 base + PCI_CAP_NEXT_PTR)) {
2013 capid = pci_cfgacc_get8(rcdip, bdf, base);
2014 if (capid == PCI_CAP_ID_PCI_HOTPLUG) {
2015 bus_p->bus_pci_hp_off = base;
2016 bus_p->bus_hp_sup_modes |= PCIE_PCI_HP_MODE;
2017 break;
2018 }
2019 }
2020 }
2021
2022 /* Then, relevant extended capabilities */
2023
2024 if (!PCIE_IS_PCIE(bus_p))
2025 goto caps_done;
2026
2027 /* Extended caps: PCIE_EXT_CAP_ID_AER */
2028 for (base = PCIE_EXT_CAP; base; base = (capid >>
2029 PCIE_EXT_CAP_NEXT_PTR_SHIFT) & PCIE_EXT_CAP_NEXT_PTR_MASK) {
2030 capid = pci_cfgacc_get32(rcdip, bdf, base);
2031 if (capid == PCI_CAP_EINVAL32)
2032 break;
2033 switch ((capid >> PCIE_EXT_CAP_ID_SHIFT) &
2034 PCIE_EXT_CAP_ID_MASK) {
2035 case PCIE_EXT_CAP_ID_AER:
2036 bus_p->bus_aer_off = base;
2037 break;
2038 case PCIE_EXT_CAP_ID_DEV3:
2039 bus_p->bus_dev3_off = base;
2040 break;
2041 }
2042 }
2043
2044 caps_done:
2045 /* save RP dip and RP bdf */
2046 if (PCIE_IS_RP(bus_p)) {
2047 bus_p->bus_rp_dip = dip;
2048 bus_p->bus_rp_bdf = bus_p->bus_bdf;
2049
2050 bus_p->bus_fab = PCIE_ZALLOC(pcie_fabric_data_t);
2051 } else {
2052 for (pdip = ddi_get_parent(dip); pdip;
2053 pdip = ddi_get_parent(pdip)) {
2054 pcie_bus_t *parent_bus_p = PCIE_DIP2BUS(pdip);
2055
2056 /*
2057 * If RP dip and RP bdf in parent's bus_t have
2058 * been initialized, simply use these instead of
2059 * continuing up to the RC.
2060 */
2061 if (parent_bus_p->bus_rp_dip != NULL) {
2062 bus_p->bus_rp_dip = parent_bus_p->bus_rp_dip;
2063 bus_p->bus_rp_bdf = parent_bus_p->bus_rp_bdf;
2064 break;
2065 }
2066
2067 /*
2068 * When debugging be aware that some NVIDIA x86
2069 * architectures have 2 nodes for each RP, One at Bus
2070 * 0x0 and one at Bus 0x80. The requester is from Bus
2071 * 0x80
2072 */
2073 if (PCIE_IS_ROOT(parent_bus_p)) {
2074 bus_p->bus_rp_dip = pdip;
2075 bus_p->bus_rp_bdf = parent_bus_p->bus_bdf;
2076 break;
2077 }
2078 }
2079 }
2080
2081 bus_p->bus_soft_state = PCI_SOFT_STATE_CLOSED;
2082 (void) atomic_swap_uint(&bus_p->bus_fm_flags, 0);
2083
2084 ndi_set_bus_private(dip, B_TRUE, DEVI_PORT_TYPE_PCI, (void *)bus_p);
2085
2086 if (PCIE_IS_HOTPLUG_CAPABLE(dip))
2087 (void) ndi_prop_create_boolean(DDI_DEV_T_NONE, dip,
2088 "hotplug-capable");
2089
2090 initial_done:
2091 if (!(flags & PCIE_BUS_FINAL))
2092 goto final_done;
2093
2094 /* already initialized? */
2095 bus_p = PCIE_DIP2BUS(dip);
2096
2097 /* Save the Range information if device is a switch/bridge */
2098 if (PCIE_IS_BDG(bus_p)) {
2099 /* get "bus_range" property */
2100 range_size = sizeof (pci_bus_range_t);
2101 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2102 "bus-range", (caddr_t)&bus_p->bus_bus_range, &range_size)
2103 != DDI_PROP_SUCCESS) {
2104 errstr = "Cannot find \"bus-range\" property";
2105 cmn_err(CE_WARN,
2106 "PCIE init err info failed BDF 0x%x:%s\n",
2107 bus_p->bus_bdf, errstr);
2108 }
2109
2110 /* get secondary bus number */
2111 rcdip = pcie_get_rc_dip(dip);
2112 ASSERT(rcdip != NULL);
2113
2114 bus_p->bus_bdg_secbus = pci_cfgacc_get8(rcdip,
2115 bus_p->bus_bdf, PCI_BCNF_SECBUS);
2116
2117 /* Get "ranges" property */
2118 if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2119 "ranges", (caddr_t)&bus_p->bus_addr_ranges,
2120 &bus_p->bus_addr_entries) != DDI_PROP_SUCCESS)
2121 bus_p->bus_addr_entries = 0;
2122 bus_p->bus_addr_entries /= sizeof (ppb_ranges_t);
2123 }
2124
2125 /* save "assigned-addresses" property array, ignore failues */
2126 if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2127 "assigned-addresses", (caddr_t)&bus_p->bus_assigned_addr,
2128 &bus_p->bus_assigned_entries) == DDI_PROP_SUCCESS)
2129 bus_p->bus_assigned_entries /= sizeof (pci_regspec_t);
2130 else
2131 bus_p->bus_assigned_entries = 0;
2132
2133 pcie_init_pfd(dip);
2134
2135 pcie_init_plat(dip);
2136
2137 pcie_capture_speeds(dip);
2138
2139 final_done:
2140
2141 PCIE_DBG("Add %s(dip 0x%p, bdf 0x%x, secbus 0x%x)\n",
2142 ddi_driver_name(dip), (void *)dip, bus_p->bus_bdf,
2143 bus_p->bus_bdg_secbus);
2144 #ifdef DEBUG
2145 if (bus_p != NULL) {
2146 pcie_print_bus(bus_p);
2147 }
2148 #endif
2149
2150 return (bus_p);
2151 }
2152
2153 /*
2154 * Invoked before destroying devinfo node, mostly during hotplug
2155 * operation to free pcie_bus_t data structure
2156 */
2157 /* ARGSUSED */
2158 void
pcie_fini_bus(dev_info_t * dip,uint8_t flags)2159 pcie_fini_bus(dev_info_t *dip, uint8_t flags)
2160 {
2161 pcie_bus_t *bus_p = PCIE_DIP2UPBUS(dip);
2162 ASSERT(bus_p);
2163
2164 if (flags & PCIE_BUS_INITIAL) {
2165 pcie_fini_plat(dip);
2166 pcie_fini_pfd(dip);
2167
2168 if (PCIE_IS_RP(bus_p)) {
2169 kmem_free(bus_p->bus_fab, sizeof (pcie_fabric_data_t));
2170 bus_p->bus_fab = NULL;
2171 }
2172
2173 kmem_free(bus_p->bus_assigned_addr,
2174 (sizeof (pci_regspec_t) * bus_p->bus_assigned_entries));
2175 kmem_free(bus_p->bus_addr_ranges,
2176 (sizeof (ppb_ranges_t) * bus_p->bus_addr_entries));
2177 /* zero out the fields that have been destroyed */
2178 bus_p->bus_assigned_addr = NULL;
2179 bus_p->bus_addr_ranges = NULL;
2180 bus_p->bus_assigned_entries = 0;
2181 bus_p->bus_addr_entries = 0;
2182 }
2183
2184 if (flags & PCIE_BUS_FINAL) {
2185 if (PCIE_IS_HOTPLUG_CAPABLE(dip)) {
2186 (void) ndi_prop_remove(DDI_DEV_T_NONE, dip,
2187 "hotplug-capable");
2188 }
2189
2190 ndi_set_bus_private(dip, B_TRUE, 0, NULL);
2191 kmem_free(bus_p, sizeof (pcie_bus_t));
2192 }
2193 }
2194
2195 int
pcie_postattach_child(dev_info_t * cdip)2196 pcie_postattach_child(dev_info_t *cdip)
2197 {
2198 pcie_bus_t *bus_p = PCIE_DIP2BUS(cdip);
2199
2200 if (!bus_p)
2201 return (DDI_FAILURE);
2202
2203 return (pcie_enable_ce(cdip));
2204 }
2205
2206 /*
2207 * PCI-Express child device de-initialization.
2208 * This function disables generic pci-express interrupts and error
2209 * handling.
2210 */
2211 void
pcie_uninitchild(dev_info_t * cdip)2212 pcie_uninitchild(dev_info_t *cdip)
2213 {
2214 pcie_disable_errors(cdip);
2215 pcie_fini_cfghdl(cdip);
2216 pcie_fini_dom(cdip);
2217 }
2218
2219 /*
2220 * find the root complex dip
2221 */
2222 dev_info_t *
pcie_get_rc_dip(dev_info_t * dip)2223 pcie_get_rc_dip(dev_info_t *dip)
2224 {
2225 dev_info_t *rcdip;
2226 pcie_bus_t *rc_bus_p;
2227
2228 for (rcdip = ddi_get_parent(dip); rcdip;
2229 rcdip = ddi_get_parent(rcdip)) {
2230 rc_bus_p = PCIE_DIP2BUS(rcdip);
2231 if (rc_bus_p && PCIE_IS_RC(rc_bus_p))
2232 break;
2233 }
2234
2235 return (rcdip);
2236 }
2237
2238 boolean_t
pcie_is_pci_device(dev_info_t * dip)2239 pcie_is_pci_device(dev_info_t *dip)
2240 {
2241 dev_info_t *pdip;
2242 char *device_type;
2243
2244 pdip = ddi_get_parent(dip);
2245 if (pdip == NULL)
2246 return (B_FALSE);
2247
2248 if (ddi_prop_lookup_string(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
2249 "device_type", &device_type) != DDI_PROP_SUCCESS)
2250 return (B_FALSE);
2251
2252 if (strcmp(device_type, "pciex") != 0 &&
2253 strcmp(device_type, "pci") != 0) {
2254 ddi_prop_free(device_type);
2255 return (B_FALSE);
2256 }
2257
2258 ddi_prop_free(device_type);
2259 return (B_TRUE);
2260 }
2261
2262 typedef struct {
2263 boolean_t init;
2264 uint8_t flags;
2265 } pcie_bus_arg_t;
2266
2267 /*ARGSUSED*/
2268 static int
pcie_fab_do_init_fini(dev_info_t * dip,void * arg)2269 pcie_fab_do_init_fini(dev_info_t *dip, void *arg)
2270 {
2271 pcie_req_id_t bdf;
2272 pcie_bus_arg_t *bus_arg = (pcie_bus_arg_t *)arg;
2273
2274 if (!pcie_is_pci_device(dip))
2275 goto out;
2276
2277 if (bus_arg->init) {
2278 if (pcie_get_bdf_from_dip(dip, &bdf) != DDI_SUCCESS)
2279 goto out;
2280
2281 (void) pcie_init_bus(dip, bdf, bus_arg->flags);
2282 } else {
2283 (void) pcie_fini_bus(dip, bus_arg->flags);
2284 }
2285
2286 return (DDI_WALK_CONTINUE);
2287
2288 out:
2289 return (DDI_WALK_PRUNECHILD);
2290 }
2291
2292 void
pcie_fab_init_bus(dev_info_t * rcdip,uint8_t flags)2293 pcie_fab_init_bus(dev_info_t *rcdip, uint8_t flags)
2294 {
2295 dev_info_t *dip = ddi_get_child(rcdip);
2296 pcie_bus_arg_t arg;
2297
2298 arg.init = B_TRUE;
2299 arg.flags = flags;
2300
2301 ndi_devi_enter(rcdip);
2302 ddi_walk_devs(dip, pcie_fab_do_init_fini, &arg);
2303 ndi_devi_exit(rcdip);
2304 }
2305
2306 void
pcie_fab_fini_bus(dev_info_t * rcdip,uint8_t flags)2307 pcie_fab_fini_bus(dev_info_t *rcdip, uint8_t flags)
2308 {
2309 dev_info_t *dip = ddi_get_child(rcdip);
2310 pcie_bus_arg_t arg;
2311
2312 arg.init = B_FALSE;
2313 arg.flags = flags;
2314
2315 ndi_devi_enter(rcdip);
2316 ddi_walk_devs(dip, pcie_fab_do_init_fini, &arg);
2317 ndi_devi_exit(rcdip);
2318 }
2319
2320 void
pcie_enable_errors(dev_info_t * dip)2321 pcie_enable_errors(dev_info_t *dip)
2322 {
2323 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2324 uint16_t reg16, tmp16;
2325 uint32_t reg32, tmp32;
2326
2327 ASSERT(bus_p);
2328
2329 /*
2330 * Clear any pending errors
2331 */
2332 pcie_clear_errors(dip);
2333
2334 if (!PCIE_IS_PCIE(bus_p))
2335 return;
2336
2337 /*
2338 * Enable Baseline Error Handling but leave CE reporting off (poweron
2339 * default).
2340 */
2341 if ((reg16 = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL)) !=
2342 PCI_CAP_EINVAL16) {
2343 tmp16 = (reg16 & (PCIE_DEVCTL_MAX_READ_REQ_MASK |
2344 PCIE_DEVCTL_MAX_PAYLOAD_MASK)) |
2345 (pcie_devctl_default & ~(PCIE_DEVCTL_MAX_READ_REQ_MASK |
2346 PCIE_DEVCTL_MAX_PAYLOAD_MASK)) |
2347 (pcie_base_err_default & (~PCIE_DEVCTL_CE_REPORTING_EN));
2348
2349 PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL, tmp16);
2350 PCIE_DBG_CAP(dip, bus_p, "DEVCTL", 16, PCIE_DEVCTL, reg16);
2351 }
2352
2353 /* Enable Root Port Baseline Error Receiving */
2354 if (PCIE_IS_ROOT(bus_p) &&
2355 (reg16 = PCIE_CAP_GET(16, bus_p, PCIE_ROOTCTL)) !=
2356 PCI_CAP_EINVAL16) {
2357
2358 tmp16 = pcie_serr_disable_flag ?
2359 (pcie_root_ctrl_default & ~PCIE_ROOT_SYS_ERR) :
2360 pcie_root_ctrl_default;
2361 PCIE_CAP_PUT(16, bus_p, PCIE_ROOTCTL, tmp16);
2362 PCIE_DBG_CAP(dip, bus_p, "ROOT DEVCTL", 16, PCIE_ROOTCTL,
2363 reg16);
2364 }
2365
2366 /*
2367 * Enable PCI-Express Advanced Error Handling if Exists
2368 */
2369 if (!PCIE_HAS_AER(bus_p))
2370 return;
2371
2372 /* Set Uncorrectable Severity */
2373 if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_UCE_SERV)) !=
2374 PCI_CAP_EINVAL32) {
2375 tmp32 = pcie_aer_uce_severity;
2376
2377 PCIE_AER_PUT(32, bus_p, PCIE_AER_UCE_SERV, tmp32);
2378 PCIE_DBG_AER(dip, bus_p, "AER UCE SEV", 32, PCIE_AER_UCE_SERV,
2379 reg32);
2380 }
2381
2382 /* Enable Uncorrectable errors */
2383 if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_UCE_MASK)) !=
2384 PCI_CAP_EINVAL32) {
2385 tmp32 = pcie_aer_uce_mask;
2386
2387 PCIE_AER_PUT(32, bus_p, PCIE_AER_UCE_MASK, tmp32);
2388 PCIE_DBG_AER(dip, bus_p, "AER UCE MASK", 32, PCIE_AER_UCE_MASK,
2389 reg32);
2390 }
2391
2392 /* Enable ECRC generation and checking */
2393 if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_CTL)) !=
2394 PCI_CAP_EINVAL32) {
2395 tmp32 = reg32 | pcie_ecrc_value;
2396 PCIE_AER_PUT(32, bus_p, PCIE_AER_CTL, tmp32);
2397 PCIE_DBG_AER(dip, bus_p, "AER CTL", 32, PCIE_AER_CTL, reg32);
2398 }
2399
2400 /* Enable Secondary Uncorrectable errors if this is a bridge */
2401 if (!PCIE_IS_PCIE_BDG(bus_p))
2402 goto root;
2403
2404 /* Set Uncorrectable Severity */
2405 if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_SUCE_SERV)) !=
2406 PCI_CAP_EINVAL32) {
2407 tmp32 = pcie_aer_suce_severity;
2408
2409 PCIE_AER_PUT(32, bus_p, PCIE_AER_SUCE_SERV, tmp32);
2410 PCIE_DBG_AER(dip, bus_p, "AER SUCE SEV", 32, PCIE_AER_SUCE_SERV,
2411 reg32);
2412 }
2413
2414 if ((reg32 = PCIE_AER_GET(32, bus_p, PCIE_AER_SUCE_MASK)) !=
2415 PCI_CAP_EINVAL32) {
2416 PCIE_AER_PUT(32, bus_p, PCIE_AER_SUCE_MASK, pcie_aer_suce_mask);
2417 PCIE_DBG_AER(dip, bus_p, "AER SUCE MASK", 32,
2418 PCIE_AER_SUCE_MASK, reg32);
2419 }
2420
2421 root:
2422 /*
2423 * Enable Root Control this is a Root device
2424 */
2425 if (!PCIE_IS_ROOT(bus_p))
2426 return;
2427
2428 if ((reg16 = PCIE_AER_GET(16, bus_p, PCIE_AER_RE_CMD)) !=
2429 PCI_CAP_EINVAL16) {
2430 PCIE_AER_PUT(16, bus_p, PCIE_AER_RE_CMD,
2431 pcie_root_error_cmd_default);
2432 PCIE_DBG_AER(dip, bus_p, "AER Root Err Cmd", 16,
2433 PCIE_AER_RE_CMD, reg16);
2434 }
2435 }
2436
2437 /*
2438 * This function is used for enabling CE reporting and setting the AER CE mask.
2439 * When called from outside the pcie module it should always be preceded by
2440 * a call to pcie_enable_errors.
2441 */
2442 int
pcie_enable_ce(dev_info_t * dip)2443 pcie_enable_ce(dev_info_t *dip)
2444 {
2445 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2446 uint16_t device_sts, device_ctl;
2447 uint32_t tmp_pcie_aer_ce_mask;
2448
2449 if (!PCIE_IS_PCIE(bus_p))
2450 return (DDI_SUCCESS);
2451
2452 /*
2453 * The "pcie_ce_mask" property is used to control both the CE reporting
2454 * enable field in the device control register and the AER CE mask. We
2455 * leave CE reporting disabled if pcie_ce_mask is set to -1.
2456 */
2457
2458 tmp_pcie_aer_ce_mask = (uint32_t)ddi_prop_get_int(DDI_DEV_T_ANY, dip,
2459 DDI_PROP_DONTPASS, "pcie_ce_mask", pcie_aer_ce_mask);
2460
2461 if (tmp_pcie_aer_ce_mask == (uint32_t)-1) {
2462 /*
2463 * Nothing to do since CE reporting has already been disabled.
2464 */
2465 return (DDI_SUCCESS);
2466 }
2467
2468 if (PCIE_HAS_AER(bus_p)) {
2469 /* Enable AER CE */
2470 PCIE_AER_PUT(32, bus_p, PCIE_AER_CE_MASK, tmp_pcie_aer_ce_mask);
2471 PCIE_DBG_AER(dip, bus_p, "AER CE MASK", 32, PCIE_AER_CE_MASK,
2472 0);
2473
2474 /* Clear any pending AER CE errors */
2475 PCIE_AER_PUT(32, bus_p, PCIE_AER_CE_STS, -1);
2476 }
2477
2478 /* clear any pending CE errors */
2479 if ((device_sts = PCIE_CAP_GET(16, bus_p, PCIE_DEVSTS)) !=
2480 PCI_CAP_EINVAL16)
2481 PCIE_CAP_PUT(16, bus_p, PCIE_DEVSTS,
2482 device_sts & (~PCIE_DEVSTS_CE_DETECTED));
2483
2484 /* Enable CE reporting */
2485 device_ctl = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL);
2486 PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL,
2487 (device_ctl & (~PCIE_DEVCTL_ERR_MASK)) | pcie_base_err_default);
2488 PCIE_DBG_CAP(dip, bus_p, "DEVCTL", 16, PCIE_DEVCTL, device_ctl);
2489
2490 return (DDI_SUCCESS);
2491 }
2492
2493 /* ARGSUSED */
2494 void
pcie_disable_errors(dev_info_t * dip)2495 pcie_disable_errors(dev_info_t *dip)
2496 {
2497 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2498 uint16_t device_ctl;
2499 uint32_t aer_reg;
2500
2501 if (!PCIE_IS_PCIE(bus_p))
2502 return;
2503
2504 /*
2505 * Disable PCI-Express Baseline Error Handling
2506 */
2507 device_ctl = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL);
2508 device_ctl &= ~PCIE_DEVCTL_ERR_MASK;
2509 PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL, device_ctl);
2510
2511 /*
2512 * Disable PCI-Express Advanced Error Handling if Exists
2513 */
2514 if (!PCIE_HAS_AER(bus_p))
2515 goto root;
2516
2517 /* Disable Uncorrectable errors */
2518 PCIE_AER_PUT(32, bus_p, PCIE_AER_UCE_MASK, PCIE_AER_UCE_BITS);
2519
2520 /* Disable Correctable errors */
2521 PCIE_AER_PUT(32, bus_p, PCIE_AER_CE_MASK, PCIE_AER_CE_BITS);
2522
2523 /* Disable ECRC generation and checking */
2524 if ((aer_reg = PCIE_AER_GET(32, bus_p, PCIE_AER_CTL)) !=
2525 PCI_CAP_EINVAL32) {
2526 aer_reg &= ~(PCIE_AER_CTL_ECRC_GEN_ENA |
2527 PCIE_AER_CTL_ECRC_CHECK_ENA);
2528
2529 PCIE_AER_PUT(32, bus_p, PCIE_AER_CTL, aer_reg);
2530 }
2531 /*
2532 * Disable Secondary Uncorrectable errors if this is a bridge
2533 */
2534 if (!PCIE_IS_PCIE_BDG(bus_p))
2535 goto root;
2536
2537 PCIE_AER_PUT(32, bus_p, PCIE_AER_SUCE_MASK, PCIE_AER_SUCE_BITS);
2538
2539 root:
2540 /*
2541 * disable Root Control this is a Root device
2542 */
2543 if (!PCIE_IS_ROOT(bus_p))
2544 return;
2545
2546 if (!pcie_serr_disable_flag) {
2547 device_ctl = PCIE_CAP_GET(16, bus_p, PCIE_ROOTCTL);
2548 device_ctl &= ~PCIE_ROOT_SYS_ERR;
2549 PCIE_CAP_PUT(16, bus_p, PCIE_ROOTCTL, device_ctl);
2550 }
2551
2552 if (!PCIE_HAS_AER(bus_p))
2553 return;
2554
2555 if ((device_ctl = PCIE_CAP_GET(16, bus_p, PCIE_AER_RE_CMD)) !=
2556 PCI_CAP_EINVAL16) {
2557 device_ctl &= ~pcie_root_error_cmd_default;
2558 PCIE_CAP_PUT(16, bus_p, PCIE_AER_RE_CMD, device_ctl);
2559 }
2560 }
2561
2562 /*
2563 * Extract bdf from "reg" property.
2564 */
2565 int
pcie_get_bdf_from_dip(dev_info_t * dip,pcie_req_id_t * bdf)2566 pcie_get_bdf_from_dip(dev_info_t *dip, pcie_req_id_t *bdf)
2567 {
2568 pci_regspec_t *regspec;
2569 int reglen;
2570
2571 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2572 "reg", (int **)®spec, (uint_t *)®len) != DDI_SUCCESS)
2573 return (DDI_FAILURE);
2574
2575 if (reglen < (sizeof (pci_regspec_t) / sizeof (int))) {
2576 ddi_prop_free(regspec);
2577 return (DDI_FAILURE);
2578 }
2579
2580 /* Get phys_hi from first element. All have same bdf. */
2581 *bdf = (regspec->pci_phys_hi & (PCI_REG_BDFR_M ^ PCI_REG_REG_M)) >> 8;
2582
2583 ddi_prop_free(regspec);
2584 return (DDI_SUCCESS);
2585 }
2586
2587 dev_info_t *
pcie_get_my_childs_dip(dev_info_t * dip,dev_info_t * rdip)2588 pcie_get_my_childs_dip(dev_info_t *dip, dev_info_t *rdip)
2589 {
2590 dev_info_t *cdip = rdip;
2591
2592 for (; ddi_get_parent(cdip) != dip; cdip = ddi_get_parent(cdip))
2593 ;
2594
2595 return (cdip);
2596 }
2597
2598 uint32_t
pcie_get_bdf_for_dma_xfer(dev_info_t * dip,dev_info_t * rdip)2599 pcie_get_bdf_for_dma_xfer(dev_info_t *dip, dev_info_t *rdip)
2600 {
2601 dev_info_t *cdip;
2602
2603 /*
2604 * As part of the probing, the PCI fcode interpreter may setup a DMA
2605 * request if a given card has a fcode on it using dip and rdip of the
2606 * hotplug connector i.e, dip and rdip of px/pcieb driver. In this
2607 * case, return a invalid value for the bdf since we cannot get to the
2608 * bdf value of the actual device which will be initiating this DMA.
2609 */
2610 if (rdip == dip)
2611 return (PCIE_INVALID_BDF);
2612
2613 cdip = pcie_get_my_childs_dip(dip, rdip);
2614
2615 /*
2616 * For a given rdip, return the bdf value of dip's (px or pcieb)
2617 * immediate child or secondary bus-id if dip is a PCIe2PCI bridge.
2618 *
2619 * XXX - For now, return a invalid bdf value for all PCI and PCI-X
2620 * devices since this needs more work.
2621 */
2622 return (PCI_GET_PCIE2PCI_SECBUS(cdip) ?
2623 PCIE_INVALID_BDF : PCI_GET_BDF(cdip));
2624 }
2625
2626 uint32_t
pcie_get_aer_uce_mask()2627 pcie_get_aer_uce_mask()
2628 {
2629 return (pcie_aer_uce_mask);
2630 }
2631 uint32_t
pcie_get_aer_ce_mask()2632 pcie_get_aer_ce_mask()
2633 {
2634 return (pcie_aer_ce_mask);
2635 }
2636 uint32_t
pcie_get_aer_suce_mask()2637 pcie_get_aer_suce_mask()
2638 {
2639 return (pcie_aer_suce_mask);
2640 }
2641 uint32_t
pcie_get_serr_mask()2642 pcie_get_serr_mask()
2643 {
2644 return (pcie_serr_disable_flag);
2645 }
2646
2647 void
pcie_set_aer_uce_mask(uint32_t mask)2648 pcie_set_aer_uce_mask(uint32_t mask)
2649 {
2650 pcie_aer_uce_mask = mask;
2651 if (mask & PCIE_AER_UCE_UR)
2652 pcie_base_err_default &= ~PCIE_DEVCTL_UR_REPORTING_EN;
2653 else
2654 pcie_base_err_default |= PCIE_DEVCTL_UR_REPORTING_EN;
2655
2656 if (mask & PCIE_AER_UCE_ECRC)
2657 pcie_ecrc_value = 0;
2658 }
2659
2660 void
pcie_set_aer_ce_mask(uint32_t mask)2661 pcie_set_aer_ce_mask(uint32_t mask)
2662 {
2663 pcie_aer_ce_mask = mask;
2664 }
2665 void
pcie_set_aer_suce_mask(uint32_t mask)2666 pcie_set_aer_suce_mask(uint32_t mask)
2667 {
2668 pcie_aer_suce_mask = mask;
2669 }
2670 void
pcie_set_serr_mask(uint32_t mask)2671 pcie_set_serr_mask(uint32_t mask)
2672 {
2673 pcie_serr_disable_flag = mask;
2674 }
2675
2676 /*
2677 * Is the rdip a child of dip. Used for checking certain CTLOPS from bubbling
2678 * up erronously. Ex. ISA ctlops to a PCI-PCI Bridge.
2679 */
2680 boolean_t
pcie_is_child(dev_info_t * dip,dev_info_t * rdip)2681 pcie_is_child(dev_info_t *dip, dev_info_t *rdip)
2682 {
2683 dev_info_t *cdip = ddi_get_child(dip);
2684 for (; cdip; cdip = ddi_get_next_sibling(cdip))
2685 if (cdip == rdip)
2686 break;
2687 return (cdip != NULL);
2688 }
2689
2690 boolean_t
pcie_is_link_disabled(dev_info_t * dip)2691 pcie_is_link_disabled(dev_info_t *dip)
2692 {
2693 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2694
2695 if (PCIE_IS_PCIE(bus_p)) {
2696 if (PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL) &
2697 PCIE_LINKCTL_LINK_DISABLE)
2698 return (B_TRUE);
2699 }
2700 return (B_FALSE);
2701 }
2702
2703 /*
2704 * Determines if there are any root ports attached to a root complex.
2705 *
2706 * dip - dip of root complex
2707 *
2708 * Returns - DDI_SUCCESS if there is at least one root port otherwise
2709 * DDI_FAILURE.
2710 */
2711 int
pcie_root_port(dev_info_t * dip)2712 pcie_root_port(dev_info_t *dip)
2713 {
2714 int port_type;
2715 uint16_t cap_ptr;
2716 ddi_acc_handle_t config_handle;
2717 dev_info_t *cdip = ddi_get_child(dip);
2718
2719 /*
2720 * Determine if any of the children of the passed in dip
2721 * are root ports.
2722 */
2723 for (; cdip; cdip = ddi_get_next_sibling(cdip)) {
2724
2725 if (pci_config_setup(cdip, &config_handle) != DDI_SUCCESS)
2726 continue;
2727
2728 if ((PCI_CAP_LOCATE(config_handle, PCI_CAP_ID_PCI_E,
2729 &cap_ptr)) == DDI_FAILURE) {
2730 pci_config_teardown(&config_handle);
2731 continue;
2732 }
2733
2734 port_type = PCI_CAP_GET16(config_handle, 0, cap_ptr,
2735 PCIE_PCIECAP) & PCIE_PCIECAP_DEV_TYPE_MASK;
2736
2737 pci_config_teardown(&config_handle);
2738
2739 if (port_type == PCIE_PCIECAP_DEV_TYPE_ROOT)
2740 return (DDI_SUCCESS);
2741 }
2742
2743 /* No root ports were found */
2744
2745 return (DDI_FAILURE);
2746 }
2747
2748 /*
2749 * Function that determines if a device a PCIe device.
2750 *
2751 * dip - dip of device.
2752 *
2753 * returns - DDI_SUCCESS if device is a PCIe device, otherwise DDI_FAILURE.
2754 */
2755 int
pcie_dev(dev_info_t * dip)2756 pcie_dev(dev_info_t *dip)
2757 {
2758 /* get parent device's device_type property */
2759 char *device_type;
2760 int rc = DDI_FAILURE;
2761 dev_info_t *pdip = ddi_get_parent(dip);
2762
2763 if (ddi_prop_lookup_string(DDI_DEV_T_ANY, pdip,
2764 DDI_PROP_DONTPASS, "device_type", &device_type)
2765 != DDI_PROP_SUCCESS) {
2766 return (DDI_FAILURE);
2767 }
2768
2769 if (strcmp(device_type, "pciex") == 0)
2770 rc = DDI_SUCCESS;
2771 else
2772 rc = DDI_FAILURE;
2773
2774 ddi_prop_free(device_type);
2775 return (rc);
2776 }
2777
2778 void
pcie_set_rber_fatal(dev_info_t * dip,boolean_t val)2779 pcie_set_rber_fatal(dev_info_t *dip, boolean_t val)
2780 {
2781 pcie_bus_t *bus_p = PCIE_DIP2UPBUS(dip);
2782 bus_p->bus_pfd->pe_rber_fatal = val;
2783 }
2784
2785 /*
2786 * Return parent Root Port's pe_rber_fatal value.
2787 */
2788 boolean_t
pcie_get_rber_fatal(dev_info_t * dip)2789 pcie_get_rber_fatal(dev_info_t *dip)
2790 {
2791 pcie_bus_t *bus_p = PCIE_DIP2UPBUS(dip);
2792 pcie_bus_t *rp_bus_p = PCIE_DIP2UPBUS(bus_p->bus_rp_dip);
2793 return (rp_bus_p->bus_pfd->pe_rber_fatal);
2794 }
2795
2796 int
pcie_ari_supported(dev_info_t * dip)2797 pcie_ari_supported(dev_info_t *dip)
2798 {
2799 uint32_t devcap2;
2800 uint16_t pciecap;
2801 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2802 uint8_t dev_type;
2803
2804 PCIE_DBG("pcie_ari_supported: dip=%p\n", dip);
2805
2806 if (bus_p == NULL)
2807 return (PCIE_ARI_FORW_NOT_SUPPORTED);
2808
2809 dev_type = bus_p->bus_dev_type;
2810
2811 if ((dev_type != PCIE_PCIECAP_DEV_TYPE_DOWN) &&
2812 (dev_type != PCIE_PCIECAP_DEV_TYPE_ROOT))
2813 return (PCIE_ARI_FORW_NOT_SUPPORTED);
2814
2815 if (pcie_disable_ari) {
2816 PCIE_DBG("pcie_ari_supported: dip=%p: ARI Disabled\n", dip);
2817 return (PCIE_ARI_FORW_NOT_SUPPORTED);
2818 }
2819
2820 pciecap = PCIE_CAP_GET(16, bus_p, PCIE_PCIECAP);
2821
2822 if ((pciecap & PCIE_PCIECAP_VER_MASK) < PCIE_PCIECAP_VER_2_0) {
2823 PCIE_DBG("pcie_ari_supported: dip=%p: Not 2.0\n", dip);
2824 return (PCIE_ARI_FORW_NOT_SUPPORTED);
2825 }
2826
2827 devcap2 = PCIE_CAP_GET(32, bus_p, PCIE_DEVCAP2);
2828
2829 PCIE_DBG("pcie_ari_supported: dip=%p: DevCap2=0x%x\n",
2830 dip, devcap2);
2831
2832 if (devcap2 & PCIE_DEVCAP2_ARI_FORWARD) {
2833 PCIE_DBG("pcie_ari_supported: "
2834 "dip=%p: ARI Forwarding is supported\n", dip);
2835 return (PCIE_ARI_FORW_SUPPORTED);
2836 }
2837 return (PCIE_ARI_FORW_NOT_SUPPORTED);
2838 }
2839
2840 int
pcie_ari_enable(dev_info_t * dip)2841 pcie_ari_enable(dev_info_t *dip)
2842 {
2843 uint16_t devctl2;
2844 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2845
2846 PCIE_DBG("pcie_ari_enable: dip=%p\n", dip);
2847
2848 if (pcie_ari_supported(dip) == PCIE_ARI_FORW_NOT_SUPPORTED)
2849 return (DDI_FAILURE);
2850
2851 devctl2 = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL2);
2852 devctl2 |= PCIE_DEVCTL2_ARI_FORWARD_EN;
2853 PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL2, devctl2);
2854
2855 PCIE_DBG("pcie_ari_enable: dip=%p: writing 0x%x to DevCtl2\n",
2856 dip, devctl2);
2857
2858 return (DDI_SUCCESS);
2859 }
2860
2861 int
pcie_ari_disable(dev_info_t * dip)2862 pcie_ari_disable(dev_info_t *dip)
2863 {
2864 uint16_t devctl2;
2865 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2866
2867 PCIE_DBG("pcie_ari_disable: dip=%p\n", dip);
2868
2869 if (pcie_ari_supported(dip) == PCIE_ARI_FORW_NOT_SUPPORTED)
2870 return (DDI_FAILURE);
2871
2872 devctl2 = PCIE_CAP_GET(16, bus_p, PCIE_DEVCTL2);
2873 devctl2 &= ~PCIE_DEVCTL2_ARI_FORWARD_EN;
2874 PCIE_CAP_PUT(16, bus_p, PCIE_DEVCTL2, devctl2);
2875
2876 PCIE_DBG("pcie_ari_disable: dip=%p: writing 0x%x to DevCtl2\n",
2877 dip, devctl2);
2878
2879 return (DDI_SUCCESS);
2880 }
2881
2882 int
pcie_ari_is_enabled(dev_info_t * dip)2883 pcie_ari_is_enabled(dev_info_t *dip)
2884 {
2885 uint16_t devctl2;
2886 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
2887
2888 PCIE_DBG("pcie_ari_is_enabled: dip=%p\n", dip);
2889
2890 if (pcie_ari_supported(dip) == PCIE_ARI_FORW_NOT_SUPPORTED)
2891 return (PCIE_ARI_FORW_DISABLED);
2892
2893 devctl2 = PCIE_CAP_GET(32, bus_p, PCIE_DEVCTL2);
2894
2895 PCIE_DBG("pcie_ari_is_enabled: dip=%p: DevCtl2=0x%x\n",
2896 dip, devctl2);
2897
2898 if (devctl2 & PCIE_DEVCTL2_ARI_FORWARD_EN) {
2899 PCIE_DBG("pcie_ari_is_enabled: "
2900 "dip=%p: ARI Forwarding is enabled\n", dip);
2901 return (PCIE_ARI_FORW_ENABLED);
2902 }
2903
2904 return (PCIE_ARI_FORW_DISABLED);
2905 }
2906
2907 int
pcie_ari_device(dev_info_t * dip)2908 pcie_ari_device(dev_info_t *dip)
2909 {
2910 ddi_acc_handle_t handle;
2911 uint16_t cap_ptr;
2912
2913 PCIE_DBG("pcie_ari_device: dip=%p\n", dip);
2914
2915 /*
2916 * XXX - This function may be called before the bus_p structure
2917 * has been populated. This code can be changed to remove
2918 * pci_config_setup()/pci_config_teardown() when the RFE
2919 * to populate the bus_p structures early in boot is putback.
2920 */
2921
2922 /* First make sure it is a PCIe device */
2923
2924 if (pci_config_setup(dip, &handle) != DDI_SUCCESS)
2925 return (PCIE_NOT_ARI_DEVICE);
2926
2927 if ((PCI_CAP_LOCATE(handle, PCI_CAP_ID_PCI_E, &cap_ptr))
2928 != DDI_SUCCESS) {
2929 pci_config_teardown(&handle);
2930 return (PCIE_NOT_ARI_DEVICE);
2931 }
2932
2933 /* Locate the ARI Capability */
2934
2935 if ((PCI_CAP_LOCATE(handle, PCI_CAP_XCFG_SPC(PCIE_EXT_CAP_ID_ARI),
2936 &cap_ptr)) == DDI_FAILURE) {
2937 pci_config_teardown(&handle);
2938 return (PCIE_NOT_ARI_DEVICE);
2939 }
2940
2941 /* ARI Capability was found so it must be a ARI device */
2942 PCIE_DBG("pcie_ari_device: ARI Device dip=%p\n", dip);
2943
2944 pci_config_teardown(&handle);
2945 return (PCIE_ARI_DEVICE);
2946 }
2947
2948 int
pcie_ari_get_next_function(dev_info_t * dip,int * func)2949 pcie_ari_get_next_function(dev_info_t *dip, int *func)
2950 {
2951 uint32_t val;
2952 uint16_t cap_ptr, next_function;
2953 ddi_acc_handle_t handle;
2954
2955 /*
2956 * XXX - This function may be called before the bus_p structure
2957 * has been populated. This code can be changed to remove
2958 * pci_config_setup()/pci_config_teardown() when the RFE
2959 * to populate the bus_p structures early in boot is putback.
2960 */
2961
2962 if (pci_config_setup(dip, &handle) != DDI_SUCCESS)
2963 return (DDI_FAILURE);
2964
2965 if ((PCI_CAP_LOCATE(handle,
2966 PCI_CAP_XCFG_SPC(PCIE_EXT_CAP_ID_ARI), &cap_ptr)) == DDI_FAILURE) {
2967 pci_config_teardown(&handle);
2968 return (DDI_FAILURE);
2969 }
2970
2971 val = PCI_CAP_GET32(handle, 0, cap_ptr, PCIE_ARI_CAP);
2972
2973 next_function = (val >> PCIE_ARI_CAP_NEXT_FUNC_SHIFT) &
2974 PCIE_ARI_CAP_NEXT_FUNC_MASK;
2975
2976 pci_config_teardown(&handle);
2977
2978 *func = next_function;
2979
2980 return (DDI_SUCCESS);
2981 }
2982
2983 dev_info_t *
pcie_func_to_dip(dev_info_t * dip,pcie_req_id_t function)2984 pcie_func_to_dip(dev_info_t *dip, pcie_req_id_t function)
2985 {
2986 pcie_req_id_t child_bdf;
2987 dev_info_t *cdip;
2988
2989 for (cdip = ddi_get_child(dip); cdip;
2990 cdip = ddi_get_next_sibling(cdip)) {
2991
2992 if (pcie_get_bdf_from_dip(cdip, &child_bdf) == DDI_FAILURE)
2993 return (NULL);
2994
2995 if ((child_bdf & PCIE_REQ_ID_ARI_FUNC_MASK) == function)
2996 return (cdip);
2997 }
2998 return (NULL);
2999 }
3000
3001 #ifdef DEBUG
3002
3003 static void
pcie_print_bus(pcie_bus_t * bus_p)3004 pcie_print_bus(pcie_bus_t *bus_p)
3005 {
3006 pcie_dbg("\tbus_dip = 0x%p\n", bus_p->bus_dip);
3007 pcie_dbg("\tbus_fm_flags = 0x%x\n", bus_p->bus_fm_flags);
3008
3009 pcie_dbg("\tbus_bdf = 0x%x\n", bus_p->bus_bdf);
3010 pcie_dbg("\tbus_dev_ven_id = 0x%x\n", bus_p->bus_dev_ven_id);
3011 pcie_dbg("\tbus_rev_id = 0x%x\n", bus_p->bus_rev_id);
3012 pcie_dbg("\tbus_hdr_type = 0x%x\n", bus_p->bus_hdr_type);
3013 pcie_dbg("\tbus_dev_type = 0x%x\n", bus_p->bus_dev_type);
3014 pcie_dbg("\tbus_bdg_secbus = 0x%x\n", bus_p->bus_bdg_secbus);
3015 pcie_dbg("\tbus_pcie_off = 0x%x\n", bus_p->bus_pcie_off);
3016 pcie_dbg("\tbus_aer_off = 0x%x\n", bus_p->bus_aer_off);
3017 pcie_dbg("\tbus_pcix_off = 0x%x\n", bus_p->bus_pcix_off);
3018 pcie_dbg("\tbus_ecc_ver = 0x%x\n", bus_p->bus_ecc_ver);
3019 }
3020
3021 /*
3022 * For debugging purposes set pcie_dbg_print != 0 to see printf messages
3023 * during interrupt.
3024 *
3025 * When a proper solution is in place this code will disappear.
3026 * Potential solutions are:
3027 * o circular buffers
3028 * o taskq to print at lower pil
3029 */
3030 int pcie_dbg_print = 0;
3031 void
pcie_dbg(char * fmt,...)3032 pcie_dbg(char *fmt, ...)
3033 {
3034 va_list ap;
3035
3036 if (!pcie_debug_flags) {
3037 return;
3038 }
3039 va_start(ap, fmt);
3040 if (servicing_interrupt()) {
3041 if (pcie_dbg_print) {
3042 prom_vprintf(fmt, ap);
3043 }
3044 } else {
3045 prom_vprintf(fmt, ap);
3046 }
3047 va_end(ap);
3048 }
3049 #endif /* DEBUG */
3050
3051 boolean_t
pcie_link_bw_supported(dev_info_t * dip)3052 pcie_link_bw_supported(dev_info_t *dip)
3053 {
3054 uint32_t linkcap;
3055 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3056
3057 if (!PCIE_IS_PCIE(bus_p)) {
3058 return (B_FALSE);
3059 }
3060
3061 if (!PCIE_IS_RP(bus_p) && !PCIE_IS_SWD(bus_p)) {
3062 return (B_FALSE);
3063 }
3064
3065 linkcap = PCIE_CAP_GET(32, bus_p, PCIE_LINKCAP);
3066 return ((linkcap & PCIE_LINKCAP_LINK_BW_NOTIFY_CAP) != 0);
3067 }
3068
3069 int
pcie_link_bw_enable(dev_info_t * dip)3070 pcie_link_bw_enable(dev_info_t *dip)
3071 {
3072 uint16_t linkctl;
3073 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3074
3075 if (pcie_disable_lbw != 0) {
3076 return (DDI_FAILURE);
3077 }
3078
3079 if (!pcie_link_bw_supported(dip)) {
3080 return (DDI_FAILURE);
3081 }
3082
3083 mutex_init(&bus_p->bus_lbw_mutex, NULL, MUTEX_DRIVER, NULL);
3084 cv_init(&bus_p->bus_lbw_cv, NULL, CV_DRIVER, NULL);
3085 linkctl = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL);
3086 linkctl |= PCIE_LINKCTL_LINK_BW_INTR_EN;
3087 linkctl |= PCIE_LINKCTL_LINK_AUTO_BW_INTR_EN;
3088 PCIE_CAP_PUT(16, bus_p, PCIE_LINKCTL, linkctl);
3089
3090 bus_p->bus_lbw_pbuf = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
3091 bus_p->bus_lbw_cbuf = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
3092 bus_p->bus_lbw_state |= PCIE_LBW_S_ENABLED;
3093
3094 return (DDI_SUCCESS);
3095 }
3096
3097 int
pcie_link_bw_disable(dev_info_t * dip)3098 pcie_link_bw_disable(dev_info_t *dip)
3099 {
3100 uint16_t linkctl;
3101 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3102
3103 if ((bus_p->bus_lbw_state & PCIE_LBW_S_ENABLED) == 0) {
3104 return (DDI_FAILURE);
3105 }
3106
3107 mutex_enter(&bus_p->bus_lbw_mutex);
3108 while ((bus_p->bus_lbw_state &
3109 (PCIE_LBW_S_DISPATCHED | PCIE_LBW_S_RUNNING)) != 0) {
3110 cv_wait(&bus_p->bus_lbw_cv, &bus_p->bus_lbw_mutex);
3111 }
3112 mutex_exit(&bus_p->bus_lbw_mutex);
3113
3114 linkctl = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL);
3115 linkctl &= ~PCIE_LINKCTL_LINK_BW_INTR_EN;
3116 linkctl &= ~PCIE_LINKCTL_LINK_AUTO_BW_INTR_EN;
3117 PCIE_CAP_PUT(16, bus_p, PCIE_LINKCTL, linkctl);
3118
3119 bus_p->bus_lbw_state &= ~PCIE_LBW_S_ENABLED;
3120 kmem_free(bus_p->bus_lbw_pbuf, MAXPATHLEN);
3121 kmem_free(bus_p->bus_lbw_cbuf, MAXPATHLEN);
3122 bus_p->bus_lbw_pbuf = NULL;
3123 bus_p->bus_lbw_cbuf = NULL;
3124
3125 mutex_destroy(&bus_p->bus_lbw_mutex);
3126 cv_destroy(&bus_p->bus_lbw_cv);
3127
3128 return (DDI_SUCCESS);
3129 }
3130
3131 void
pcie_link_bw_taskq(void * arg)3132 pcie_link_bw_taskq(void *arg)
3133 {
3134 dev_info_t *dip = arg;
3135 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3136 dev_info_t *cdip;
3137 boolean_t again;
3138 sysevent_t *se;
3139 sysevent_value_t se_val;
3140 sysevent_id_t eid;
3141 sysevent_attr_list_t *ev_attr_list;
3142
3143 top:
3144 ndi_devi_enter(dip);
3145 se = NULL;
3146 ev_attr_list = NULL;
3147 mutex_enter(&bus_p->bus_lbw_mutex);
3148 bus_p->bus_lbw_state &= ~PCIE_LBW_S_DISPATCHED;
3149 bus_p->bus_lbw_state |= PCIE_LBW_S_RUNNING;
3150 mutex_exit(&bus_p->bus_lbw_mutex);
3151
3152 /*
3153 * Update our own speeds as we've likely changed something.
3154 */
3155 pcie_capture_speeds(dip);
3156
3157 /*
3158 * Walk our children. We only care about updating this on function 0
3159 * because the PCIe specification requires that these all be the same
3160 * otherwise.
3161 */
3162 for (cdip = ddi_get_child(dip); cdip != NULL;
3163 cdip = ddi_get_next_sibling(cdip)) {
3164 pcie_bus_t *cbus_p = PCIE_DIP2BUS(cdip);
3165
3166 if (cbus_p == NULL) {
3167 continue;
3168 }
3169
3170 if ((cbus_p->bus_bdf & PCIE_REQ_ID_FUNC_MASK) != 0) {
3171 continue;
3172 }
3173
3174 /*
3175 * It's possible that this can fire while a child is otherwise
3176 * only partially constructed. Therefore, if we don't have the
3177 * config handle, don't bother updating the child.
3178 */
3179 if (cbus_p->bus_cfg_hdl == NULL) {
3180 continue;
3181 }
3182
3183 pcie_capture_speeds(cdip);
3184 break;
3185 }
3186
3187 se = sysevent_alloc(EC_PCIE, ESC_PCIE_LINK_STATE,
3188 ILLUMOS_KERN_PUB "pcie", SE_SLEEP);
3189
3190 (void) ddi_pathname(dip, bus_p->bus_lbw_pbuf);
3191 se_val.value_type = SE_DATA_TYPE_STRING;
3192 se_val.value.sv_string = bus_p->bus_lbw_pbuf;
3193 if (sysevent_add_attr(&ev_attr_list, PCIE_EV_DETECTOR_PATH, &se_val,
3194 SE_SLEEP) != 0) {
3195 ndi_devi_exit(dip);
3196 goto err;
3197 }
3198
3199 if (cdip != NULL) {
3200 (void) ddi_pathname(cdip, bus_p->bus_lbw_cbuf);
3201
3202 se_val.value_type = SE_DATA_TYPE_STRING;
3203 se_val.value.sv_string = bus_p->bus_lbw_cbuf;
3204
3205 /*
3206 * If this fails, that's OK. We'd rather get the event off and
3207 * there's a chance that there may not be anything there for us.
3208 */
3209 (void) sysevent_add_attr(&ev_attr_list, PCIE_EV_CHILD_PATH,
3210 &se_val, SE_SLEEP);
3211 }
3212
3213 ndi_devi_exit(dip);
3214
3215 /*
3216 * Before we generate and send down a sysevent, we need to tell the
3217 * system that parts of the devinfo cache need to be invalidated. While
3218 * the function below takes several args, it ignores them all. Because
3219 * this is a global invalidation, we don't bother trying to do much more
3220 * than requesting a global invalidation, lest we accidentally kick off
3221 * several in a row.
3222 */
3223 ddi_prop_cache_invalidate(DDI_DEV_T_NONE, NULL, NULL, 0);
3224
3225 if (sysevent_attach_attributes(se, ev_attr_list) != 0) {
3226 goto err;
3227 }
3228 ev_attr_list = NULL;
3229
3230 if (log_sysevent(se, SE_SLEEP, &eid) != 0) {
3231 goto err;
3232 }
3233
3234 err:
3235 sysevent_free_attr(ev_attr_list);
3236 sysevent_free(se);
3237
3238 mutex_enter(&bus_p->bus_lbw_mutex);
3239 bus_p->bus_lbw_state &= ~PCIE_LBW_S_RUNNING;
3240 cv_broadcast(&bus_p->bus_lbw_cv);
3241 again = (bus_p->bus_lbw_state & PCIE_LBW_S_DISPATCHED) != 0;
3242 mutex_exit(&bus_p->bus_lbw_mutex);
3243
3244 if (again) {
3245 goto top;
3246 }
3247 }
3248
3249 int
pcie_link_bw_intr(dev_info_t * dip)3250 pcie_link_bw_intr(dev_info_t *dip)
3251 {
3252 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3253 uint16_t linksts;
3254 uint16_t flags = PCIE_LINKSTS_LINK_BW_MGMT | PCIE_LINKSTS_AUTO_BW;
3255 hrtime_t now;
3256
3257 if ((bus_p->bus_lbw_state & PCIE_LBW_S_ENABLED) == 0) {
3258 return (DDI_INTR_UNCLAIMED);
3259 }
3260
3261 linksts = PCIE_CAP_GET(16, bus_p, PCIE_LINKSTS);
3262 if ((linksts & flags) == 0) {
3263 return (DDI_INTR_UNCLAIMED);
3264 }
3265
3266 now = gethrtime();
3267
3268 /*
3269 * Check if we've already dispatched this event. If we have already
3270 * dispatched it, then there's nothing else to do, we coalesce multiple
3271 * events.
3272 */
3273 mutex_enter(&bus_p->bus_lbw_mutex);
3274 bus_p->bus_lbw_nevents++;
3275 bus_p->bus_lbw_last_ts = now;
3276 if ((bus_p->bus_lbw_state & PCIE_LBW_S_DISPATCHED) == 0) {
3277 if ((bus_p->bus_lbw_state & PCIE_LBW_S_RUNNING) == 0) {
3278 taskq_dispatch_ent(pcie_link_tq, pcie_link_bw_taskq,
3279 dip, 0, &bus_p->bus_lbw_ent);
3280 }
3281
3282 bus_p->bus_lbw_state |= PCIE_LBW_S_DISPATCHED;
3283 }
3284 mutex_exit(&bus_p->bus_lbw_mutex);
3285
3286 PCIE_CAP_PUT(16, bus_p, PCIE_LINKSTS, flags);
3287 return (DDI_INTR_CLAIMED);
3288 }
3289
3290 int
pcie_link_set_target(dev_info_t * dip,pcie_link_speed_t speed)3291 pcie_link_set_target(dev_info_t *dip, pcie_link_speed_t speed)
3292 {
3293 uint16_t ctl2, rval;
3294 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3295
3296 if (!PCIE_IS_PCIE(bus_p)) {
3297 return (ENOTSUP);
3298 }
3299
3300 if (!PCIE_IS_RP(bus_p) && !PCIE_IS_SWD(bus_p)) {
3301 return (ENOTSUP);
3302 }
3303
3304 if (bus_p->bus_pcie_vers < 2) {
3305 return (ENOTSUP);
3306 }
3307
3308 switch (speed) {
3309 case PCIE_LINK_SPEED_2_5:
3310 rval = PCIE_LINKCTL2_TARGET_SPEED_2_5;
3311 break;
3312 case PCIE_LINK_SPEED_5:
3313 rval = PCIE_LINKCTL2_TARGET_SPEED_5;
3314 break;
3315 case PCIE_LINK_SPEED_8:
3316 rval = PCIE_LINKCTL2_TARGET_SPEED_8;
3317 break;
3318 case PCIE_LINK_SPEED_16:
3319 rval = PCIE_LINKCTL2_TARGET_SPEED_16;
3320 break;
3321 case PCIE_LINK_SPEED_32:
3322 rval = PCIE_LINKCTL2_TARGET_SPEED_32;
3323 break;
3324 case PCIE_LINK_SPEED_64:
3325 rval = PCIE_LINKCTL2_TARGET_SPEED_64;
3326 break;
3327 default:
3328 return (EINVAL);
3329 }
3330
3331 mutex_enter(&bus_p->bus_speed_mutex);
3332 if ((bus_p->bus_sup_speed & speed) == 0) {
3333 mutex_exit(&bus_p->bus_speed_mutex);
3334 return (ENOTSUP);
3335 }
3336
3337 bus_p->bus_target_speed = speed;
3338 bus_p->bus_speed_flags |= PCIE_LINK_F_ADMIN_TARGET;
3339
3340 ctl2 = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL2);
3341 ctl2 &= ~PCIE_LINKCTL2_TARGET_SPEED_MASK;
3342 ctl2 |= rval;
3343 PCIE_CAP_PUT(16, bus_p, PCIE_LINKCTL2, ctl2);
3344 mutex_exit(&bus_p->bus_speed_mutex);
3345
3346 /*
3347 * Make sure our updates have been reflected in devinfo.
3348 */
3349 pcie_capture_speeds(dip);
3350
3351 return (0);
3352 }
3353
3354 int
pcie_link_retrain(dev_info_t * dip)3355 pcie_link_retrain(dev_info_t *dip)
3356 {
3357 uint16_t ctl;
3358 pcie_bus_t *bus_p = PCIE_DIP2BUS(dip);
3359
3360 if (!PCIE_IS_PCIE(bus_p)) {
3361 return (ENOTSUP);
3362 }
3363
3364 if (!PCIE_IS_RP(bus_p) && !PCIE_IS_SWD(bus_p)) {
3365 return (ENOTSUP);
3366 }
3367
3368 /*
3369 * The PCIe specification suggests that we make sure that the link isn't
3370 * in training before issuing this command in case there was a state
3371 * machine transition prior to when we got here. We wait and then go
3372 * ahead and issue the command anyways.
3373 */
3374 for (uint32_t i = 0; i < pcie_link_retrain_count; i++) {
3375 uint16_t sts;
3376
3377 sts = PCIE_CAP_GET(16, bus_p, PCIE_LINKSTS);
3378 if ((sts & PCIE_LINKSTS_LINK_TRAINING) == 0)
3379 break;
3380 delay(drv_usectohz(pcie_link_retrain_delay_ms * 1000));
3381 }
3382
3383 ctl = PCIE_CAP_GET(16, bus_p, PCIE_LINKCTL);
3384 ctl |= PCIE_LINKCTL_RETRAIN_LINK;
3385 PCIE_CAP_PUT(16, bus_p, PCIE_LINKCTL, ctl);
3386
3387 /*
3388 * Wait again to see if it clears before returning to the user.
3389 */
3390 for (uint32_t i = 0; i < pcie_link_retrain_count; i++) {
3391 uint16_t sts;
3392
3393 sts = PCIE_CAP_GET(16, bus_p, PCIE_LINKSTS);
3394 if ((sts & PCIE_LINKSTS_LINK_TRAINING) == 0)
3395 break;
3396 delay(drv_usectohz(pcie_link_retrain_delay_ms * 1000));
3397 }
3398
3399 return (0);
3400 }
3401
3402 /*
3403 * Here we're going through and grabbing information about a given PCIe device.
3404 * Our situation is a little bit complicated at this point. This gets invoked
3405 * both during early initialization and during hotplug events. We cannot rely on
3406 * the device node having been fully set up, that is, while the pcie_bus_t
3407 * normally contains a ddi_acc_handle_t for configuration space, that may not be
3408 * valid yet as this can occur before child initialization or we may be dealing
3409 * with a function that will never have a handle.
3410 *
3411 * However, we should always have a fully furnished pcie_bus_t, which means that
3412 * we can get its bdf and use that to access the devices configuration space.
3413 */
3414 static int
pcie_fabric_feature_scan(dev_info_t * dip,void * arg)3415 pcie_fabric_feature_scan(dev_info_t *dip, void *arg)
3416 {
3417 pcie_bus_t *bus_p;
3418 uint32_t devcap;
3419 uint16_t mps;
3420 dev_info_t *rcdip;
3421 pcie_fabric_data_t *fab = arg;
3422
3423 /*
3424 * Skip over non-PCIe devices. If we encounter something here, we don't
3425 * bother going through any of its children because we don't have reason
3426 * to believe that a PCIe device that this will impact will exist below
3427 * this. While it is possible that there's a PCIe fabric downstream an
3428 * intermediate old PCI/PCI-X bus, at that point, we'll still trigger
3429 * our complex fabric detection and use the minimums.
3430 *
3431 * The reason this doesn't trigger an immediate flagging as a complex
3432 * case like the one below is because we could be scanning a device that
3433 * is a nexus driver and has children already (albeit that would be
3434 * somewhat surprising as we don't anticipate being called at this
3435 * point).
3436 */
3437 if (pcie_dev(dip) != DDI_SUCCESS) {
3438 return (DDI_WALK_PRUNECHILD);
3439 }
3440
3441 /*
3442 * If we fail to find a pcie_bus_t for some reason, that's somewhat
3443 * surprising. We log this fact and set the complex flag and indicate it
3444 * was because of this case. This immediately transitions us to a
3445 * "complex" case which means use the minimal, safe, settings.
3446 */
3447 bus_p = PCIE_DIP2BUS(dip);
3448 if (bus_p == NULL) {
3449 dev_err(dip, CE_WARN, "failed to find associated pcie_bus_t "
3450 "during fabric scan");
3451 fab->pfd_flags |= PCIE_FABRIC_F_COMPLEX;
3452 return (DDI_WALK_TERMINATE);
3453 }
3454
3455 /*
3456 * In a similar case, there is hardware out there which is a PCIe
3457 * device, but does not advertise a PCIe capability. An example of this
3458 * is the IDT Tsi382A which can hide its PCIe capability. If this is
3459 * the case, we immediately terminate scanning and flag this as a
3460 * 'complex' case which causes us to use guaranteed safe settings.
3461 */
3462 if (bus_p->bus_pcie_off == 0) {
3463 dev_err(dip, CE_WARN, "encountered PCIe device without PCIe "
3464 "capability");
3465 fab->pfd_flags |= PCIE_FABRIC_F_COMPLEX;
3466 return (DDI_WALK_TERMINATE);
3467 }
3468
3469 rcdip = pcie_get_rc_dip(dip);
3470
3471 /*
3472 * First, start by determining what the device's tagging and max packet
3473 * size is. All PCIe devices will always have the 8-bit tag information
3474 * as this has existed since PCIe 1.0. 10-bit tagging requires a V2
3475 * PCIe capability. 14-bit requires the DEV3 cap. If we are missing a
3476 * version or capability, then we always treat that as lacking the bits
3477 * in the fabric.
3478 */
3479 ASSERT3U(bus_p->bus_pcie_off, !=, 0);
3480 devcap = pci_cfgacc_get32(rcdip, bus_p->bus_bdf, bus_p->bus_pcie_off +
3481 PCIE_DEVCAP);
3482 mps = devcap & PCIE_DEVCAP_MAX_PAYLOAD_MASK;
3483 if (mps < fab->pfd_mps_found) {
3484 fab->pfd_mps_found = mps;
3485 }
3486
3487 if ((devcap & PCIE_DEVCAP_EXT_TAG_8BIT) == 0) {
3488 fab->pfd_tag_found &= ~PCIE_TAG_8B;
3489 }
3490
3491 if (bus_p->bus_pcie_vers == PCIE_PCIECAP_VER_2_0) {
3492 uint32_t devcap2 = pci_cfgacc_get32(rcdip, bus_p->bus_bdf,
3493 bus_p->bus_pcie_off + PCIE_DEVCAP2);
3494 if ((devcap2 & PCIE_DEVCAP2_10B_TAG_COMP_SUP) == 0) {
3495 fab->pfd_tag_found &= ~PCIE_TAG_10B_COMP;
3496 }
3497 } else {
3498 fab->pfd_tag_found &= ~PCIE_TAG_10B_COMP;
3499 }
3500
3501 if (bus_p->bus_dev3_off != 0) {
3502 uint32_t devcap3 = pci_cfgacc_get32(rcdip, bus_p->bus_bdf,
3503 bus_p->bus_dev3_off + PCIE_DEVCAP3);
3504 if ((devcap3 & PCIE_DEVCAP3_14B_TAG_COMP_SUP) == 0) {
3505 fab->pfd_tag_found &= ~PCIE_TAG_14B_COMP;
3506 }
3507 } else {
3508 fab->pfd_tag_found &= ~PCIE_TAG_14B_COMP;
3509 }
3510
3511 /*
3512 * Now that we have captured device information, we must go and ask
3513 * questions of the topology here. The big theory statement enumerates
3514 * several types of cases. The big question we need to answer is have we
3515 * encountered a hotpluggable bridge that means we need to mark this as
3516 * complex.
3517 *
3518 * The big theory statement notes several different kinds of hotplug
3519 * topologies that exist that we can theoretically support. Right now we
3520 * opt to keep our lives simple and focus solely on (4) and (5). These
3521 * can both be summarized by a single, fairly straightforward rule:
3522 *
3523 * The only allowed hotpluggable entity is a root port.
3524 *
3525 * The reason that this can work and detect cases like (6), (7), and our
3526 * other invalid ones is that the hotplug code will scan and find all
3527 * children before we are called into here.
3528 */
3529 if (bus_p->bus_hp_sup_modes != 0) {
3530 /*
3531 * We opt to terminate in this case because there's no value in
3532 * scanning the rest of the tree at this point.
3533 */
3534 if (!PCIE_IS_RP(bus_p)) {
3535 fab->pfd_flags |= PCIE_FABRIC_F_COMPLEX;
3536 return (DDI_WALK_TERMINATE);
3537 }
3538
3539 fab->pfd_flags |= PCIE_FABRIC_F_RP_HP;
3540 }
3541
3542 /*
3543 * As our walk starts at a root port, we need to make sure that we don't
3544 * pick up any of its siblings and their children as those would be
3545 * different PCIe fabric domains for us to scan. In many hardware
3546 * platforms multiple root ports are all at the same level in the tree.
3547 */
3548 if (bus_p->bus_rp_dip == dip) {
3549 return (DDI_WALK_PRUNESIB);
3550 }
3551
3552 return (DDI_WALK_CONTINUE);
3553 }
3554
3555 static int
pcie_fabric_feature_set(dev_info_t * dip,void * arg)3556 pcie_fabric_feature_set(dev_info_t *dip, void *arg)
3557 {
3558 pcie_bus_t *bus_p;
3559 dev_info_t *rcdip;
3560 pcie_fabric_data_t *fab = arg;
3561 uint32_t devcap, devctl;
3562
3563 if (pcie_dev(dip) != DDI_SUCCESS) {
3564 return (DDI_WALK_PRUNECHILD);
3565 }
3566
3567 /*
3568 * The missing bus_t sent us into the complex case previously. We still
3569 * need to make sure all devices have values we expect here and thus
3570 * don't terminate like the above. The same is true for the case where
3571 * there is no PCIe capability.
3572 */
3573 bus_p = PCIE_DIP2BUS(dip);
3574 if (bus_p == NULL || bus_p->bus_pcie_off == 0) {
3575 return (DDI_WALK_CONTINUE);
3576 }
3577 rcdip = pcie_get_rc_dip(dip);
3578
3579 devcap = pci_cfgacc_get32(rcdip, bus_p->bus_bdf, bus_p->bus_pcie_off +
3580 PCIE_DEVCAP);
3581 devctl = pci_cfgacc_get16(rcdip, bus_p->bus_bdf, bus_p->bus_pcie_off +
3582 PCIE_DEVCTL);
3583
3584 if ((devcap & PCIE_DEVCAP_EXT_TAG_8BIT) != 0 &&
3585 (fab->pfd_tag_act & PCIE_TAG_8B) != 0) {
3586 devctl |= PCIE_DEVCTL_EXT_TAG_FIELD_EN;
3587 }
3588
3589 devctl &= ~PCIE_DEVCTL_MAX_PAYLOAD_MASK;
3590 ASSERT0(fab->pfd_mps_act & ~PCIE_DEVCAP_MAX_PAYLOAD_MASK);
3591 devctl |= fab->pfd_mps_act << PCIE_DEVCTL_MAX_PAYLOAD_SHIFT;
3592
3593 pci_cfgacc_put16(rcdip, bus_p->bus_bdf, bus_p->bus_pcie_off +
3594 PCIE_DEVCTL, devctl);
3595
3596 if (bus_p->bus_pcie_vers == PCIE_PCIECAP_VER_2_0 &&
3597 (fab->pfd_tag_act & PCIE_TAG_10B_COMP) != 0) {
3598 uint32_t devcap2 = pci_cfgacc_get32(rcdip, bus_p->bus_bdf,
3599 bus_p->bus_pcie_off + PCIE_DEVCAP2);
3600
3601 if ((devcap2 & PCIE_DEVCAP2_10B_TAG_REQ_SUP) == 0) {
3602 uint16_t devctl2 = pci_cfgacc_get16(rcdip,
3603 bus_p->bus_bdf, bus_p->bus_pcie_off + PCIE_DEVCTL2);
3604 devctl2 |= PCIE_DEVCTL2_10B_TAG_REQ_EN;
3605 pci_cfgacc_put16(rcdip, bus_p->bus_bdf,
3606 bus_p->bus_pcie_off + PCIE_DEVCTL2, devctl2);
3607 }
3608 }
3609
3610 if (bus_p->bus_dev3_off != 0 &&
3611 (fab->pfd_tag_act & PCIE_TAG_14B_COMP) != 0) {
3612 uint32_t devcap3 = pci_cfgacc_get32(rcdip, bus_p->bus_bdf,
3613 bus_p->bus_dev3_off + PCIE_DEVCAP3);
3614
3615 if ((devcap3 & PCIE_DEVCAP3_14B_TAG_REQ_SUP) == 0) {
3616 uint16_t devctl3 = pci_cfgacc_get16(rcdip,
3617 bus_p->bus_bdf, bus_p->bus_dev3_off + PCIE_DEVCTL3);
3618 devctl3 |= PCIE_DEVCTL3_14B_TAG_REQ_EN;
3619 pci_cfgacc_put16(rcdip, bus_p->bus_bdf,
3620 bus_p->bus_pcie_off + PCIE_DEVCTL2, devctl3);
3621 }
3622 }
3623
3624 /*
3625 * As our walk starts at a root port, we need to make sure that we don't
3626 * pick up any of its siblings and their children as those would be
3627 * different PCIe fabric domains for us to scan. In many hardware
3628 * platforms multiple root ports are all at the same level in the tree.
3629 */
3630 if (bus_p->bus_rp_dip == dip) {
3631 return (DDI_WALK_PRUNESIB);
3632 }
3633
3634 return (DDI_WALK_CONTINUE);
3635 }
3636
3637 /*
3638 * This is used to scan and determine the total set of PCIe fabric settings that
3639 * we should have in the system for everything downstream of this specified root
3640 * port. Note, it is only really safe to call this while working from the
3641 * perspective of a root port as we will be walking down the entire device tree.
3642 *
3643 * However, our callers, particularly hoptlug, don't have all the information
3644 * we'd like. In particular, we need to check that:
3645 *
3646 * o This is actually a PCIe device.
3647 * o That this is a root port (see the big theory statement to understand this
3648 * constraint).
3649 */
3650 void
pcie_fabric_setup(dev_info_t * dip)3651 pcie_fabric_setup(dev_info_t *dip)
3652 {
3653 pcie_bus_t *bus_p;
3654 pcie_fabric_data_t *fab;
3655 dev_info_t *pdip;
3656
3657 bus_p = PCIE_DIP2BUS(dip);
3658 if (bus_p == NULL || !PCIE_IS_RP(bus_p)) {
3659 return;
3660 }
3661
3662 VERIFY3P(bus_p->bus_fab, !=, NULL);
3663 fab = bus_p->bus_fab;
3664
3665 /*
3666 * For us to call ddi_walk_devs(), our parent needs to be held.
3667 * ddi_walk_devs() will take care of grabbing our dip as part of its
3668 * walk before we iterate over our children.
3669 *
3670 * A reasonable question to ask here is why is it safe to ask for our
3671 * parent? In this case, because we have entered here through some
3672 * thread that's operating on us whether as part of attach or a hotplug
3673 * event, our dip somewhat by definition has to be valid. If we were
3674 * looking at our dip's children and then asking them for a parent, then
3675 * that would be a race condition.
3676 */
3677 pdip = ddi_get_parent(dip);
3678 VERIFY3P(pdip, !=, NULL);
3679 ndi_devi_enter(pdip);
3680 fab->pfd_flags |= PCIE_FABRIC_F_SCANNING;
3681
3682 /*
3683 * Reinitialize the tracking structure to basically set the maximum
3684 * caps. These will be chipped away during the scan.
3685 */
3686 fab->pfd_mps_found = PCIE_DEVCAP_MAX_PAYLOAD_4096;
3687 fab->pfd_tag_found = PCIE_TAG_ALL;
3688 fab->pfd_flags &= ~PCIE_FABRIC_F_COMPLEX;
3689
3690 ddi_walk_devs(dip, pcie_fabric_feature_scan, fab);
3691
3692 if ((fab->pfd_flags & PCIE_FABRIC_F_COMPLEX) != 0) {
3693 fab->pfd_tag_act = PCIE_TAG_5B;
3694 fab->pfd_mps_act = PCIE_DEVCAP_MAX_PAYLOAD_128;
3695 } else {
3696 fab->pfd_tag_act = fab->pfd_tag_found;
3697 fab->pfd_mps_act = fab->pfd_mps_found;
3698 }
3699
3700 ddi_walk_devs(dip, pcie_fabric_feature_set, fab);
3701
3702 fab->pfd_flags &= ~PCIE_FABRIC_F_SCANNING;
3703 ndi_devi_exit(pdip);
3704 }
3705