xref: /illumos-gate/usr/src/uts/i86pc/io/pciex/npe.c (revision 7257d1b4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  *	Host to PCI-Express local bus driver
31  */
32 
33 #include <sys/conf.h>
34 #include <sys/modctl.h>
35 #include <sys/pci_impl.h>
36 #include <sys/pcie_impl.h>
37 #include <sys/sysmacros.h>
38 #include <sys/ddi_intr.h>
39 #include <sys/sunndi.h>
40 #include <sys/sunddi.h>
41 #include <sys/ddifm.h>
42 #include <sys/ndifm.h>
43 #include <sys/fm/util.h>
44 #include <sys/hotplug/pci/pcihp.h>
45 #include <io/pci/pci_tools_ext.h>
46 #include <io/pci/pci_common.h>
47 #include <io/pciex/pcie_nvidia.h>
48 
49 /*
50  * Bus Operation functions
51  */
52 static int	npe_bus_map(dev_info_t *, dev_info_t *, ddi_map_req_t *,
53 		    off_t, off_t, caddr_t *);
54 static int	npe_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t,
55 		    void *, void *);
56 static int	npe_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
57 		    ddi_intr_handle_impl_t *, void *);
58 static int	npe_fm_init(dev_info_t *, dev_info_t *, int,
59 		    ddi_iblock_cookie_t *);
60 
61 static int	npe_fm_callback(dev_info_t *, ddi_fm_error_t *, const void *);
62 
63 /*
64  * Disable URs and Received MA for all PCIe devices.  Until x86 SW is changed so
65  * that random drivers do not do PIO accesses on devices that it does not own,
66  * these error bits must be disabled.  SERR must also be disabled if URs have
67  * been masked.
68  */
69 uint32_t	npe_aer_uce_mask = PCIE_AER_UCE_UR;
70 uint32_t	npe_aer_ce_mask = 0;
71 uint32_t	npe_aer_suce_mask = PCIE_AER_SUCE_RCVD_MA;
72 
73 struct bus_ops npe_bus_ops = {
74 	BUSO_REV,
75 	npe_bus_map,
76 	NULL,
77 	NULL,
78 	NULL,
79 	i_ddi_map_fault,
80 	ddi_dma_map,
81 	ddi_dma_allochdl,
82 	ddi_dma_freehdl,
83 	ddi_dma_bindhdl,
84 	ddi_dma_unbindhdl,
85 	ddi_dma_flush,
86 	ddi_dma_win,
87 	ddi_dma_mctl,
88 	npe_ctlops,
89 	ddi_bus_prop_op,
90 	0,		/* (*bus_get_eventcookie)();	*/
91 	0,		/* (*bus_add_eventcall)();	*/
92 	0,		/* (*bus_remove_eventcall)();	*/
93 	0,		/* (*bus_post_event)();		*/
94 	0,		/* (*bus_intr_ctl)(); */
95 	0,		/* (*bus_config)(); */
96 	0,		/* (*bus_unconfig)(); */
97 	npe_fm_init,	/* (*bus_fm_init)(); */
98 	NULL,		/* (*bus_fm_fini)(); */
99 	NULL,		/* (*bus_fm_access_enter)(); */
100 	NULL,		/* (*bus_fm_access_exit)(); */
101 	NULL,		/* (*bus_power)(); */
102 	npe_intr_ops	/* (*bus_intr_op)(); */
103 };
104 
105 /*
106  * One goal here is to leverage off of the pcihp.c source without making
107  * changes to it.  Call into it's cb_ops directly if needed, piggybacking
108  * anything else needed by the pci_tools.c module.  Only pci_tools and pcihp
109  * will be using the PCI devctl node.
110  */
111 static int	npe_open(dev_t *, int, int, cred_t *);
112 static int	npe_close(dev_t, int, int, cred_t *);
113 static int	npe_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
114 static int	npe_prop_op(dev_t, dev_info_t *, ddi_prop_op_t, int, char *,
115 		    caddr_t, int *);
116 static int	npe_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
117 
118 struct cb_ops npe_cb_ops = {
119 	npe_open,			/* open */
120 	npe_close,			/* close */
121 	nodev,				/* strategy */
122 	nodev,				/* print */
123 	nodev,				/* dump */
124 	nodev,				/* read */
125 	nodev,				/* write */
126 	npe_ioctl,			/* ioctl */
127 	nodev,				/* devmap */
128 	nodev,				/* mmap */
129 	nodev,				/* segmap */
130 	nochpoll,			/* poll */
131 	npe_prop_op,			/* cb_prop_op */
132 	NULL,				/* streamtab */
133 	D_NEW | D_MP | D_HOTPLUG,	/* Driver compatibility flag */
134 	CB_REV,				/* rev */
135 	nodev,				/* int (*cb_aread)() */
136 	nodev				/* int (*cb_awrite)() */
137 };
138 
139 
140 /*
141  * Device Node Operation functions
142  */
143 static int	npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
144 static int	npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
145 
146 struct dev_ops npe_ops = {
147 	DEVO_REV,		/* devo_rev */
148 	0,			/* refcnt  */
149 	npe_info,		/* info */
150 	nulldev,		/* identify */
151 	nulldev,		/* probe */
152 	npe_attach,		/* attach */
153 	npe_detach,		/* detach */
154 	nulldev,		/* reset */
155 	&npe_cb_ops,		/* driver operations */
156 	&npe_bus_ops		/* bus operations */
157 };
158 
159 /*
160  * Internal routines in support of particular npe_ctlops.
161  */
162 static int npe_removechild(dev_info_t *child);
163 static int npe_initchild(dev_info_t *child);
164 
165 /*
166  * External support routine
167  */
168 extern void	npe_query_acpi_mcfg(dev_info_t *dip);
169 extern void	npe_ck804_fix_aer_ptr(ddi_acc_handle_t cfg_hdl);
170 extern int	npe_disable_empty_bridges_workaround(dev_info_t *child);
171 extern void	npe_nvidia_error_mask(ddi_acc_handle_t cfg_hdl);
172 extern void	npe_intel_error_mask(ddi_acc_handle_t cfg_hdl);
173 
174 /*
175  * Module linkage information for the kernel.
176  */
177 static struct modldrv modldrv = {
178 	&mod_driverops, /* Type of module */
179 	"Host to PCIe nexus driver %I%",
180 	&npe_ops,	/* driver ops */
181 };
182 
183 static struct modlinkage modlinkage = {
184 	MODREV_1,
185 	(void *)&modldrv,
186 	NULL
187 };
188 
189 /* Save minimal state. */
190 void *npe_statep;
191 
192 int
193 _init(void)
194 {
195 	int e;
196 
197 	/*
198 	 * Initialize per-pci bus soft state pointer.
199 	 */
200 	e = ddi_soft_state_init(&npe_statep, sizeof (pci_state_t), 1);
201 	if (e != 0)
202 		return (e);
203 
204 	if ((e = mod_install(&modlinkage)) != 0)
205 		ddi_soft_state_fini(&npe_statep);
206 
207 	return (e);
208 }
209 
210 
211 int
212 _fini(void)
213 {
214 	int rc;
215 
216 	rc = mod_remove(&modlinkage);
217 	if (rc != 0)
218 		return (rc);
219 
220 	ddi_soft_state_fini(&npe_statep);
221 	return (rc);
222 }
223 
224 
225 int
226 _info(struct modinfo *modinfop)
227 {
228 	return (mod_info(&modlinkage, modinfop));
229 }
230 
231 /*ARGSUSED*/
232 static int
233 npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
234 {
235 	/*
236 	 * Use the minor number as constructed by pcihp, as the index value to
237 	 * ddi_soft_state_zalloc.
238 	 */
239 	int instance = ddi_get_instance(devi);
240 	pci_state_t *pcip = NULL;
241 
242 	if (cmd == DDI_RESUME)
243 		return (DDI_SUCCESS);
244 
245 	if (ddi_prop_update_string(DDI_DEV_T_NONE, devi, "device_type",
246 	    "pciex") != DDI_PROP_SUCCESS) {
247 		cmn_err(CE_WARN, "npe:  'device_type' prop create failed");
248 	}
249 
250 	if (ddi_soft_state_zalloc(npe_statep, instance) == DDI_SUCCESS)
251 		pcip = ddi_get_soft_state(npe_statep, instance);
252 
253 	if (pcip == NULL)
254 		return (DDI_FAILURE);
255 
256 	pcip->pci_dip = devi;
257 
258 	pcie_rc_init_bus(devi);
259 
260 	/*
261 	 * Initialize hotplug support on this bus. At minimum
262 	 * (for non hotplug bus) this would create ":devctl" minor
263 	 * node to support DEVCTL_DEVICE_* and DEVCTL_BUS_* ioctls
264 	 * to this bus.
265 	 */
266 	if (pcihp_init(devi) != DDI_SUCCESS) {
267 		cmn_err(CE_WARN, "npe: Failed to setup hotplug framework");
268 		ddi_soft_state_free(npe_statep, instance);
269 		return (DDI_FAILURE);
270 	}
271 
272 	/* Second arg: initialize for pci_express root nexus */
273 	if (pcitool_init(devi, B_TRUE) != DDI_SUCCESS) {
274 		(void) pcihp_uninit(devi);
275 		ddi_soft_state_free(npe_statep, instance);
276 		return (DDI_FAILURE);
277 	}
278 
279 	pcip->pci_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
280 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
281 	ddi_fm_init(devi, &pcip->pci_fmcap, &pcip->pci_fm_ibc);
282 
283 	if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE) {
284 		ddi_fm_handler_register(devi, npe_fm_callback, NULL);
285 	}
286 
287 	PCIE_DIP2PFD(devi) = kmem_zalloc(sizeof (pf_data_t), KM_SLEEP);
288 	pcie_rc_init_pfd(devi, PCIE_DIP2PFD(devi));
289 
290 	npe_query_acpi_mcfg(devi);
291 	ddi_report_dev(devi);
292 	return (DDI_SUCCESS);
293 
294 }
295 
296 /*ARGSUSED*/
297 static int
298 npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
299 {
300 	int instance = ddi_get_instance(devi);
301 	pci_state_t *pcip;
302 
303 	pcip = ddi_get_soft_state(npe_statep, ddi_get_instance(devi));
304 
305 	switch (cmd) {
306 	case DDI_DETACH:
307 
308 		/* Uninitialize pcitool support. */
309 		pcitool_uninit(devi);
310 
311 		/*
312 		 * Uninitialize hotplug support on this bus.
313 		 */
314 		(void) pcihp_uninit(devi);
315 
316 		if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE)
317 			ddi_fm_handler_unregister(devi);
318 
319 		pcie_rc_fini_bus(devi);
320 		pcie_rc_fini_pfd(PCIE_DIP2PFD(devi));
321 		kmem_free(PCIE_DIP2PFD(devi), sizeof (pf_data_t));
322 
323 		ddi_fm_fini(devi);
324 		ddi_soft_state_free(npe_statep, instance);
325 		return (DDI_SUCCESS);
326 
327 	case DDI_SUSPEND:
328 		return (DDI_SUCCESS);
329 	default:
330 		return (DDI_FAILURE);
331 	}
332 }
333 
334 
335 static int
336 npe_bus_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
337     off_t offset, off_t len, caddr_t *vaddrp)
338 {
339 	int 		rnumber;
340 	int		length;
341 	int		space;
342 	ddi_acc_impl_t	*ap;
343 	ddi_acc_hdl_t	*hp;
344 	ddi_map_req_t	mr;
345 	pci_regspec_t	pci_reg;
346 	pci_regspec_t	*pci_rp;
347 	struct regspec	reg;
348 	pci_acc_cfblk_t	*cfp;
349 	int		retval;
350 
351 	mr = *mp; /* Get private copy of request */
352 	mp = &mr;
353 
354 	/*
355 	 * check for register number
356 	 */
357 	switch (mp->map_type) {
358 	case DDI_MT_REGSPEC:
359 		pci_reg = *(pci_regspec_t *)(mp->map_obj.rp);
360 		pci_rp = &pci_reg;
361 		if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
362 			return (DDI_FAILURE);
363 		break;
364 	case DDI_MT_RNUMBER:
365 		rnumber = mp->map_obj.rnumber;
366 		/*
367 		 * get ALL "reg" properties for dip, select the one of
368 		 * of interest. In x86, "assigned-addresses" property
369 		 * is identical to the "reg" property, so there is no
370 		 * need to cross check the two to determine the physical
371 		 * address of the registers.
372 		 * This routine still performs some validity checks to
373 		 * make sure that everything is okay.
374 		 */
375 		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
376 		    DDI_PROP_DONTPASS, "reg", (int **)&pci_rp,
377 		    (uint_t *)&length) != DDI_PROP_SUCCESS)
378 			return (DDI_FAILURE);
379 
380 		/*
381 		 * validate the register number.
382 		 */
383 		length /= (sizeof (pci_regspec_t) / sizeof (int));
384 		if (rnumber >= length) {
385 			ddi_prop_free(pci_rp);
386 			return (DDI_FAILURE);
387 		}
388 
389 		/*
390 		 * copy the required entry.
391 		 */
392 		pci_reg = pci_rp[rnumber];
393 
394 		/*
395 		 * free the memory allocated by ddi_prop_lookup_int_array
396 		 */
397 		ddi_prop_free(pci_rp);
398 
399 		pci_rp = &pci_reg;
400 		if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
401 			return (DDI_FAILURE);
402 		mp->map_type = DDI_MT_REGSPEC;
403 		break;
404 	default:
405 		return (DDI_ME_INVAL);
406 	}
407 
408 	space = pci_rp->pci_phys_hi & PCI_REG_ADDR_M;
409 
410 	/*
411 	 * check for unmap and unlock of address space
412 	 */
413 	if ((mp->map_op == DDI_MO_UNMAP) || (mp->map_op == DDI_MO_UNLOCK)) {
414 		switch (space) {
415 		case PCI_ADDR_IO:
416 			reg.regspec_bustype = 1;
417 			break;
418 
419 		case PCI_ADDR_CONFIG:
420 			/*
421 			 * Check for AMD's northbridges
422 			 *	AND
423 			 * for any PCI device.
424 			 *
425 			 * This is a workaround fix for
426 			 * AMD-8132's inability to handle MMCFG
427 			 * accesses on Galaxy's PE servers
428 			 *	AND
429 			 * to disable MMCFG for any PCI device.
430 			 *
431 			 * If a device is *not* found to have PCIe
432 			 * capability, then assume it is a PCI device.
433 			 */
434 
435 			if (is_amd_northbridge(rdip) == 0 ||
436 			    (ddi_prop_get_int(DDI_DEV_T_ANY, rdip,
437 			    DDI_PROP_DONTPASS, "pcie-capid-pointer",
438 			    PCI_CAP_NEXT_PTR_NULL) == PCI_CAP_NEXT_PTR_NULL)) {
439 				if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
440 				    mp->map_handlep->ah_acc.devacc_attr_access
441 				    != DDI_DEFAULT_ACC) {
442 					ndi_fmc_remove(rdip, ACC_HANDLE,
443 					    (void *)mp->map_handlep);
444 				}
445 				return (DDI_SUCCESS);
446 			}
447 
448 
449 			/* FALLTHROUGH */
450 		case PCI_ADDR_MEM64:
451 			/*
452 			 * MEM64 requires special treatment on map, to check
453 			 * that the device is below 4G.  On unmap, however,
454 			 * we can assume that everything is OK... the map
455 			 * must have succeeded.
456 			 */
457 			/* FALLTHROUGH */
458 		case PCI_ADDR_MEM32:
459 			reg.regspec_bustype = 0;
460 			break;
461 
462 		default:
463 			return (DDI_FAILURE);
464 		}
465 
466 		/*
467 		 * Adjust offset and length
468 		 * A non-zero length means override the one in the regspec.
469 		 */
470 		pci_rp->pci_phys_low += (uint_t)offset;
471 		if (len != 0)
472 			pci_rp->pci_size_low = len;
473 
474 		reg.regspec_addr = pci_rp->pci_phys_low;
475 		reg.regspec_size = pci_rp->pci_size_low;
476 
477 		mp->map_obj.rp = &reg;
478 		retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
479 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
480 		    mp->map_handlep->ah_acc.devacc_attr_access !=
481 		    DDI_DEFAULT_ACC) {
482 			ndi_fmc_remove(rdip, ACC_HANDLE,
483 			    (void *)mp->map_handlep);
484 		}
485 		return (retval);
486 
487 	}
488 
489 	/* check for user mapping request - not legal for Config */
490 	if (mp->map_op == DDI_MO_MAP_HANDLE && space == PCI_ADDR_CONFIG) {
491 		cmn_err(CE_NOTE, "npe: Config mapping request from user\n");
492 		return (DDI_FAILURE);
493 	}
494 
495 
496 	/*
497 	 * Note that pci_fm_acc_setup() is called to serve two purposes
498 	 * i) enable legacy PCI I/O style config space access
499 	 * ii) register with FMA
500 	 */
501 	if (space == PCI_ADDR_CONFIG) {
502 		/* Can't map config space without a handle */
503 		hp = (ddi_acc_hdl_t *)mp->map_handlep;
504 		if (hp == NULL)
505 			return (DDI_FAILURE);
506 
507 		/* record the device address for future reference */
508 		cfp = (pci_acc_cfblk_t *)&hp->ah_bus_private;
509 		cfp->c_busnum = PCI_REG_BUS_G(pci_rp->pci_phys_hi);
510 		cfp->c_devnum = PCI_REG_DEV_G(pci_rp->pci_phys_hi);
511 		cfp->c_funcnum = PCI_REG_FUNC_G(pci_rp->pci_phys_hi);
512 
513 		*vaddrp = (caddr_t)offset;
514 
515 		/*
516 		 * Check for AMD's northbridges, pci devices and
517 		 * devices underneath a pci bridge.  This is to setup
518 		 * I/O based config space access.
519 		 */
520 		if (is_amd_northbridge(rdip) == 0 ||
521 		    (ddi_prop_get_int(DDI_DEV_T_ANY, rdip, DDI_PROP_DONTPASS,
522 		    "pcie-capid-pointer", PCI_CAP_NEXT_PTR_NULL) ==
523 		    PCI_CAP_NEXT_PTR_NULL)) {
524 			int ret;
525 
526 			if ((ret = pci_fm_acc_setup(hp, offset, len)) ==
527 			    DDI_SUCCESS) {
528 				if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
529 				    mp->map_handlep->ah_acc.devacc_attr_access
530 				    != DDI_DEFAULT_ACC) {
531 					ndi_fmc_insert(rdip, ACC_HANDLE,
532 					    (void *)mp->map_handlep, NULL);
533 				}
534 			}
535 			return (ret);
536 		}
537 
538 		pci_rp->pci_phys_low = ddi_prop_get_int64(DDI_DEV_T_ANY,
539 		    rdip, 0, "ecfga-base-address", 0);
540 
541 		pci_rp->pci_phys_low += ((cfp->c_busnum << 20) |
542 		    (cfp->c_devnum) << 15 | (cfp->c_funcnum << 12));
543 
544 		pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
545 	}
546 
547 	length = pci_rp->pci_size_low;
548 
549 	/*
550 	 * range check
551 	 */
552 	if ((offset >= length) || (len > length) || (offset + len > length))
553 		return (DDI_FAILURE);
554 
555 	/*
556 	 * Adjust offset and length
557 	 * A non-zero length means override the one in the regspec.
558 	 */
559 	pci_rp->pci_phys_low += (uint_t)offset;
560 	if (len != 0)
561 		pci_rp->pci_size_low = len;
562 
563 	/*
564 	 * convert the pci regsec into the generic regspec used by the
565 	 * parent root nexus driver.
566 	 */
567 	switch (space) {
568 	case PCI_ADDR_IO:
569 		reg.regspec_bustype = 1;
570 		break;
571 	case PCI_ADDR_CONFIG:
572 	case PCI_ADDR_MEM64:
573 		/*
574 		 * We can't handle 64-bit devices that are mapped above
575 		 * 4G or that are larger than 4G.
576 		 */
577 		if (pci_rp->pci_phys_mid != 0 || pci_rp->pci_size_hi != 0)
578 			return (DDI_FAILURE);
579 		/*
580 		 * Other than that, we can treat them as 32-bit mappings
581 		 */
582 		/* FALLTHROUGH */
583 	case PCI_ADDR_MEM32:
584 		reg.regspec_bustype = 0;
585 		break;
586 	default:
587 		return (DDI_FAILURE);
588 	}
589 
590 	reg.regspec_addr = pci_rp->pci_phys_low;
591 	reg.regspec_size = pci_rp->pci_size_low;
592 
593 	mp->map_obj.rp = &reg;
594 	retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
595 	if (retval == DDI_SUCCESS) {
596 		/*
597 		 * For config space gets force use of cautious access routines.
598 		 * These will handle default and protected mode accesses too.
599 		 */
600 		if (space == PCI_ADDR_CONFIG) {
601 			ap = (ddi_acc_impl_t *)mp->map_handlep;
602 			ap->ahi_acc_attr &= ~DDI_ACCATTR_DIRECT;
603 			ap->ahi_acc_attr |= DDI_ACCATTR_CONFIG_SPACE;
604 			ap->ahi_get8 = i_ddi_caut_get8;
605 			ap->ahi_get16 = i_ddi_caut_get16;
606 			ap->ahi_get32 = i_ddi_caut_get32;
607 			ap->ahi_get64 = i_ddi_caut_get64;
608 			ap->ahi_rep_get8 = i_ddi_caut_rep_get8;
609 			ap->ahi_rep_get16 = i_ddi_caut_rep_get16;
610 			ap->ahi_rep_get32 = i_ddi_caut_rep_get32;
611 			ap->ahi_rep_get64 = i_ddi_caut_rep_get64;
612 		}
613 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
614 		    mp->map_handlep->ah_acc.devacc_attr_access !=
615 		    DDI_DEFAULT_ACC) {
616 			ndi_fmc_insert(rdip, ACC_HANDLE,
617 			    (void *)mp->map_handlep, NULL);
618 		}
619 	}
620 	return (retval);
621 }
622 
623 
624 
625 /*ARGSUSED*/
626 static int
627 npe_ctlops(dev_info_t *dip, dev_info_t *rdip,
628 	ddi_ctl_enum_t ctlop, void *arg, void *result)
629 {
630 	int		rn;
631 	int		totreg;
632 	uint_t		reglen;
633 	pci_regspec_t	*drv_regp;
634 	struct attachspec *asp;
635 	struct detachspec *dsp;
636 	pci_state_t	*pci_p = ddi_get_soft_state(npe_statep,
637 	    ddi_get_instance(dip));
638 
639 	switch (ctlop) {
640 	case DDI_CTLOPS_REPORTDEV:
641 		if (rdip == (dev_info_t *)0)
642 			return (DDI_FAILURE);
643 		cmn_err(CE_CONT, "?PCI Express-device: %s@%s, %s%d\n",
644 		    ddi_node_name(rdip), ddi_get_name_addr(rdip),
645 		    ddi_driver_name(rdip), ddi_get_instance(rdip));
646 		return (DDI_SUCCESS);
647 
648 	case DDI_CTLOPS_INITCHILD:
649 		return (npe_initchild((dev_info_t *)arg));
650 
651 	case DDI_CTLOPS_UNINITCHILD:
652 		return (npe_removechild((dev_info_t *)arg));
653 
654 	case DDI_CTLOPS_SIDDEV:
655 		return (DDI_SUCCESS);
656 
657 	case DDI_CTLOPS_REGSIZE:
658 	case DDI_CTLOPS_NREGS:
659 		if (rdip == (dev_info_t *)0)
660 			return (DDI_FAILURE);
661 
662 		*(int *)result = 0;
663 		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
664 		    DDI_PROP_DONTPASS, "reg", (int **)&drv_regp,
665 		    &reglen) != DDI_PROP_SUCCESS) {
666 			return (DDI_FAILURE);
667 		}
668 
669 		totreg = (reglen * sizeof (int)) / sizeof (pci_regspec_t);
670 		if (ctlop == DDI_CTLOPS_NREGS)
671 			*(int *)result = totreg;
672 		else if (ctlop == DDI_CTLOPS_REGSIZE) {
673 			rn = *(int *)arg;
674 			if (rn >= totreg) {
675 				ddi_prop_free(drv_regp);
676 				return (DDI_FAILURE);
677 			}
678 			*(off_t *)result = drv_regp[rn].pci_size_low;
679 		}
680 		ddi_prop_free(drv_regp);
681 
682 		return (DDI_SUCCESS);
683 
684 	case DDI_CTLOPS_POWER:
685 	{
686 		power_req_t	*reqp = (power_req_t *)arg;
687 		/*
688 		 * We currently understand reporting of PCI_PM_IDLESPEED
689 		 * capability. Everything else is passed up.
690 		 */
691 		if ((reqp->request_type == PMR_REPORT_PMCAP) &&
692 		    (reqp->req.report_pmcap_req.cap ==  PCI_PM_IDLESPEED))
693 			return (DDI_SUCCESS);
694 
695 		break;
696 	}
697 
698 	case DDI_CTLOPS_PEEK:
699 	case DDI_CTLOPS_POKE:
700 		return (pci_common_peekpoke(dip, rdip, ctlop, arg, result));
701 
702 	/* X86 systems support PME wakeup from suspended state */
703 	case DDI_CTLOPS_ATTACH:
704 		if (!pcie_is_child(dip, rdip))
705 			return (DDI_SUCCESS);
706 
707 		asp = (struct attachspec *)arg;
708 		if ((asp->when == DDI_POST) && (asp->result == DDI_SUCCESS)) {
709 			pf_init(rdip, (void *)pci_p->pci_fm_ibc, asp->cmd);
710 			(void) pcie_postattach_child(rdip);
711 		}
712 
713 		/* only do this for immediate children */
714 		if (asp->cmd == DDI_RESUME && asp->when == DDI_PRE &&
715 		    ddi_get_parent(rdip) == dip)
716 			if (pci_pre_resume(rdip) != DDI_SUCCESS) {
717 				/* Not good, better stop now. */
718 				cmn_err(CE_PANIC,
719 				    "Couldn't pre-resume device %p",
720 				    (void *) dip);
721 				/* NOTREACHED */
722 			}
723 
724 		return (DDI_SUCCESS);
725 
726 	case DDI_CTLOPS_DETACH:
727 		if (!pcie_is_child(dip, rdip))
728 			return (DDI_SUCCESS);
729 
730 		dsp = (struct detachspec *)arg;
731 
732 		if (dsp->when == DDI_PRE)
733 			pf_fini(rdip, dsp->cmd);
734 
735 		/* only do this for immediate children */
736 		if (dsp->cmd == DDI_SUSPEND && dsp->when == DDI_POST &&
737 		    ddi_get_parent(rdip) == dip)
738 			if (pci_post_suspend(rdip) != DDI_SUCCESS)
739 				return (DDI_FAILURE);
740 
741 		return (DDI_SUCCESS);
742 
743 	default:
744 		break;
745 	}
746 
747 	return (ddi_ctlops(dip, rdip, ctlop, arg, result));
748 
749 }
750 
751 
752 /*
753  * npe_intr_ops
754  */
755 static int
756 npe_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
757     ddi_intr_handle_impl_t *hdlp, void *result)
758 {
759 	return (pci_common_intr_ops(pdip, rdip, intr_op, hdlp, result));
760 }
761 
762 
763 static int
764 npe_initchild(dev_info_t *child)
765 {
766 	char		name[80];
767 	pcie_bus_t	*bus_p;
768 	uint32_t	regs;
769 	ddi_acc_handle_t	cfg_hdl;
770 
771 	/*
772 	 * Do not bind drivers to empty bridges.
773 	 * Fail above, if the bridge is found to be hotplug capable
774 	 */
775 	if (npe_disable_empty_bridges_workaround(child) == 1)
776 		return (DDI_FAILURE);
777 
778 	if (pci_common_name_child(child, name, 80) != DDI_SUCCESS)
779 		return (DDI_FAILURE);
780 
781 	ddi_set_name_addr(child, name);
782 
783 	/*
784 	 * Pseudo nodes indicate a prototype node with per-instance
785 	 * properties to be merged into the real h/w device node.
786 	 * The interpretation of the unit-address is DD[,F]
787 	 * where DD is the device id and F is the function.
788 	 */
789 	if (ndi_dev_is_persistent_node(child) == 0) {
790 		extern int pci_allow_pseudo_children;
791 
792 		ddi_set_parent_data(child, NULL);
793 
794 		/*
795 		 * Try to merge the properties from this prototype
796 		 * node into real h/w nodes.
797 		 */
798 		if (ndi_merge_node(child, pci_common_name_child) ==
799 		    DDI_SUCCESS) {
800 			/*
801 			 * Merged ok - return failure to remove the node.
802 			 */
803 			ddi_set_name_addr(child, NULL);
804 			return (DDI_FAILURE);
805 		}
806 
807 		/* workaround for DDIVS to run under PCI Express */
808 		if (pci_allow_pseudo_children) {
809 			/*
810 			 * If the "interrupts" property doesn't exist,
811 			 * this must be the ddivs no-intr case, and it returns
812 			 * DDI_SUCCESS instead of DDI_FAILURE.
813 			 */
814 			if (ddi_prop_get_int(DDI_DEV_T_ANY, child,
815 			    DDI_PROP_DONTPASS, "interrupts", -1) == -1)
816 				return (DDI_SUCCESS);
817 			/*
818 			 * Create the ddi_parent_private_data for a pseudo
819 			 * child.
820 			 */
821 			pci_common_set_parent_private_data(child);
822 			return (DDI_SUCCESS);
823 		}
824 
825 		/*
826 		 * The child was not merged into a h/w node,
827 		 * but there's not much we can do with it other
828 		 * than return failure to cause the node to be removed.
829 		 */
830 		cmn_err(CE_WARN, "!%s@%s: %s.conf properties not merged",
831 		    ddi_get_name(child), ddi_get_name_addr(child),
832 		    ddi_get_name(child));
833 		ddi_set_name_addr(child, NULL);
834 		return (DDI_NOT_WELL_FORMED);
835 	}
836 
837 	if (ddi_prop_get_int(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
838 	    "interrupts", -1) != -1)
839 		pci_common_set_parent_private_data(child);
840 	else
841 		ddi_set_parent_data(child, NULL);
842 
843 	/* Disable certain errors on PCIe drivers for x86 platforms */
844 	regs = pcie_get_aer_uce_mask() | npe_aer_uce_mask;
845 	pcie_set_aer_uce_mask(regs);
846 	regs = pcie_get_aer_ce_mask() | npe_aer_ce_mask;
847 	pcie_set_aer_ce_mask(regs);
848 	regs = pcie_get_aer_suce_mask() | npe_aer_suce_mask;
849 	pcie_set_aer_suce_mask(regs);
850 
851 	/*
852 	 * If URs are disabled, mask SERRs as well, otherwise the system will
853 	 * still be notified of URs
854 	 */
855 	if (npe_aer_uce_mask & PCIE_AER_UCE_UR)
856 		pcie_set_serr_mask(1);
857 
858 	if (pci_config_setup(child, &cfg_hdl) == DDI_SUCCESS) {
859 		npe_ck804_fix_aer_ptr(cfg_hdl);
860 		npe_nvidia_error_mask(cfg_hdl);
861 		npe_intel_error_mask(cfg_hdl);
862 		pci_config_teardown(&cfg_hdl);
863 	}
864 
865 	bus_p = pcie_init_bus(child);
866 	if (bus_p) {
867 		uint16_t device_id = (uint16_t)(bus_p->bus_dev_ven_id >> 16);
868 		uint16_t vendor_id = (uint16_t)(bus_p->bus_dev_ven_id & 0xFFFF);
869 		uint16_t rev_id = bus_p->bus_rev_id;
870 
871 		/* Disable AER for certain NVIDIA Chipsets */
872 		if ((vendor_id == NVIDIA_VENDOR_ID) &&
873 		    (device_id == NVIDIA_CK804_DEVICE_ID) &&
874 		    (rev_id < NVIDIA_CK804_AER_VALID_REVID))
875 			bus_p->bus_aer_off = 0;
876 
877 		(void) pcie_initchild(child);
878 	}
879 
880 	return (DDI_SUCCESS);
881 }
882 
883 
884 static int
885 npe_removechild(dev_info_t *dip)
886 {
887 	pcie_uninitchild(dip);
888 
889 	ddi_set_name_addr(dip, NULL);
890 
891 	/*
892 	 * Strip the node to properly convert it back to prototype form
893 	 */
894 	ddi_remove_minor_node(dip, NULL);
895 
896 	ddi_prop_remove_all(dip);
897 
898 	return (DDI_SUCCESS);
899 }
900 
901 
902 /*
903  * When retrofitting this module for pci_tools, functions such as open, close,
904  * and ioctl are now pulled into this module.  Before this, the functions in
905  * the pcihp module were referenced directly.  Now they are called or
906  * referenced through the pcihp cb_ops structure from functions in this module.
907  */
908 static int
909 npe_open(dev_t *devp, int flags, int otyp, cred_t *credp)
910 {
911 	return ((pcihp_get_cb_ops())->cb_open(devp, flags, otyp, credp));
912 }
913 
914 static int
915 npe_close(dev_t dev, int flags, int otyp, cred_t *credp)
916 {
917 	return ((pcihp_get_cb_ops())->cb_close(dev, flags, otyp, credp));
918 }
919 
920 static int
921 npe_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
922 {
923 	minor_t		minor = getminor(dev);
924 	int		instance = PCIHP_AP_MINOR_NUM_TO_INSTANCE(minor);
925 	pci_state_t	*pci_p = ddi_get_soft_state(npe_statep, instance);
926 	dev_info_t	*dip;
927 
928 	if (pci_p == NULL)
929 		return (ENXIO);
930 
931 	dip = pci_p->pci_dip;
932 
933 	return (pci_common_ioctl(dip, dev, cmd, arg, mode, credp, rvalp));
934 }
935 
936 static int
937 npe_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op,
938 	int flags, char *name, caddr_t valuep, int *lengthp)
939 {
940 	return ((pcihp_get_cb_ops())->cb_prop_op(dev, dip, prop_op, flags,
941 	    name, valuep, lengthp));
942 }
943 
944 static int
945 npe_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
946 {
947 	return (pcihp_info(dip, cmd, arg, result));
948 }
949 
950 /*ARGSUSED*/
951 static int
952 npe_fm_init(dev_info_t *dip, dev_info_t *tdip, int cap,
953     ddi_iblock_cookie_t *ibc)
954 {
955 	pci_state_t  *pcip = ddi_get_soft_state(npe_statep,
956 	    ddi_get_instance(dip));
957 
958 	ASSERT(ibc != NULL);
959 	*ibc = pcip->pci_fm_ibc;
960 
961 	return (pcip->pci_fmcap);
962 }
963 
964 /*ARGSUSED*/
965 static int
966 npe_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr, const void *no_used)
967 {
968 	/*
969 	 * On current x86 systems, npe's callback does not get called for failed
970 	 * loads.  If in the future this feature is used, the fault PA should be
971 	 * logged in the derr->fme_bus_specific field.  The appropriate PCIe
972 	 * error handling code should be called and needs to be coordinated with
973 	 * safe access handling.
974 	 */
975 
976 	return (DDI_FM_OK);
977 }
978