xref: /illumos-gate/usr/src/uts/sun4v/io/px/px_lib4v.c (revision 3c5e027b)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/sysmacros.h>
28 #include <sys/ddi.h>
29 #include <sys/async.h>
30 #include <sys/sunddi.h>
31 #include <sys/ddifm.h>
32 #include <sys/fm/protocol.h>
33 #include <sys/vmem.h>
34 #include <sys/intr.h>
35 #include <sys/ivintr.h>
36 #include <sys/errno.h>
37 #include <sys/hypervisor_api.h>
38 #include <sys/hsvc.h>
39 #include <px_obj.h>
40 #include <sys/machsystm.h>
41 #include <sys/hotplug/pci/pcihp.h>
42 #include "px_lib4v.h"
43 #include "px_err.h"
44 #include <vm/vm_dep.h>
45 #include <vm/hat_sfmmu.h>
46 
47 /* mask for the ranges property in calculating the real PFN range */
48 uint_t px_ranges_phi_mask = ((1 << 28) -1);
49 
50 /*
51  * Hypervisor VPCI services information for the px nexus driver.
52  */
53 static	uint64_t	px_vpci_min_ver; /* Negotiated VPCI API minor version */
54 static	uint_t		px_vpci_users = 0; /* VPCI API users */
55 
56 static hsvc_info_t px_hsvc = {
57 	HSVC_REV_1, NULL, HSVC_GROUP_VPCI, PX_VPCI_MAJOR_VER,
58 	PX_VPCI_MINOR_VER, "PX"
59 };
60 
61 int
62 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl)
63 {
64 	px_nexus_regspec_t	*rp;
65 	uint_t			reglen;
66 	int			ret;
67 
68 	uint64_t mjrnum;
69 	uint64_t mnrnum;
70 
71 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p\n", dip);
72 
73 	/*
74 	 * Check HV intr group api versioning.
75 	 * This driver uses the old interrupt routines which are supported
76 	 * in old firmware in the CORE API group and in newer firmware in
77 	 * the INTR API group.  Support for these calls will be dropped
78 	 * once the INTR API group major goes to 2.
79 	 */
80 	if ((hsvc_version(HSVC_GROUP_INTR, &mjrnum, &mnrnum) == 0) &&
81 	    (mjrnum > 1)) {
82 		cmn_err(CE_WARN, "niumx: unsupported intr api group: "
83 		    "maj:0x%lx, min:0x%lx", mjrnum, mnrnum);
84 		return (ENOTSUP);
85 	}
86 
87 	ret = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
88 	    "reg", (uchar_t **)&rp, &reglen);
89 	if (ret != DDI_PROP_SUCCESS) {
90 		DBG(DBG_ATTACH, dip, "px_lib_dev_init failed ret=%d\n", ret);
91 		return (DDI_FAILURE);
92 	}
93 
94 	/*
95 	 * Initilize device handle. The device handle uniquely identifies
96 	 * a SUN4V device. It consists of the lower 28-bits of the hi-cell
97 	 * of the first entry of the SUN4V device's "reg" property as
98 	 * defined by the SUN4V Bus Binding to Open Firmware.
99 	 */
100 	*dev_hdl = (devhandle_t)((rp->phys_addr >> 32) & DEVHDLE_MASK);
101 	ddi_prop_free(rp);
102 
103 	/*
104 	 * hotplug implementation requires this property to be associated with
105 	 * any indirect PCI config access services
106 	 */
107 	(void) ddi_prop_update_int(makedevice(ddi_driver_major(dip),
108 	    PCIHP_AP_MINOR_NUM(ddi_get_instance(dip), PCIHP_DEVCTL_MINOR)), dip,
109 	    PCI_BUS_CONF_MAP_PROP, 1);
110 
111 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl);
112 
113 	/*
114 	 * Negotiate the API version for VPCI hypervisor services.
115 	 */
116 	if (px_vpci_users++)
117 		return (DDI_SUCCESS);
118 
119 	if ((ret = hsvc_register(&px_hsvc, &px_vpci_min_ver)) != 0) {
120 		cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services "
121 		    "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d\n",
122 		    px_hsvc.hsvc_modname, px_hsvc.hsvc_group,
123 		    px_hsvc.hsvc_major, px_hsvc.hsvc_minor, ret);
124 
125 		return (DDI_FAILURE);
126 	}
127 
128 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated VPCI API version, "
129 	    "major 0x%lx minor 0x%lx\n", px_hsvc.hsvc_major, px_vpci_min_ver);
130 
131 	return (DDI_SUCCESS);
132 }
133 
134 /*ARGSUSED*/
135 int
136 px_lib_dev_fini(dev_info_t *dip)
137 {
138 	DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip);
139 
140 	(void) ddi_prop_remove(makedevice(ddi_driver_major(dip),
141 	    PCIHP_AP_MINOR_NUM(ddi_get_instance(dip), PCIHP_DEVCTL_MINOR)), dip,
142 	    PCI_BUS_CONF_MAP_PROP);
143 
144 	if (--px_vpci_users == 0)
145 		(void) hsvc_unregister(&px_hsvc);
146 
147 	return (DDI_SUCCESS);
148 }
149 
150 /*ARGSUSED*/
151 int
152 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino,
153     sysino_t *sysino)
154 {
155 	uint64_t	ret;
156 
157 	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p "
158 	    "devino 0x%x\n", dip, devino);
159 
160 	if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip),
161 	    devino, sysino)) != H_EOK) {
162 		DBG(DBG_LIB_INT, dip,
163 		    "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret);
164 		return (DDI_FAILURE);
165 	}
166 
167 	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n",
168 	    *sysino);
169 
170 	return (DDI_SUCCESS);
171 }
172 
173 /*ARGSUSED*/
174 int
175 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino,
176     intr_valid_state_t *intr_valid_state)
177 {
178 	uint64_t	ret;
179 
180 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n",
181 	    dip, sysino);
182 
183 	if ((ret = hvio_intr_getvalid(sysino,
184 	    (int *)intr_valid_state)) != H_EOK) {
185 		DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n",
186 		    ret);
187 		return (DDI_FAILURE);
188 	}
189 
190 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n",
191 	    *intr_valid_state);
192 
193 	return (DDI_SUCCESS);
194 }
195 
196 /*ARGSUSED*/
197 int
198 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino,
199     intr_valid_state_t intr_valid_state)
200 {
201 	uint64_t	ret;
202 
203 	DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx "
204 	    "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state);
205 
206 	if ((ret = hvio_intr_setvalid(sysino, intr_valid_state)) != H_EOK) {
207 		DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n",
208 		    ret);
209 		return (DDI_FAILURE);
210 	}
211 
212 	return (DDI_SUCCESS);
213 }
214 
215 /*ARGSUSED*/
216 int
217 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino,
218     intr_state_t *intr_state)
219 {
220 	uint64_t	ret;
221 
222 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n",
223 	    dip, sysino);
224 
225 	if ((ret = hvio_intr_getstate(sysino, (int *)intr_state)) != H_EOK) {
226 		DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n",
227 		    ret);
228 		return (DDI_FAILURE);
229 	}
230 
231 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n",
232 	    *intr_state);
233 
234 	return (DDI_SUCCESS);
235 }
236 
237 /*ARGSUSED*/
238 int
239 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino,
240     intr_state_t intr_state)
241 {
242 	uint64_t	ret;
243 
244 	DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx "
245 	    "intr_state 0x%x\n", dip, sysino, intr_state);
246 
247 	if ((ret = hvio_intr_setstate(sysino, intr_state)) != H_EOK) {
248 		DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n",
249 		    ret);
250 		return (DDI_FAILURE);
251 	}
252 
253 	return (DDI_SUCCESS);
254 }
255 
256 /*ARGSUSED*/
257 int
258 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid)
259 {
260 	uint64_t	ret;
261 
262 	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n",
263 	    dip, sysino);
264 
265 	if ((ret = hvio_intr_gettarget(sysino, cpuid)) != H_EOK) {
266 		DBG(DBG_LIB_INT, dip,
267 		    "hvio_intr_gettarget failed, ret 0x%lx\n", ret);
268 		return (DDI_FAILURE);
269 	}
270 
271 	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", *cpuid);
272 
273 	return (DDI_SUCCESS);
274 }
275 
276 /*ARGSUSED*/
277 int
278 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid)
279 {
280 	uint64_t	ret;
281 
282 	DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx "
283 	    "cpuid 0x%x\n", dip, sysino, cpuid);
284 
285 	ret = hvio_intr_settarget(sysino, cpuid);
286 	if (ret == H_ECPUERROR) {
287 		cmn_err(CE_PANIC,
288 		    "px_lib_intr_settarget: hvio_intr_settarget failed, "
289 		    "ret = 0x%lx, cpuid = 0x%x, sysino = 0x%lx\n", ret,
290 		    cpuid, sysino);
291 	} else if (ret != H_EOK) {
292 		DBG(DBG_LIB_INT, dip,
293 		    "hvio_intr_settarget failed, ret 0x%lx\n", ret);
294 		return (DDI_FAILURE);
295 	}
296 
297 	return (DDI_SUCCESS);
298 }
299 
300 /*ARGSUSED*/
301 int
302 px_lib_intr_reset(dev_info_t *dip)
303 {
304 	px_t		*px_p = DIP_TO_STATE(dip);
305 	px_ib_t		*ib_p = px_p->px_ib_p;
306 	px_ino_t	*ino_p;
307 
308 	DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip);
309 
310 	mutex_enter(&ib_p->ib_ino_lst_mutex);
311 
312 	/* Reset all Interrupts */
313 	for (ino_p = ib_p->ib_ino_lst; ino_p; ino_p = ino_p->ino_next_p) {
314 		if (px_lib_intr_setstate(dip, ino_p->ino_sysino,
315 		    INTR_IDLE_STATE) != DDI_SUCCESS)
316 			return (BF_FATAL);
317 	}
318 
319 	mutex_exit(&ib_p->ib_ino_lst_mutex);
320 
321 	return (BF_NONE);
322 }
323 
324 /*ARGSUSED*/
325 int
326 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages,
327     io_attributes_t attr, void *addr, size_t pfn_index, int flags)
328 {
329 	tsbnum_t	tsb_num = PCI_TSBID_TO_TSBNUM(tsbid);
330 	tsbindex_t	tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid);
331 	io_page_list_t	*pfns, *pfn_p;
332 	pages_t		ttes_mapped = 0;
333 	int		i, err = DDI_SUCCESS;
334 
335 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx "
336 	    "pages 0x%x attr 0x%llx addr 0x%p pfn_index 0x%llx flags 0x%x\n",
337 	    dip, tsbid, pages, attr, addr, pfn_index, flags);
338 
339 	if ((pfns = pfn_p = kmem_zalloc((pages * sizeof (io_page_list_t)),
340 	    KM_NOSLEEP)) == NULL) {
341 		DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: kmem_zalloc failed\n");
342 		return (DDI_FAILURE);
343 	}
344 
345 	for (i = 0; i < pages; i++)
346 		pfns[i] = MMU_PTOB(PX_ADDR2PFN(addr, pfn_index, flags, i));
347 
348 	/*
349 	 * If HV VPCI version is 1.1 and higher, pass BDF, phantom function,
350 	 * and relaxed ordering attributes. Otherwise, pass only read or write
351 	 * attribute.
352 	 */
353 	if (px_vpci_min_ver == PX_VPCI_MINOR_VER_0)
354 		attr = attr & (PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE);
355 
356 	while ((ttes_mapped = pfn_p - pfns) < pages) {
357 		uintptr_t	ra = va_to_pa(pfn_p);
358 		pages_t		ttes2map;
359 		uint64_t	ret;
360 
361 		ttes2map = (MMU_PAGE_SIZE - P2PHASE(ra, MMU_PAGE_SIZE)) >> 3;
362 		ra = MMU_PTOB(MMU_BTOP(ra));
363 
364 		for (ttes2map = MIN(ttes2map, pages - ttes_mapped); ttes2map;
365 		    ttes2map -= ttes_mapped, pfn_p += ttes_mapped) {
366 
367 			ttes_mapped = 0;
368 			if ((ret = hvio_iommu_map(DIP_TO_HANDLE(dip),
369 			    PCI_TSBID(tsb_num, tsb_index + (pfn_p - pfns)),
370 			    ttes2map, attr, (io_page_list_t *)(ra |
371 			    ((uintptr_t)pfn_p & MMU_PAGE_OFFSET)),
372 			    &ttes_mapped)) != H_EOK) {
373 				DBG(DBG_LIB_DMA, dip, "hvio_iommu_map failed "
374 				    "ret 0x%lx\n", ret);
375 
376 				ttes_mapped = pfn_p - pfns;
377 				err = DDI_FAILURE;
378 				goto cleanup;
379 			}
380 
381 			DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: tsb_num 0x%x "
382 			    "tsb_index 0x%lx ttes_to_map 0x%lx attr 0x%llx "
383 			    "ra 0x%p ttes_mapped 0x%x\n", tsb_num,
384 			    tsb_index + (pfn_p - pfns), ttes2map, attr,
385 			    ra | ((uintptr_t)pfn_p & MMU_PAGE_OFFSET),
386 			    ttes_mapped);
387 		}
388 	}
389 
390 cleanup:
391 	if ((err == DDI_FAILURE) && ttes_mapped)
392 		(void) px_lib_iommu_demap(dip, tsbid, ttes_mapped);
393 
394 	kmem_free(pfns, pages * sizeof (io_page_list_t));
395 	return (err);
396 }
397 
398 /*ARGSUSED*/
399 int
400 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages)
401 {
402 	tsbnum_t	tsb_num = PCI_TSBID_TO_TSBNUM(tsbid);
403 	tsbindex_t	tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid);
404 	pages_t		ttes2demap, ttes_demapped = 0;
405 	uint64_t	ret;
406 
407 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx "
408 	    "pages 0x%x\n", dip, tsbid, pages);
409 
410 	for (ttes2demap = pages; ttes2demap;
411 	    ttes2demap -= ttes_demapped, tsb_index += ttes_demapped) {
412 		if ((ret = hvio_iommu_demap(DIP_TO_HANDLE(dip),
413 		    PCI_TSBID(tsb_num, tsb_index), ttes2demap,
414 		    &ttes_demapped)) != H_EOK) {
415 			DBG(DBG_LIB_DMA, dip, "hvio_iommu_demap failed, "
416 			    "ret 0x%lx\n", ret);
417 
418 			return (DDI_FAILURE);
419 		}
420 
421 		DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: tsb_num 0x%x "
422 		    "tsb_index 0x%lx ttes_to_demap 0x%lx ttes_demapped 0x%x\n",
423 		    tsb_num, tsb_index, ttes2demap, ttes_demapped);
424 	}
425 
426 	return (DDI_SUCCESS);
427 }
428 
429 /*ARGSUSED*/
430 int
431 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p,
432     r_addr_t *r_addr_p)
433 {
434 	uint64_t	ret;
435 
436 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n",
437 	    dip, tsbid);
438 
439 	if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), tsbid,
440 	    attr_p, r_addr_p)) != H_EOK) {
441 		DBG(DBG_LIB_DMA, dip,
442 		    "hvio_iommu_getmap failed, ret 0x%lx\n", ret);
443 
444 		return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE);
445 	}
446 
447 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%llx "
448 	    "r_addr 0x%llx\n", *attr_p, *r_addr_p);
449 
450 	return (DDI_SUCCESS);
451 }
452 
453 /*ARGSUSED*/
454 uint64_t
455 px_get_rng_parent_hi_mask(px_t *px_p)
456 {
457 	return (PX_RANGE_PROP_MASK);
458 }
459 
460 /*
461  * Checks dma attributes against system bypass ranges
462  * A sun4v device must be capable of generating the entire 64-bit
463  * address in order to perform bypass DMA.
464  */
465 /*ARGSUSED*/
466 int
467 px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p,
468     uint64_t *lo_p, uint64_t *hi_p)
469 {
470 	if ((attr_p->dma_attr_addr_lo != 0ull) ||
471 	    (attr_p->dma_attr_addr_hi != UINT64_MAX)) {
472 
473 		return (DDI_DMA_BADATTR);
474 	}
475 
476 	*lo_p = 0ull;
477 	*hi_p = UINT64_MAX;
478 
479 	return (DDI_SUCCESS);
480 }
481 
482 
483 /*ARGSUSED*/
484 int
485 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr,
486     io_addr_t *io_addr_p)
487 {
488 	uint64_t	ret;
489 
490 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx "
491 	    "attr 0x%llx\n", dip, ra, attr);
492 
493 	if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), ra,
494 	    attr, io_addr_p)) != H_EOK) {
495 		DBG(DBG_LIB_DMA, dip,
496 		    "hvio_iommu_getbypass failed, ret 0x%lx\n", ret);
497 		return (ret == H_ENOTSUPPORTED ? DDI_ENOTSUP : DDI_FAILURE);
498 	}
499 
500 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n",
501 	    *io_addr_p);
502 
503 	return (DDI_SUCCESS);
504 }
505 
506 /*
507  * Returns any needed IO address bit(s) for relaxed ordering in IOMMU
508  * bypass mode.
509  */
510 /* ARGSUSED */
511 uint64_t
512 px_lib_ro_bypass(dev_info_t *dip, io_attributes_t attr, uint64_t ioaddr)
513 {
514 	return (ioaddr);
515 }
516 
517 /*ARGSUSED*/
518 int
519 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
520 	off_t off, size_t len, uint_t cache_flags)
521 {
522 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
523 	uint64_t sync_dir;
524 	size_t bytes_synced;
525 	int end, idx;
526 	off_t pg_off;
527 	devhandle_t hdl = DIP_TO_HANDLE(dip); /* need to cache hdl */
528 
529 	DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p "
530 	    "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n",
531 	    dip, rdip, handle, off, len, cache_flags);
532 
533 	if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) {
534 		cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.",
535 		    ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp);
536 		return (DDI_FAILURE);
537 	}
538 
539 	if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC)
540 		return (DDI_SUCCESS);
541 
542 	if (!len)
543 		len = mp->dmai_size;
544 
545 	if (mp->dmai_rflags & DDI_DMA_READ)
546 		sync_dir = HVIO_DMA_SYNC_DIR_FROM_DEV;
547 	else
548 		sync_dir = HVIO_DMA_SYNC_DIR_TO_DEV;
549 
550 	if (force_sync_icache_after_dma == 0 && !icache_is_coherent)
551 		sync_dir |= HVIO_DMA_SYNC_DIR_NO_ICACHE_FLUSH;
552 
553 	off += mp->dmai_offset;
554 	pg_off = off & MMU_PAGEOFFSET;
555 
556 	DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: page offset %x size %x\n",
557 	    pg_off, len);
558 
559 	/* sync on page basis */
560 	end = MMU_BTOPR(off + len - 1);
561 	for (idx = MMU_BTOP(off); idx < end; idx++,
562 	    len -= bytes_synced, pg_off = 0) {
563 		size_t bytes_to_sync =  MIN(len, MMU_PAGESIZE - pg_off);
564 
565 		while (hvio_dma_sync(hdl,
566 		    MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + pg_off,
567 		    bytes_to_sync, sync_dir, &bytes_synced) != H_EOK) {
568 
569 			if (!(sync_dir & HVIO_DMA_SYNC_DIR_NO_ICACHE_FLUSH)) {
570 				bytes_synced = 0;
571 				break;
572 			}
573 
574 			/*
575 			 * Some versions of firmware do not support
576 			 * this sync_dir flag. If the call fails clear
577 			 * the flag and retry the call. Also, set the
578 			 * global so that we dont set the sync_dir
579 			 * flag again.
580 			 */
581 			sync_dir &= ~HVIO_DMA_SYNC_DIR_NO_ICACHE_FLUSH;
582 			force_sync_icache_after_dma = 1;
583 		}
584 
585 		DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: Called hvio_dma_sync "
586 		    "ra = %p bytes to sync = %x bytes synced %x\n",
587 		    MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + pg_off, bytes_to_sync,
588 		    bytes_synced);
589 
590 		if (bytes_to_sync != bytes_synced)
591 			break;
592 	}
593 
594 	return (len ? DDI_FAILURE : DDI_SUCCESS);
595 }
596 
597 
598 /*
599  * MSIQ Functions:
600  */
601 
602 /*ARGSUSED*/
603 int
604 px_lib_msiq_init(dev_info_t *dip)
605 {
606 	px_t		*px_p = DIP_TO_STATE(dip);
607 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
608 	r_addr_t	ra;
609 	size_t		msiq_size;
610 	uint_t		rec_cnt;
611 	int		i, err = DDI_SUCCESS;
612 	uint64_t	ret;
613 
614 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip);
615 
616 	msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
617 
618 	/* sun4v requires all EQ allocation to be on q size boundary */
619 	if ((msiq_state_p->msiq_buf_p = contig_mem_alloc_align(
620 	    msiq_state_p->msiq_cnt * msiq_size, msiq_size)) == NULL) {
621 		DBG(DBG_LIB_MSIQ, dip,
622 		    "px_lib_msiq_init: Contig alloc failed\n");
623 
624 		return (DDI_FAILURE);
625 	}
626 
627 	for (i = 0; i < msiq_state_p->msiq_cnt; i++) {
628 		msiq_state_p->msiq_p[i].msiq_base_p = (msiqhead_t *)
629 		    ((caddr_t)msiq_state_p->msiq_buf_p + (i * msiq_size));
630 
631 		ra = (r_addr_t)va_to_pa((caddr_t)msiq_state_p->msiq_buf_p +
632 		    (i * msiq_size));
633 
634 		if ((ret = hvio_msiq_conf(DIP_TO_HANDLE(dip),
635 		    (i + msiq_state_p->msiq_1st_msiq_id),
636 		    ra, msiq_state_p->msiq_rec_cnt)) != H_EOK) {
637 			DBG(DBG_LIB_MSIQ, dip,
638 			    "hvio_msiq_conf failed, ret 0x%lx\n", ret);
639 			err = DDI_FAILURE;
640 			break;
641 		}
642 
643 		if ((err = px_lib_msiq_info(dip,
644 		    (i + msiq_state_p->msiq_1st_msiq_id),
645 		    &ra, &rec_cnt)) != DDI_SUCCESS) {
646 			DBG(DBG_LIB_MSIQ, dip,
647 			    "px_lib_msiq_info failed, ret 0x%x\n", err);
648 			err = DDI_FAILURE;
649 			break;
650 		}
651 
652 		DBG(DBG_LIB_MSIQ, dip,
653 		    "px_lib_msiq_init: ra 0x%p rec_cnt 0x%x\n", ra, rec_cnt);
654 	}
655 
656 	return (err);
657 }
658 
659 /*ARGSUSED*/
660 int
661 px_lib_msiq_fini(dev_info_t *dip)
662 {
663 	px_t		*px_p = DIP_TO_STATE(dip);
664 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
665 	size_t		msiq_size;
666 
667 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip);
668 	msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
669 
670 	if (msiq_state_p->msiq_buf_p != NULL)
671 		contig_mem_free(msiq_state_p->msiq_buf_p,
672 		    msiq_state_p->msiq_cnt * msiq_size);
673 
674 	return (DDI_SUCCESS);
675 }
676 
677 /*ARGSUSED*/
678 int
679 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p,
680     uint_t *msiq_rec_cnt_p)
681 {
682 	uint64_t	ret;
683 
684 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n",
685 	    dip, msiq_id);
686 
687 	if ((ret = hvio_msiq_info(DIP_TO_HANDLE(dip),
688 	    msiq_id, ra_p, msiq_rec_cnt_p)) != H_EOK) {
689 		DBG(DBG_LIB_MSIQ, dip,
690 		    "hvio_msiq_info failed, ret 0x%lx\n", ret);
691 		return (DDI_FAILURE);
692 	}
693 
694 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n",
695 	    ra_p, *msiq_rec_cnt_p);
696 
697 	return (DDI_SUCCESS);
698 }
699 
700 /*ARGSUSED*/
701 int
702 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id,
703     pci_msiq_valid_state_t *msiq_valid_state)
704 {
705 	uint64_t	ret;
706 
707 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n",
708 	    dip, msiq_id);
709 
710 	if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip),
711 	    msiq_id, msiq_valid_state)) != H_EOK) {
712 		DBG(DBG_LIB_MSIQ, dip,
713 		    "hvio_msiq_getvalid failed, ret 0x%lx\n", ret);
714 		return (DDI_FAILURE);
715 	}
716 
717 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n",
718 	    *msiq_valid_state);
719 
720 	return (DDI_SUCCESS);
721 }
722 
723 /*ARGSUSED*/
724 int
725 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id,
726     pci_msiq_valid_state_t msiq_valid_state)
727 {
728 	uint64_t	ret;
729 
730 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x "
731 	    "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state);
732 
733 	if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip),
734 	    msiq_id, msiq_valid_state)) != H_EOK) {
735 		DBG(DBG_LIB_MSIQ, dip,
736 		    "hvio_msiq_setvalid failed, ret 0x%lx\n", ret);
737 		return (DDI_FAILURE);
738 	}
739 
740 	return (DDI_SUCCESS);
741 }
742 
743 /*ARGSUSED*/
744 int
745 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id,
746     pci_msiq_state_t *msiq_state)
747 {
748 	uint64_t	ret;
749 
750 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n",
751 	    dip, msiq_id);
752 
753 	if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip),
754 	    msiq_id, msiq_state)) != H_EOK) {
755 		DBG(DBG_LIB_MSIQ, dip,
756 		    "hvio_msiq_getstate failed, ret 0x%lx\n", ret);
757 		return (DDI_FAILURE);
758 	}
759 
760 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n",
761 	    *msiq_state);
762 
763 	return (DDI_SUCCESS);
764 }
765 
766 /*ARGSUSED*/
767 int
768 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id,
769     pci_msiq_state_t msiq_state)
770 {
771 	uint64_t	ret;
772 
773 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x "
774 	    "msiq_state 0x%x\n", dip, msiq_id, msiq_state);
775 
776 	if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip),
777 	    msiq_id, msiq_state)) != H_EOK) {
778 		DBG(DBG_LIB_MSIQ, dip,
779 		    "hvio_msiq_setstate failed, ret 0x%lx\n", ret);
780 		return (DDI_FAILURE);
781 	}
782 
783 	return (DDI_SUCCESS);
784 }
785 
786 /*ARGSUSED*/
787 int
788 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id,
789     msiqhead_t *msiq_head_p)
790 {
791 	uint64_t	ret;
792 
793 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n",
794 	    dip, msiq_id);
795 
796 	if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip),
797 	    msiq_id, msiq_head_p)) != H_EOK) {
798 		DBG(DBG_LIB_MSIQ, dip,
799 		    "hvio_msiq_gethead failed, ret 0x%lx\n", ret);
800 		return (DDI_FAILURE);
801 	}
802 
803 	*msiq_head_p =  (*msiq_head_p / sizeof (msiq_rec_t));
804 
805 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_gethead: msiq_head 0x%x\n",
806 	    *msiq_head_p);
807 
808 	return (DDI_SUCCESS);
809 }
810 
811 /*ARGSUSED*/
812 int
813 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id,
814     msiqhead_t msiq_head)
815 {
816 	uint64_t	ret;
817 
818 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x "
819 	    "msiq_head 0x%x\n", dip, msiq_id, msiq_head);
820 
821 	if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip),
822 	    msiq_id, msiq_head * sizeof (msiq_rec_t))) != H_EOK) {
823 		DBG(DBG_LIB_MSIQ, dip,
824 		    "hvio_msiq_sethead failed, ret 0x%lx\n", ret);
825 		return (DDI_FAILURE);
826 	}
827 
828 	return (DDI_SUCCESS);
829 }
830 
831 /*ARGSUSED*/
832 int
833 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id,
834     msiqtail_t *msiq_tail_p)
835 {
836 	uint64_t	ret;
837 
838 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n",
839 	    dip, msiq_id);
840 
841 	if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip),
842 	    msiq_id, msiq_tail_p)) != H_EOK) {
843 		DBG(DBG_LIB_MSIQ, dip,
844 		    "hvio_msiq_gettail failed, ret 0x%lx\n", ret);
845 		return (DDI_FAILURE);
846 	}
847 
848 	*msiq_tail_p =  (*msiq_tail_p / sizeof (msiq_rec_t));
849 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n",
850 	    *msiq_tail_p);
851 
852 	return (DDI_SUCCESS);
853 }
854 
855 /*ARGSUSED*/
856 void
857 px_lib_get_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p,
858     msiq_rec_t *msiq_rec_p)
859 {
860 	msiq_rec_t	*curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p;
861 
862 	DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p\n", dip);
863 
864 	if (!curr_msiq_rec_p->msiq_rec_type) {
865 		/* Set msiq_rec_type to zero */
866 		msiq_rec_p->msiq_rec_type = 0;
867 
868 		return;
869 	}
870 
871 	*msiq_rec_p = *curr_msiq_rec_p;
872 }
873 
874 /*ARGSUSED*/
875 void
876 px_lib_clr_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p)
877 {
878 	msiq_rec_t	*curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p;
879 
880 	DBG(DBG_LIB_MSIQ, dip, "px_lib_clr_msiq_rec: dip 0x%p\n", dip);
881 
882 	/* Zero out msiq_rec_type field */
883 	curr_msiq_rec_p->msiq_rec_type  = 0;
884 }
885 
886 /*
887  * MSI Functions:
888  */
889 
890 /*ARGSUSED*/
891 int
892 px_lib_msi_init(dev_info_t *dip)
893 {
894 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip);
895 
896 	/* Noop */
897 	return (DDI_SUCCESS);
898 }
899 
900 /*ARGSUSED*/
901 int
902 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num,
903     msiqid_t *msiq_id)
904 {
905 	uint64_t	ret;
906 
907 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n",
908 	    dip, msi_num);
909 
910 	if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip),
911 	    msi_num, msiq_id)) != H_EOK) {
912 		DBG(DBG_LIB_MSI, dip,
913 		    "hvio_msi_getmsiq failed, ret 0x%lx\n", ret);
914 		return (DDI_FAILURE);
915 	}
916 
917 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n",
918 	    *msiq_id);
919 
920 	return (DDI_SUCCESS);
921 }
922 
923 /*ARGSUSED*/
924 int
925 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num,
926     msiqid_t msiq_id, msi_type_t msitype)
927 {
928 	uint64_t	ret;
929 
930 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x "
931 	    "msq_id 0x%x\n", dip, msi_num, msiq_id);
932 
933 	if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip),
934 	    msi_num, msiq_id, msitype)) != H_EOK) {
935 		DBG(DBG_LIB_MSI, dip,
936 		    "hvio_msi_setmsiq failed, ret 0x%lx\n", ret);
937 		return (DDI_FAILURE);
938 	}
939 
940 	return (DDI_SUCCESS);
941 }
942 
943 /*ARGSUSED*/
944 int
945 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num,
946     pci_msi_valid_state_t *msi_valid_state)
947 {
948 	uint64_t	ret;
949 
950 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n",
951 	    dip, msi_num);
952 
953 	if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip),
954 	    msi_num, msi_valid_state)) != H_EOK) {
955 		DBG(DBG_LIB_MSI, dip,
956 		    "hvio_msi_getvalid failed, ret 0x%lx\n", ret);
957 		return (DDI_FAILURE);
958 	}
959 
960 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n",
961 	    *msi_valid_state);
962 
963 	return (DDI_SUCCESS);
964 }
965 
966 /*ARGSUSED*/
967 int
968 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num,
969     pci_msi_valid_state_t msi_valid_state)
970 {
971 	uint64_t	ret;
972 
973 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x "
974 	    "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state);
975 
976 	if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip),
977 	    msi_num, msi_valid_state)) != H_EOK) {
978 		DBG(DBG_LIB_MSI, dip,
979 		    "hvio_msi_setvalid failed, ret 0x%lx\n", ret);
980 		return (DDI_FAILURE);
981 	}
982 
983 	return (DDI_SUCCESS);
984 }
985 
986 /*ARGSUSED*/
987 int
988 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num,
989     pci_msi_state_t *msi_state)
990 {
991 	uint64_t	ret;
992 
993 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n",
994 	    dip, msi_num);
995 
996 	if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip),
997 	    msi_num, msi_state)) != H_EOK) {
998 		DBG(DBG_LIB_MSI, dip,
999 		    "hvio_msi_getstate failed, ret 0x%lx\n", ret);
1000 		return (DDI_FAILURE);
1001 	}
1002 
1003 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n",
1004 	    *msi_state);
1005 
1006 	return (DDI_SUCCESS);
1007 }
1008 
1009 /*ARGSUSED*/
1010 int
1011 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num,
1012     pci_msi_state_t msi_state)
1013 {
1014 	uint64_t	ret;
1015 
1016 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x "
1017 	    "msi_state 0x%x\n", dip, msi_num, msi_state);
1018 
1019 	if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip),
1020 	    msi_num, msi_state)) != H_EOK) {
1021 		DBG(DBG_LIB_MSI, dip,
1022 		    "hvio_msi_setstate failed, ret 0x%lx\n", ret);
1023 		return (DDI_FAILURE);
1024 	}
1025 
1026 	return (DDI_SUCCESS);
1027 }
1028 
1029 /*
1030  * MSG Functions:
1031  */
1032 
1033 /*ARGSUSED*/
1034 int
1035 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1036     msiqid_t *msiq_id)
1037 {
1038 	uint64_t	ret;
1039 
1040 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n",
1041 	    dip, msg_type);
1042 
1043 	if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip),
1044 	    msg_type, msiq_id)) != H_EOK) {
1045 		DBG(DBG_LIB_MSG, dip,
1046 		    "hvio_msg_getmsiq failed, ret 0x%lx\n", ret);
1047 		return (DDI_FAILURE);
1048 	}
1049 
1050 	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n",
1051 	    *msiq_id);
1052 
1053 	return (DDI_SUCCESS);
1054 }
1055 
1056 /*ARGSUSED*/
1057 int
1058 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1059     msiqid_t msiq_id)
1060 {
1061 	uint64_t	ret;
1062 
1063 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_setmsiq: dip 0x%p msg_type 0x%x "
1064 	    "msq_id 0x%x\n", dip, msg_type, msiq_id);
1065 
1066 	if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip),
1067 	    msg_type, msiq_id)) != H_EOK) {
1068 		DBG(DBG_LIB_MSG, dip,
1069 		    "hvio_msg_setmsiq failed, ret 0x%lx\n", ret);
1070 		return (DDI_FAILURE);
1071 	}
1072 
1073 	return (DDI_SUCCESS);
1074 }
1075 
1076 /*ARGSUSED*/
1077 int
1078 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1079     pcie_msg_valid_state_t *msg_valid_state)
1080 {
1081 	uint64_t	ret;
1082 
1083 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n",
1084 	    dip, msg_type);
1085 
1086 	if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type,
1087 	    msg_valid_state)) != H_EOK) {
1088 		DBG(DBG_LIB_MSG, dip,
1089 		    "hvio_msg_getvalid failed, ret 0x%lx\n", ret);
1090 		return (DDI_FAILURE);
1091 	}
1092 
1093 	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n",
1094 	    *msg_valid_state);
1095 
1096 	return (DDI_SUCCESS);
1097 }
1098 
1099 /*ARGSUSED*/
1100 int
1101 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1102     pcie_msg_valid_state_t msg_valid_state)
1103 {
1104 	uint64_t	ret;
1105 
1106 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x "
1107 	    "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state);
1108 
1109 	if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type,
1110 	    msg_valid_state)) != H_EOK) {
1111 		DBG(DBG_LIB_MSG, dip,
1112 		    "hvio_msg_setvalid failed, ret 0x%lx\n", ret);
1113 		return (DDI_FAILURE);
1114 	}
1115 
1116 	return (DDI_SUCCESS);
1117 }
1118 
1119 /*
1120  * Suspend/Resume Functions:
1121  * Currently unsupported by hypervisor and all functions are noops.
1122  */
1123 /*ARGSUSED*/
1124 int
1125 px_lib_suspend(dev_info_t *dip)
1126 {
1127 	DBG(DBG_ATTACH, dip, "px_lib_suspend: Not supported\n");
1128 
1129 	/* Not supported */
1130 	return (DDI_FAILURE);
1131 }
1132 
1133 /*ARGSUSED*/
1134 void
1135 px_lib_resume(dev_info_t *dip)
1136 {
1137 	DBG(DBG_ATTACH, dip, "px_lib_resume: Not supported\n");
1138 
1139 	/* Noop */
1140 }
1141 
1142 /*
1143  * Misc Functions:
1144  * Currently unsupported by hypervisor and all functions are noops.
1145  */
1146 /*ARGSUSED*/
1147 static int
1148 px_lib_config_get(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off,
1149     uint8_t size, pci_cfg_data_t *data_p)
1150 {
1151 	uint64_t	ret;
1152 
1153 	DBG(DBG_LIB_CFG, dip, "px_lib_config_get: dip 0x%p, bdf 0x%llx "
1154 	    "off 0x%x size 0x%x\n", dip, bdf, off, size);
1155 
1156 	if ((ret = hvio_config_get(DIP_TO_HANDLE(dip), bdf, off,
1157 	    size, data_p)) != H_EOK) {
1158 		DBG(DBG_LIB_CFG, dip,
1159 		    "hvio_config_get failed, ret 0x%lx\n", ret);
1160 		return (DDI_FAILURE);
1161 	}
1162 	DBG(DBG_LIB_CFG, dip, "px_config_get: data 0x%x\n", data_p->dw);
1163 
1164 	return (DDI_SUCCESS);
1165 }
1166 
1167 /*ARGSUSED*/
1168 static int
1169 px_lib_config_put(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off,
1170     uint8_t size, pci_cfg_data_t data)
1171 {
1172 	uint64_t	ret;
1173 
1174 	DBG(DBG_LIB_CFG, dip, "px_lib_config_put: dip 0x%p, bdf 0x%llx "
1175 	    "off 0x%x size 0x%x data 0x%llx\n", dip, bdf, off, size, data.qw);
1176 
1177 	if ((ret = hvio_config_put(DIP_TO_HANDLE(dip), bdf, off,
1178 	    size, data)) != H_EOK) {
1179 		DBG(DBG_LIB_CFG, dip,
1180 		    "hvio_config_put failed, ret 0x%lx\n", ret);
1181 		return (DDI_FAILURE);
1182 	}
1183 
1184 	return (DDI_SUCCESS);
1185 }
1186 
1187 static uint32_t
1188 px_pci_config_get(ddi_acc_impl_t *handle, uint32_t *addr, int size)
1189 {
1190 	px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *)
1191 	    handle->ahi_common.ah_bus_private;
1192 	uint32_t pci_dev_addr = px_pvt->raddr;
1193 	uint32_t vaddr = px_pvt->vaddr;
1194 	uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff;
1195 	uint32_t rdata = 0;
1196 
1197 	if (px_lib_config_get(px_pvt->dip, pci_dev_addr, off,
1198 	    size, (pci_cfg_data_t *)&rdata) != DDI_SUCCESS)
1199 		/* XXX update error kstats */
1200 		return (0xffffffff);
1201 	return (rdata);
1202 }
1203 
1204 static void
1205 px_pci_config_put(ddi_acc_impl_t *handle, uint32_t *addr,
1206 		int size, pci_cfg_data_t wdata)
1207 {
1208 	px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *)
1209 	    handle->ahi_common.ah_bus_private;
1210 	uint32_t pci_dev_addr = px_pvt->raddr;
1211 	uint32_t vaddr = px_pvt->vaddr;
1212 	uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff;
1213 
1214 	if (px_lib_config_put(px_pvt->dip, pci_dev_addr, off,
1215 	    size, wdata) != DDI_SUCCESS) {
1216 		/*EMPTY*/
1217 		/* XXX update error kstats */
1218 	}
1219 }
1220 
1221 static uint8_t
1222 px_pci_config_get8(ddi_acc_impl_t *handle, uint8_t *addr)
1223 {
1224 	return ((uint8_t)px_pci_config_get(handle, (uint32_t *)addr, 1));
1225 }
1226 
1227 static uint16_t
1228 px_pci_config_get16(ddi_acc_impl_t *handle, uint16_t *addr)
1229 {
1230 	return ((uint16_t)px_pci_config_get(handle, (uint32_t *)addr, 2));
1231 }
1232 
1233 static uint32_t
1234 px_pci_config_get32(ddi_acc_impl_t *handle, uint32_t *addr)
1235 {
1236 	return ((uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4));
1237 }
1238 
1239 static uint64_t
1240 px_pci_config_get64(ddi_acc_impl_t *handle, uint64_t *addr)
1241 {
1242 	uint32_t rdatah, rdatal;
1243 
1244 	rdatal = (uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4);
1245 	rdatah = (uint32_t)px_pci_config_get(handle,
1246 	    (uint32_t *)((char *)addr+4), 4);
1247 	return (((uint64_t)rdatah << 32) | rdatal);
1248 }
1249 
1250 static void
1251 px_pci_config_put8(ddi_acc_impl_t *handle, uint8_t *addr, uint8_t data)
1252 {
1253 	pci_cfg_data_t wdata = { 0 };
1254 
1255 	wdata.qw = (uint8_t)data;
1256 	px_pci_config_put(handle, (uint32_t *)addr, 1, wdata);
1257 }
1258 
1259 static void
1260 px_pci_config_put16(ddi_acc_impl_t *handle, uint16_t *addr, uint16_t data)
1261 {
1262 	pci_cfg_data_t wdata = { 0 };
1263 
1264 	wdata.qw = (uint16_t)data;
1265 	px_pci_config_put(handle, (uint32_t *)addr, 2, wdata);
1266 }
1267 
1268 static void
1269 px_pci_config_put32(ddi_acc_impl_t *handle, uint32_t *addr, uint32_t data)
1270 {
1271 	pci_cfg_data_t wdata = { 0 };
1272 
1273 	wdata.qw = (uint32_t)data;
1274 	px_pci_config_put(handle, (uint32_t *)addr, 4, wdata);
1275 }
1276 
1277 static void
1278 px_pci_config_put64(ddi_acc_impl_t *handle, uint64_t *addr, uint64_t data)
1279 {
1280 	pci_cfg_data_t wdata = { 0 };
1281 
1282 	wdata.qw = (uint32_t)(data & 0xffffffff);
1283 	px_pci_config_put(handle, (uint32_t *)addr, 4, wdata);
1284 	wdata.qw = (uint32_t)((data >> 32) & 0xffffffff);
1285 	px_pci_config_put(handle, (uint32_t *)((char *)addr+4), 4, wdata);
1286 }
1287 
1288 static void
1289 px_pci_config_rep_get8(ddi_acc_impl_t *handle, uint8_t *host_addr,
1290 			uint8_t *dev_addr, size_t repcount, uint_t flags)
1291 {
1292 	if (flags == DDI_DEV_AUTOINCR)
1293 		for (; repcount; repcount--)
1294 			*host_addr++ = px_pci_config_get8(handle, dev_addr++);
1295 	else
1296 		for (; repcount; repcount--)
1297 			*host_addr++ = px_pci_config_get8(handle, dev_addr);
1298 }
1299 
1300 /*
1301  * Function to rep read 16 bit data off the PCI configuration space behind
1302  * the 21554's host interface.
1303  */
1304 static void
1305 px_pci_config_rep_get16(ddi_acc_impl_t *handle, uint16_t *host_addr,
1306 			uint16_t *dev_addr, size_t repcount, uint_t flags)
1307 {
1308 	if (flags == DDI_DEV_AUTOINCR)
1309 		for (; repcount; repcount--)
1310 			*host_addr++ = px_pci_config_get16(handle, dev_addr++);
1311 	else
1312 		for (; repcount; repcount--)
1313 			*host_addr++ = px_pci_config_get16(handle, dev_addr);
1314 }
1315 
1316 /*
1317  * Function to rep read 32 bit data off the PCI configuration space behind
1318  * the 21554's host interface.
1319  */
1320 static void
1321 px_pci_config_rep_get32(ddi_acc_impl_t *handle, uint32_t *host_addr,
1322 			uint32_t *dev_addr, size_t repcount, uint_t flags)
1323 {
1324 	if (flags == DDI_DEV_AUTOINCR)
1325 		for (; repcount; repcount--)
1326 			*host_addr++ = px_pci_config_get32(handle, dev_addr++);
1327 	else
1328 		for (; repcount; repcount--)
1329 			*host_addr++ = px_pci_config_get32(handle, dev_addr);
1330 }
1331 
1332 /*
1333  * Function to rep read 64 bit data off the PCI configuration space behind
1334  * the 21554's host interface.
1335  */
1336 static void
1337 px_pci_config_rep_get64(ddi_acc_impl_t *handle, uint64_t *host_addr,
1338 			uint64_t *dev_addr, size_t repcount, uint_t flags)
1339 {
1340 	if (flags == DDI_DEV_AUTOINCR)
1341 		for (; repcount; repcount--)
1342 			*host_addr++ = px_pci_config_get64(handle, dev_addr++);
1343 	else
1344 		for (; repcount; repcount--)
1345 			*host_addr++ = px_pci_config_get64(handle, dev_addr);
1346 }
1347 
1348 /*
1349  * Function to rep write 8 bit data into the PCI configuration space behind
1350  * the 21554's host interface.
1351  */
1352 static void
1353 px_pci_config_rep_put8(ddi_acc_impl_t *handle, uint8_t *host_addr,
1354 			uint8_t *dev_addr, size_t repcount, uint_t flags)
1355 {
1356 	if (flags == DDI_DEV_AUTOINCR)
1357 		for (; repcount; repcount--)
1358 			px_pci_config_put8(handle, dev_addr++, *host_addr++);
1359 	else
1360 		for (; repcount; repcount--)
1361 			px_pci_config_put8(handle, dev_addr, *host_addr++);
1362 }
1363 
1364 /*
1365  * Function to rep write 16 bit data into the PCI configuration space behind
1366  * the 21554's host interface.
1367  */
1368 static void
1369 px_pci_config_rep_put16(ddi_acc_impl_t *handle, uint16_t *host_addr,
1370 			uint16_t *dev_addr, size_t repcount, uint_t flags)
1371 {
1372 	if (flags == DDI_DEV_AUTOINCR)
1373 		for (; repcount; repcount--)
1374 			px_pci_config_put16(handle, dev_addr++, *host_addr++);
1375 	else
1376 		for (; repcount; repcount--)
1377 			px_pci_config_put16(handle, dev_addr, *host_addr++);
1378 }
1379 
1380 /*
1381  * Function to rep write 32 bit data into the PCI configuration space behind
1382  * the 21554's host interface.
1383  */
1384 static void
1385 px_pci_config_rep_put32(ddi_acc_impl_t *handle, uint32_t *host_addr,
1386 			uint32_t *dev_addr, size_t repcount, uint_t flags)
1387 {
1388 	if (flags == DDI_DEV_AUTOINCR)
1389 		for (; repcount; repcount--)
1390 			px_pci_config_put32(handle, dev_addr++, *host_addr++);
1391 	else
1392 		for (; repcount; repcount--)
1393 			px_pci_config_put32(handle, dev_addr, *host_addr++);
1394 }
1395 
1396 /*
1397  * Function to rep write 64 bit data into the PCI configuration space behind
1398  * the 21554's host interface.
1399  */
1400 static void
1401 px_pci_config_rep_put64(ddi_acc_impl_t *handle, uint64_t *host_addr,
1402 			uint64_t *dev_addr, size_t repcount, uint_t flags)
1403 {
1404 	if (flags == DDI_DEV_AUTOINCR)
1405 		for (; repcount; repcount--)
1406 			px_pci_config_put64(handle, dev_addr++, *host_addr++);
1407 	else
1408 		for (; repcount; repcount--)
1409 			px_pci_config_put64(handle, dev_addr, *host_addr++);
1410 }
1411 
1412 /*
1413  * Provide a private access handle to route config access calls to Hypervisor.
1414  * Beware: Do all error checking for config space accesses before calling
1415  * this function. ie. do error checking from the calling function.
1416  * Due to a lack of meaningful error code in DDI, the gauranteed return of
1417  * DDI_SUCCESS from here makes the code organization readable/easier from
1418  * the generic code.
1419  */
1420 /*ARGSUSED*/
1421 int
1422 px_lib_map_vconfig(dev_info_t *dip,
1423 	ddi_map_req_t *mp, pci_config_offset_t off,
1424 	pci_regspec_t *rp, caddr_t *addrp)
1425 {
1426 	int fmcap;
1427 	ndi_err_t *errp;
1428 	on_trap_data_t *otp;
1429 	ddi_acc_hdl_t *hp;
1430 	ddi_acc_impl_t *ap;
1431 	uchar_t busnum;	/* bus number */
1432 	uchar_t devnum;	/* device number */
1433 	uchar_t funcnum; /* function number */
1434 	px_config_acc_pvt_t *px_pvt;
1435 
1436 	hp = (ddi_acc_hdl_t *)mp->map_handlep;
1437 	ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1438 
1439 	/* Check for mapping teardown operation */
1440 	if ((mp->map_op == DDI_MO_UNMAP) ||
1441 	    (mp->map_op == DDI_MO_UNLOCK)) {
1442 		/* free up memory allocated for the private access handle. */
1443 		px_pvt = (px_config_acc_pvt_t *)hp->ah_bus_private;
1444 		kmem_free((void *)px_pvt, sizeof (px_config_acc_pvt_t));
1445 
1446 		/* unmap operation of PCI IO/config space. */
1447 		return (DDI_SUCCESS);
1448 	}
1449 
1450 	fmcap = ddi_fm_capable(dip);
1451 	if (DDI_FM_ACC_ERR_CAP(fmcap)) {
1452 		errp = ((ddi_acc_impl_t *)hp)->ahi_err;
1453 		otp = (on_trap_data_t *)errp->err_ontrap;
1454 		otp->ot_handle = (void *)(hp);
1455 		otp->ot_prot = OT_DATA_ACCESS;
1456 		errp->err_status = DDI_FM_OK;
1457 		errp->err_expected = DDI_FM_ERR_UNEXPECTED;
1458 		errp->err_cf = px_err_cfg_hdl_check;
1459 	}
1460 
1461 	ap->ahi_get8 = px_pci_config_get8;
1462 	ap->ahi_get16 = px_pci_config_get16;
1463 	ap->ahi_get32 = px_pci_config_get32;
1464 	ap->ahi_get64 = px_pci_config_get64;
1465 	ap->ahi_put8 = px_pci_config_put8;
1466 	ap->ahi_put16 = px_pci_config_put16;
1467 	ap->ahi_put32 = px_pci_config_put32;
1468 	ap->ahi_put64 = px_pci_config_put64;
1469 	ap->ahi_rep_get8 = px_pci_config_rep_get8;
1470 	ap->ahi_rep_get16 = px_pci_config_rep_get16;
1471 	ap->ahi_rep_get32 = px_pci_config_rep_get32;
1472 	ap->ahi_rep_get64 = px_pci_config_rep_get64;
1473 	ap->ahi_rep_put8 = px_pci_config_rep_put8;
1474 	ap->ahi_rep_put16 = px_pci_config_rep_put16;
1475 	ap->ahi_rep_put32 = px_pci_config_rep_put32;
1476 	ap->ahi_rep_put64 = px_pci_config_rep_put64;
1477 
1478 	/* Initialize to default check/notify functions */
1479 	ap->ahi_fault = 0;
1480 	ap->ahi_fault_check = i_ddi_acc_fault_check;
1481 	ap->ahi_fault_notify = i_ddi_acc_fault_notify;
1482 
1483 	/* allocate memory for our private handle */
1484 	px_pvt = (px_config_acc_pvt_t *)
1485 	    kmem_zalloc(sizeof (px_config_acc_pvt_t), KM_SLEEP);
1486 	hp->ah_bus_private = (void *)px_pvt;
1487 
1488 	busnum = PCI_REG_BUS_G(rp->pci_phys_hi);
1489 	devnum = PCI_REG_DEV_G(rp->pci_phys_hi);
1490 	funcnum = PCI_REG_FUNC_G(rp->pci_phys_hi);
1491 
1492 	/* set up private data for use during IO routines */
1493 
1494 	/* addr needed by the HV APIs */
1495 	px_pvt->raddr = busnum << 16 | devnum << 11 | funcnum << 8;
1496 	/*
1497 	 * Address that specifies the actual offset into the 256MB
1498 	 * memory mapped configuration space, 4K per device.
1499 	 * First 12bits form the offset into 4K config space.
1500 	 * This address is only used during the IO routines to calculate
1501 	 * the offset at which the transaction must be performed.
1502 	 * Drivers bypassing DDI functions to access PCI config space will
1503 	 * panic the system since the following is a bogus virtual address.
1504 	 */
1505 	px_pvt->vaddr = busnum << 20 | devnum << 15 | funcnum << 12 | off;
1506 	px_pvt->dip = dip;
1507 
1508 	DBG(DBG_LIB_CFG, dip, "px_config_setup: raddr 0x%x, vaddr 0x%x\n",
1509 	    px_pvt->raddr, px_pvt->vaddr);
1510 	*addrp = (caddr_t)(uintptr_t)px_pvt->vaddr;
1511 	return (DDI_SUCCESS);
1512 }
1513 
1514 /*ARGSUSED*/
1515 void
1516 px_lib_map_attr_check(ddi_map_req_t *mp)
1517 {
1518 }
1519 
1520 /*
1521  * px_lib_log_safeacc_err:
1522  * Imitate a cpu/mem trap call when a peek/poke fails.
1523  * This will initiate something similar to px_fm_callback.
1524  */
1525 static void
1526 px_lib_log_safeacc_err(px_t *px_p, ddi_acc_handle_t handle, int fme_flag,
1527     r_addr_t addr)
1528 {
1529 	uint32_t	addr_high, addr_low;
1530 	pcie_req_id_t	bdf = PCIE_INVALID_BDF;
1531 	px_ranges_t	*ranges_p;
1532 	int		range_len, i;
1533 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1534 	ddi_fm_error_t derr;
1535 
1536 	derr.fme_status = DDI_FM_NONFATAL;
1537 	derr.fme_version = DDI_FME_VERSION;
1538 	derr.fme_flag = fme_flag;
1539 	derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
1540 	derr.fme_acc_handle = handle;
1541 	if (hp)
1542 		hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED;
1543 
1544 	addr_high = (uint32_t)(addr >> 32);
1545 	addr_low = (uint32_t)addr;
1546 
1547 	/*
1548 	 * Make sure this failed load came from this PCIe port.  Check by
1549 	 * matching the upper 32 bits of the address with the ranges property.
1550 	 */
1551 	range_len = px_p->px_ranges_length / sizeof (px_ranges_t);
1552 	i = 0;
1553 	for (ranges_p = px_p->px_ranges_p; i < range_len; i++, ranges_p++) {
1554 		if (ranges_p->parent_high == addr_high) {
1555 			switch (ranges_p->child_high & PCI_ADDR_MASK) {
1556 			case PCI_ADDR_CONFIG:
1557 				bdf = (pcie_req_id_t)(addr_low >> 12);
1558 				break;
1559 			default:
1560 				bdf = PCIE_INVALID_BDF;
1561 				break;
1562 			}
1563 			break;
1564 		}
1565 	}
1566 
1567 	px_rp_en_q(px_p, bdf, addr, NULL);
1568 
1569 	if (px_fm_enter(px_p) == DDI_SUCCESS) {
1570 		(void) px_scan_fabric(px_p, px_p->px_dip, &derr);
1571 		px_fm_exit(px_p);
1572 	}
1573 }
1574 
1575 
1576 #ifdef  DEBUG
1577 int	px_peekfault_cnt = 0;
1578 int	px_pokefault_cnt = 0;
1579 #endif  /* DEBUG */
1580 
1581 /*
1582  * Do a safe write to a device.
1583  *
1584  * When this function is given a handle (cautious access), all errors are
1585  * suppressed.
1586  *
1587  * When this function is not given a handle (poke), only Unsupported Request
1588  * and Completer Abort errors are suppressed.
1589  *
1590  * In all cases, all errors are returned in the function return status.
1591  */
1592 
1593 int
1594 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip,
1595     peekpoke_ctlops_t *in_args)
1596 {
1597 	px_t *px_p = DIP_TO_STATE(dip);
1598 	px_pec_t *pec_p = px_p->px_pec_p;
1599 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
1600 
1601 	size_t repcount = in_args->repcount;
1602 	size_t size = in_args->size;
1603 	uintptr_t dev_addr = in_args->dev_addr;
1604 	uintptr_t host_addr = in_args->host_addr;
1605 
1606 	int err	= DDI_SUCCESS;
1607 	uint64_t hvio_poke_status;
1608 	uint32_t wrt_stat;
1609 
1610 	r_addr_t ra;
1611 	uint64_t pokeval;
1612 	pcie_req_id_t bdf;
1613 
1614 	ra = (r_addr_t)va_to_pa((void *)dev_addr);
1615 	for (; repcount; repcount--) {
1616 
1617 		switch (size) {
1618 		case sizeof (uint8_t):
1619 			pokeval = *(uint8_t *)host_addr;
1620 			break;
1621 		case sizeof (uint16_t):
1622 			pokeval = *(uint16_t *)host_addr;
1623 			break;
1624 		case sizeof (uint32_t):
1625 			pokeval = *(uint32_t *)host_addr;
1626 			break;
1627 		case sizeof (uint64_t):
1628 			pokeval = *(uint64_t *)host_addr;
1629 			break;
1630 		default:
1631 			DBG(DBG_MAP, px_p->px_dip,
1632 			    "poke: invalid size %d passed\n", size);
1633 			err = DDI_FAILURE;
1634 			goto done;
1635 		}
1636 
1637 		/*
1638 		 * Grab pokefault mutex since hypervisor does not guarantee
1639 		 * poke serialization.
1640 		 */
1641 		if (hp) {
1642 			i_ndi_busop_access_enter(hp->ahi_common.ah_dip,
1643 			    (ddi_acc_handle_t)hp);
1644 			pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1645 		} else {
1646 			mutex_enter(&pec_p->pec_pokefault_mutex);
1647 			pec_p->pec_safeacc_type = DDI_FM_ERR_POKE;
1648 		}
1649 
1650 		if (pcie_get_bdf_from_dip(rdip, &bdf) != DDI_SUCCESS) {
1651 			err = DDI_FAILURE;
1652 			goto done;
1653 		}
1654 
1655 		hvio_poke_status = hvio_poke(px_p->px_dev_hdl, ra, size,
1656 		    pokeval, bdf << 8, &wrt_stat);
1657 
1658 		if ((hvio_poke_status != H_EOK) || (wrt_stat != H_EOK)) {
1659 			err = DDI_FAILURE;
1660 #ifdef  DEBUG
1661 			px_pokefault_cnt++;
1662 #endif
1663 			/*
1664 			 * For CAUTIOUS and POKE access, notify FMA to
1665 			 * cleanup.  Imitate a cpu/mem trap call like in sun4u.
1666 			 */
1667 			px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp,
1668 			    (hp ? DDI_FM_ERR_EXPECTED :
1669 			    DDI_FM_ERR_POKE), ra);
1670 
1671 			pec_p->pec_ontrap_data = NULL;
1672 			pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1673 			if (hp) {
1674 				i_ndi_busop_access_exit(hp->ahi_common.ah_dip,
1675 				    (ddi_acc_handle_t)hp);
1676 			} else {
1677 				mutex_exit(&pec_p->pec_pokefault_mutex);
1678 			}
1679 			goto done;
1680 		}
1681 
1682 		pec_p->pec_ontrap_data = NULL;
1683 		pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1684 		if (hp) {
1685 			i_ndi_busop_access_exit(hp->ahi_common.ah_dip,
1686 			    (ddi_acc_handle_t)hp);
1687 		} else {
1688 			mutex_exit(&pec_p->pec_pokefault_mutex);
1689 		}
1690 
1691 		host_addr += size;
1692 
1693 		if (in_args->flags == DDI_DEV_AUTOINCR) {
1694 			dev_addr += size;
1695 			ra = (r_addr_t)va_to_pa((void *)dev_addr);
1696 		}
1697 	}
1698 
1699 done:
1700 	return (err);
1701 }
1702 
1703 
1704 /*ARGSUSED*/
1705 int
1706 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip,
1707     peekpoke_ctlops_t *in_args, void *result)
1708 {
1709 	px_t *px_p = DIP_TO_STATE(dip);
1710 	px_pec_t *pec_p = px_p->px_pec_p;
1711 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
1712 
1713 	size_t repcount = in_args->repcount;
1714 	uintptr_t dev_addr = in_args->dev_addr;
1715 	uintptr_t host_addr = in_args->host_addr;
1716 
1717 	r_addr_t ra;
1718 	uint32_t read_status;
1719 	uint64_t hvio_peek_status;
1720 	uint64_t peekval;
1721 	int err = DDI_SUCCESS;
1722 
1723 	result = (void *)in_args->host_addr;
1724 
1725 	ra = (r_addr_t)va_to_pa((void *)dev_addr);
1726 	for (; repcount; repcount--) {
1727 
1728 		/* Lock pokefault mutex so read doesn't mask a poke fault. */
1729 		if (hp) {
1730 			i_ndi_busop_access_enter(hp->ahi_common.ah_dip,
1731 			    (ddi_acc_handle_t)hp);
1732 			pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1733 		} else {
1734 			mutex_enter(&pec_p->pec_pokefault_mutex);
1735 			pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK;
1736 		}
1737 
1738 		hvio_peek_status = hvio_peek(px_p->px_dev_hdl, ra,
1739 		    in_args->size, &read_status, &peekval);
1740 
1741 		if ((hvio_peek_status != H_EOK) || (read_status != H_EOK)) {
1742 			err = DDI_FAILURE;
1743 
1744 			/*
1745 			 * For CAUTIOUS and PEEK access, notify FMA to
1746 			 * cleanup.  Imitate a cpu/mem trap call like in sun4u.
1747 			 */
1748 			px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp,
1749 			    (hp ? DDI_FM_ERR_EXPECTED :
1750 			    DDI_FM_ERR_PEEK), ra);
1751 
1752 			/* Stuff FFs in host addr if peek. */
1753 			if (hp == NULL) {
1754 				int i;
1755 				uint8_t *ff_addr = (uint8_t *)host_addr;
1756 				for (i = 0; i < in_args->size; i++)
1757 					*ff_addr++ = 0xff;
1758 			}
1759 #ifdef  DEBUG
1760 			px_peekfault_cnt++;
1761 #endif
1762 			pec_p->pec_ontrap_data = NULL;
1763 			pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1764 			if (hp) {
1765 				i_ndi_busop_access_exit(hp->ahi_common.ah_dip,
1766 				    (ddi_acc_handle_t)hp);
1767 			} else {
1768 				mutex_exit(&pec_p->pec_pokefault_mutex);
1769 			}
1770 			goto done;
1771 
1772 		}
1773 		pec_p->pec_ontrap_data = NULL;
1774 		pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1775 		if (hp) {
1776 			i_ndi_busop_access_exit(hp->ahi_common.ah_dip,
1777 			    (ddi_acc_handle_t)hp);
1778 		} else {
1779 			mutex_exit(&pec_p->pec_pokefault_mutex);
1780 		}
1781 
1782 		switch (in_args->size) {
1783 		case sizeof (uint8_t):
1784 			*(uint8_t *)host_addr = (uint8_t)peekval;
1785 			break;
1786 		case sizeof (uint16_t):
1787 			*(uint16_t *)host_addr = (uint16_t)peekval;
1788 			break;
1789 		case sizeof (uint32_t):
1790 			*(uint32_t *)host_addr = (uint32_t)peekval;
1791 			break;
1792 		case sizeof (uint64_t):
1793 			*(uint64_t *)host_addr = (uint64_t)peekval;
1794 			break;
1795 		default:
1796 			DBG(DBG_MAP, px_p->px_dip,
1797 			    "peek: invalid size %d passed\n",
1798 			    in_args->size);
1799 			err = DDI_FAILURE;
1800 			goto done;
1801 		}
1802 
1803 		host_addr += in_args->size;
1804 
1805 		if (in_args->flags == DDI_DEV_AUTOINCR) {
1806 			dev_addr += in_args->size;
1807 			ra = (r_addr_t)va_to_pa((void *)dev_addr);
1808 		}
1809 	}
1810 done:
1811 	return (err);
1812 }
1813 
1814 
1815 /* add interrupt vector */
1816 int
1817 px_err_add_intr(px_fault_t *px_fault_p)
1818 {
1819 	px_t	*px_p = DIP_TO_STATE(px_fault_p->px_fh_dip);
1820 
1821 	DBG(DBG_LIB_INT, px_p->px_dip,
1822 	    "px_err_add_intr: calling add_ivintr");
1823 
1824 	VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL,
1825 	    (intrfunc)px_fault_p->px_err_func, (caddr_t)px_fault_p, NULL,
1826 	    (caddr_t)&px_fault_p->px_intr_payload[0]) == 0);
1827 
1828 	DBG(DBG_LIB_INT, px_p->px_dip,
1829 	    "px_err_add_intr: ib_intr_enable ");
1830 
1831 	px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino);
1832 
1833 	return (DDI_SUCCESS);
1834 }
1835 
1836 /* remove interrupt vector */
1837 void
1838 px_err_rem_intr(px_fault_t *px_fault_p)
1839 {
1840 	px_t	*px_p = DIP_TO_STATE(px_fault_p->px_fh_dip);
1841 
1842 	px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino,
1843 	    IB_INTR_WAIT);
1844 
1845 	VERIFY(rem_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL) == 0);
1846 }
1847 
1848 void
1849 px_cb_intr_redist(void *arg)
1850 {
1851 	px_t	*px_p = (px_t *)arg;
1852 	px_ib_intr_dist_en(px_p->px_dip, intr_dist_cpuid(),
1853 	    px_p->px_inos[PX_INTR_XBC], B_FALSE);
1854 }
1855 
1856 int
1857 px_cb_add_intr(px_fault_t *f_p)
1858 {
1859 	px_t	*px_p = DIP_TO_STATE(f_p->px_fh_dip);
1860 
1861 	DBG(DBG_LIB_INT, px_p->px_dip,
1862 	    "px_err_add_intr: calling add_ivintr");
1863 
1864 	VERIFY(add_ivintr(f_p->px_fh_sysino, PX_ERR_PIL,
1865 	    (intrfunc)f_p->px_err_func, (caddr_t)f_p, NULL,
1866 	    (caddr_t)&f_p->px_intr_payload[0]) == 0);
1867 
1868 	intr_dist_add(px_cb_intr_redist, px_p);
1869 
1870 	DBG(DBG_LIB_INT, px_p->px_dip,
1871 	    "px_err_add_intr: ib_intr_enable ");
1872 
1873 	px_ib_intr_enable(px_p, intr_dist_cpuid(), f_p->px_intr_ino);
1874 
1875 	return (DDI_SUCCESS);
1876 }
1877 
1878 void
1879 px_cb_rem_intr(px_fault_t *f_p)
1880 {
1881 	intr_dist_rem(px_cb_intr_redist, DIP_TO_STATE(f_p->px_fh_dip));
1882 	px_err_rem_intr(f_p);
1883 }
1884 
1885 #ifdef FMA
1886 void
1887 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status)
1888 {
1889 	px_pec_err_t	*err_pkt;
1890 
1891 	err_pkt = (px_pec_err_t *)px_fault_p->px_intr_payload;
1892 
1893 	/* initialise all the structure members */
1894 	rc_status->status_valid = 0;
1895 
1896 	if (err_pkt->pec_descr.P) {
1897 		/* PCI Status Register */
1898 		rc_status->pci_err_status = err_pkt->pci_err_status;
1899 		rc_status->status_valid |= PCI_ERR_STATUS_VALID;
1900 	}
1901 
1902 	if (err_pkt->pec_descr.E) {
1903 		/* PCIe Status Register */
1904 		rc_status->pcie_err_status = err_pkt->pcie_err_status;
1905 		rc_status->status_valid |= PCIE_ERR_STATUS_VALID;
1906 	}
1907 
1908 	if (err_pkt->pec_descr.U) {
1909 		rc_status->ue_status = err_pkt->ue_reg_status;
1910 		rc_status->status_valid |= UE_STATUS_VALID;
1911 	}
1912 
1913 	if (err_pkt->pec_descr.H) {
1914 		rc_status->ue_hdr1 = err_pkt->hdr[0];
1915 		rc_status->status_valid |= UE_HDR1_VALID;
1916 	}
1917 
1918 	if (err_pkt->pec_descr.I) {
1919 		rc_status->ue_hdr2 = err_pkt->hdr[1];
1920 		rc_status->status_valid |= UE_HDR2_VALID;
1921 	}
1922 
1923 	/* ue_fst_err_ptr - not available for sun4v?? */
1924 
1925 
1926 	if (err_pkt->pec_descr.S) {
1927 		rc_status->source_id = err_pkt->err_src_reg;
1928 		rc_status->status_valid |= SOURCE_ID_VALID;
1929 	}
1930 
1931 	if (err_pkt->pec_descr.R) {
1932 		rc_status->root_err_status = err_pkt->root_err_status;
1933 		rc_status->status_valid |= CE_STATUS_VALID;
1934 	}
1935 }
1936 #endif
1937 
1938 /*ARGSUSED*/
1939 int
1940 px_lib_pmctl(int cmd, px_t *px_p)
1941 {
1942 	return (DDI_FAILURE);
1943 }
1944 
1945 /*ARGSUSED*/
1946 uint_t
1947 px_pmeq_intr(caddr_t arg)
1948 {
1949 	return (DDI_INTR_CLAIMED);
1950 }
1951 
1952 /*
1953  * Unprotected raw reads/writes of fabric device's config space.
1954  * Only used for temporary PCI-E Fabric Error Handling.
1955  */
1956 uint32_t
1957 px_fab_get(px_t *px_p, pcie_req_id_t bdf, uint16_t offset) {
1958 	uint32_t 	data = 0;
1959 
1960 	(void) hvio_config_get(px_p->px_dev_hdl,
1961 	    (bdf << PX_RA_BDF_SHIFT), offset, 4,
1962 	    (pci_cfg_data_t *)&data);
1963 
1964 	return (data);
1965 }
1966 
1967 void
1968 px_fab_set(px_t *px_p, pcie_req_id_t bdf, uint16_t offset,
1969     uint32_t val) {
1970 	pci_cfg_data_t	wdata = { 0 };
1971 
1972 	wdata.qw = (uint32_t)val;
1973 	(void) hvio_config_put(px_p->px_dev_hdl,
1974 	    (bdf << PX_RA_BDF_SHIFT), offset, 4, wdata);
1975 }
1976 
1977 /*ARGSUSED*/
1978 int
1979 px_lib_hotplug_init(dev_info_t *dip, void *arg)
1980 {
1981 	return (DDI_ENOTSUP);
1982 }
1983 
1984 /*ARGSUSED*/
1985 void
1986 px_lib_hotplug_uninit(dev_info_t *dip)
1987 {
1988 }
1989 
1990 /*ARGSUSED*/
1991 void
1992 px_hp_intr_redist(px_t *px_p)
1993 {
1994 }
1995 
1996 /* Dummy cpr add callback */
1997 /*ARGSUSED*/
1998 void
1999 px_cpr_add_callb(px_t *px_p)
2000 {
2001 }
2002 
2003 /* Dummy cpr rem callback */
2004 /*ARGSUSED*/
2005 void
2006 px_cpr_rem_callb(px_t *px_p)
2007 {
2008 }
2009 
2010 /*ARGSUSED*/
2011 boolean_t
2012 px_lib_is_in_drain_state(px_t *px_p)
2013 {
2014 	return (B_FALSE);
2015 }
2016 
2017 /*
2018  * There is no IOAPI to get the BDF of the pcie root port nexus at this moment.
2019  * Assume it is 0x0000, until otherwise noted.  For now, all sun4v platforms
2020  * have programmed the BDF to be 0x0000.
2021  */
2022 /*ARGSUSED*/
2023 pcie_req_id_t
2024 px_lib_get_bdf(px_t *px_p)
2025 {
2026 	return (0x0000);
2027 }
2028 
2029 int
2030 px_lib_get_root_complex_mps(px_t *px_p, dev_info_t *dip, int *mps)
2031 {
2032 	pci_device_t	bdf = px_lib_get_bdf(px_p);
2033 
2034 	if (hvio_get_rp_mps_cap(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK)
2035 		return (DDI_SUCCESS);
2036 	else
2037 		return (DDI_FAILURE);
2038 }
2039 
2040 int
2041 px_lib_set_root_complex_mps(px_t *px_p,  dev_info_t *dip, int mps)
2042 {
2043 	pci_device_t	bdf = px_lib_get_bdf(px_p);
2044 
2045 	if (hvio_set_rp_mps(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK)
2046 		return (DDI_SUCCESS);
2047 	else
2048 		return (DDI_FAILURE);
2049 }
2050