1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26 /*
27 * Copyright 2014 Garrett D'Amore <garrett@damore.org>
28 */
29
30 #include <sys/conf.h>
31 #include <sys/kmem.h>
32 #include <sys/ddi_impldefs.h>
33 #include <sys/ddi.h>
34 #include <sys/sunddi.h>
35 #include <sys/ddifm.h>
36 #include <sys/fm/io/ddi.h>
37 #include <sys/fm/protocol.h>
38 #include <sys/ontrap.h>
39
40
41 /*
42 * DDI DMA Engine functions for x86.
43 * These functions are more naturally generic, but do not apply to SPARC.
44 */
45
46 int
ddi_dmae_alloc(dev_info_t * dip,int chnl,int (* dmae_waitfp)(),caddr_t arg)47 ddi_dmae_alloc(dev_info_t *dip, int chnl, int (*dmae_waitfp)(), caddr_t arg)
48 {
49 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_ACQUIRE,
50 (off_t *)dmae_waitfp, (size_t *)arg,
51 (caddr_t *)(uintptr_t)chnl, 0));
52 }
53
54 int
ddi_dmae_release(dev_info_t * dip,int chnl)55 ddi_dmae_release(dev_info_t *dip, int chnl)
56 {
57 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_FREE, 0, 0,
58 (caddr_t *)(uintptr_t)chnl, 0));
59 }
60
61 int
ddi_dmae_getattr(dev_info_t * dip,ddi_dma_attr_t * attrp)62 ddi_dmae_getattr(dev_info_t *dip, ddi_dma_attr_t *attrp)
63 {
64 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_GETATTR, 0, 0,
65 (caddr_t *)attrp, 0));
66 }
67
68 int
ddi_dmae_1stparty(dev_info_t * dip,int chnl)69 ddi_dmae_1stparty(dev_info_t *dip, int chnl)
70 {
71 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_1STPTY, 0, 0,
72 (caddr_t *)(uintptr_t)chnl, 0));
73 }
74
75 int
ddi_dmae_prog(dev_info_t * dip,struct ddi_dmae_req * dmaereqp,ddi_dma_cookie_t * cookiep,int chnl)76 ddi_dmae_prog(dev_info_t *dip, struct ddi_dmae_req *dmaereqp,
77 ddi_dma_cookie_t *cookiep, int chnl)
78 {
79 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_PROG, (off_t *)dmaereqp,
80 (size_t *)cookiep, (caddr_t *)(uintptr_t)chnl, 0));
81 }
82
83 int
ddi_dmae_swsetup(dev_info_t * dip,struct ddi_dmae_req * dmaereqp,ddi_dma_cookie_t * cookiep,int chnl)84 ddi_dmae_swsetup(dev_info_t *dip, struct ddi_dmae_req *dmaereqp,
85 ddi_dma_cookie_t *cookiep, int chnl)
86 {
87 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_SWSETUP, (off_t *)dmaereqp,
88 (size_t *)cookiep, (caddr_t *)(uintptr_t)chnl, 0));
89 }
90
91 int
ddi_dmae_swstart(dev_info_t * dip,int chnl)92 ddi_dmae_swstart(dev_info_t *dip, int chnl)
93 {
94 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_SWSTART, 0, 0,
95 (caddr_t *)(uintptr_t)chnl, 0));
96 }
97
98 int
ddi_dmae_stop(dev_info_t * dip,int chnl)99 ddi_dmae_stop(dev_info_t *dip, int chnl)
100 {
101 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_STOP, 0, 0,
102 (caddr_t *)(uintptr_t)chnl, 0));
103 }
104
105 int
ddi_dmae_enable(dev_info_t * dip,int chnl)106 ddi_dmae_enable(dev_info_t *dip, int chnl)
107 {
108 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_ENABLE, 0, 0,
109 (caddr_t *)(uintptr_t)chnl, 0));
110 }
111
112 int
ddi_dmae_disable(dev_info_t * dip,int chnl)113 ddi_dmae_disable(dev_info_t *dip, int chnl)
114 {
115 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_DISABLE, 0, 0,
116 (caddr_t *)(uintptr_t)chnl, 0));
117 }
118
119 int
ddi_dmae_getcnt(dev_info_t * dip,int chnl,int * countp)120 ddi_dmae_getcnt(dev_info_t *dip, int chnl, int *countp)
121 {
122 return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_GETCNT, 0, (size_t *)countp,
123 (caddr_t *)(uintptr_t)chnl, 0));
124 }
125
126 /*
127 * implementation specific access handle and routines:
128 */
129
130 static uintptr_t impl_acc_hdl_id = 0;
131
132 /*
133 * access handle allocator
134 */
135 ddi_acc_hdl_t *
impl_acc_hdl_get(ddi_acc_handle_t hdl)136 impl_acc_hdl_get(ddi_acc_handle_t hdl)
137 {
138 /*
139 * recast to ddi_acc_hdl_t instead of
140 * casting to ddi_acc_impl_t and then return the ah_platform_private
141 *
142 * this optimization based on the ddi_acc_hdl_t is the
143 * first member of the ddi_acc_impl_t.
144 */
145 return ((ddi_acc_hdl_t *)hdl);
146 }
147
148 ddi_acc_handle_t
impl_acc_hdl_alloc(int (* waitfp)(caddr_t),caddr_t arg)149 impl_acc_hdl_alloc(int (*waitfp)(caddr_t), caddr_t arg)
150 {
151 ddi_acc_impl_t *hp;
152 on_trap_data_t *otp;
153 int sleepflag;
154
155 sleepflag = ((waitfp == (int (*)())KM_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
156 /*
157 * Allocate and initialize the data access handle and error status.
158 */
159 if ((hp = kmem_zalloc(sizeof (ddi_acc_impl_t), sleepflag)) == NULL)
160 goto fail;
161 if ((hp->ahi_err = (ndi_err_t *)kmem_zalloc(
162 sizeof (ndi_err_t), sleepflag)) == NULL) {
163 kmem_free(hp, sizeof (ddi_acc_impl_t));
164 goto fail;
165 }
166 if ((otp = (on_trap_data_t *)kmem_zalloc(
167 sizeof (on_trap_data_t), sleepflag)) == NULL) {
168 kmem_free(hp->ahi_err, sizeof (ndi_err_t));
169 kmem_free(hp, sizeof (ddi_acc_impl_t));
170 goto fail;
171 }
172 hp->ahi_err->err_ontrap = otp;
173 hp->ahi_common.ah_platform_private = (void *)hp;
174
175 return ((ddi_acc_handle_t)hp);
176 fail:
177 if ((waitfp != (int (*)())KM_SLEEP) &&
178 (waitfp != (int (*)())KM_NOSLEEP))
179 ddi_set_callback(waitfp, arg, &impl_acc_hdl_id);
180 return (NULL);
181 }
182
183 void
impl_acc_hdl_free(ddi_acc_handle_t handle)184 impl_acc_hdl_free(ddi_acc_handle_t handle)
185 {
186 ddi_acc_impl_t *hp;
187
188 /*
189 * The supplied (ddi_acc_handle_t) is actually a (ddi_acc_impl_t *),
190 * because that's what we allocated in impl_acc_hdl_alloc() above.
191 */
192 hp = (ddi_acc_impl_t *)handle;
193 if (hp) {
194 kmem_free(hp->ahi_err->err_ontrap, sizeof (on_trap_data_t));
195 kmem_free(hp->ahi_err, sizeof (ndi_err_t));
196 kmem_free(hp, sizeof (ddi_acc_impl_t));
197 if (impl_acc_hdl_id)
198 ddi_run_callback(&impl_acc_hdl_id);
199 }
200 }
201
202 /*
203 * Function used to check if a given access handle owns the failing address.
204 * Called by ndi_fmc_error, when we detect a PIO error.
205 */
206 /* ARGSUSED */
207 static int
impl_acc_check(dev_info_t * dip,const void * handle,const void * addr,const void * not_used)208 impl_acc_check(dev_info_t *dip, const void *handle, const void *addr,
209 const void *not_used)
210 {
211 pfn_t pfn, fault_pfn;
212 ddi_acc_hdl_t *hp;
213
214 hp = impl_acc_hdl_get((ddi_acc_handle_t)handle);
215
216 ASSERT(hp);
217
218 if (addr != NULL) {
219 pfn = hp->ah_pfn;
220 fault_pfn = mmu_btop(*(uint64_t *)addr);
221 if (fault_pfn >= pfn && fault_pfn < (pfn + hp->ah_pnum))
222 return (DDI_FM_NONFATAL);
223 }
224 return (DDI_FM_UNKNOWN);
225 }
226
227 void
impl_acc_err_init(ddi_acc_hdl_t * handlep)228 impl_acc_err_init(ddi_acc_hdl_t *handlep)
229 {
230 int fmcap;
231 ndi_err_t *errp;
232 on_trap_data_t *otp;
233 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handlep;
234
235 fmcap = ddi_fm_capable(handlep->ah_dip);
236
237 if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
238 !DDI_FM_ACC_ERR_CAP(fmcap)) {
239 handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
240 } else if (handlep->ah_acc.devacc_attr_access == DDI_FLAGERR_ACC &&
241 hp->ahi_scan == NULL) {
242 handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
243 } else if (DDI_FM_ACC_ERR_CAP(fmcap)) {
244 if (handlep->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
245 if (handlep->ah_xfermodes)
246 return;
247 i_ddi_drv_ereport_post(handlep->ah_dip, DVR_EFMCAP,
248 NULL, DDI_NOSLEEP);
249 } else {
250 errp = hp->ahi_err;
251 otp = (on_trap_data_t *)errp->err_ontrap;
252 otp->ot_handle = (void *)(hp);
253 otp->ot_prot = OT_DATA_ACCESS;
254 errp->err_status = DDI_FM_OK;
255 errp->err_expected = DDI_FM_ERR_UNEXPECTED;
256 errp->err_cf = impl_acc_check;
257 }
258 }
259 }
260
261 /* ARGSUSED */
262 int
impl_dma_check(dev_info_t * dip,const void * handle,const void * pci_hdl,const void * not_used)263 impl_dma_check(dev_info_t *dip, const void *handle, const void *pci_hdl,
264 const void *not_used)
265 {
266 return (DDI_FM_UNKNOWN);
267 }
268
269 void
impl_acc_hdl_init(ddi_acc_hdl_t * handlep)270 impl_acc_hdl_init(ddi_acc_hdl_t *handlep)
271 {
272 ddi_acc_impl_t *hp;
273 int fmcap;
274 int devacc_attr_access;
275
276 if (!handlep)
277 return;
278 fmcap = ddi_fm_capable(handlep->ah_dip);
279 if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
280 !DDI_FM_ACC_ERR_CAP(fmcap))
281 devacc_attr_access = DDI_DEFAULT_ACC;
282 else
283 devacc_attr_access = handlep->ah_acc.devacc_attr_access;
284
285 hp = (ddi_acc_impl_t *)handlep->ah_platform_private;
286
287 /*
288 * Can only do FLAGERR if scan callback is set up. This should
289 * also guarantee that the peekpoke_mutex and err_mutex are defined.
290 */
291 if (devacc_attr_access == DDI_FLAGERR_ACC && hp->ahi_scan == NULL)
292 devacc_attr_access = DDI_DEFAULT_ACC;
293
294 switch (devacc_attr_access) {
295 case DDI_CAUTIOUS_ACC:
296 hp->ahi_get8 = i_ddi_caut_get8;
297 hp->ahi_put8 = i_ddi_caut_put8;
298 hp->ahi_rep_get8 = i_ddi_caut_rep_get8;
299 hp->ahi_rep_put8 = i_ddi_caut_rep_put8;
300 hp->ahi_get16 = i_ddi_caut_get16;
301 hp->ahi_get32 = i_ddi_caut_get32;
302 hp->ahi_put16 = i_ddi_caut_put16;
303 hp->ahi_put32 = i_ddi_caut_put32;
304 hp->ahi_rep_get16 = i_ddi_caut_rep_get16;
305 hp->ahi_rep_get32 = i_ddi_caut_rep_get32;
306 hp->ahi_rep_put16 = i_ddi_caut_rep_put16;
307 hp->ahi_rep_put32 = i_ddi_caut_rep_put32;
308 hp->ahi_get64 = i_ddi_caut_get64;
309 hp->ahi_put64 = i_ddi_caut_put64;
310 hp->ahi_rep_get64 = i_ddi_caut_rep_get64;
311 hp->ahi_rep_put64 = i_ddi_caut_rep_put64;
312 break;
313 case DDI_FLAGERR_ACC:
314 if (hp->ahi_acc_attr & DDI_ACCATTR_IO_SPACE) {
315 hp->ahi_get8 = i_ddi_prot_io_get8;
316 hp->ahi_put8 = i_ddi_prot_io_put8;
317 hp->ahi_rep_get8 = i_ddi_prot_io_rep_get8;
318 hp->ahi_rep_put8 = i_ddi_prot_io_rep_put8;
319
320 /* temporary set these 64 functions to no-ops */
321 hp->ahi_get64 = i_ddi_io_get64;
322 hp->ahi_put64 = i_ddi_io_put64;
323 hp->ahi_rep_get64 = i_ddi_io_rep_get64;
324 hp->ahi_rep_put64 = i_ddi_io_rep_put64;
325
326 /*
327 * check for BIG endian access
328 */
329 if (handlep->ah_acc.devacc_attr_endian_flags ==
330 DDI_STRUCTURE_BE_ACC) {
331 hp->ahi_get16 = i_ddi_prot_io_swap_get16;
332 hp->ahi_get32 = i_ddi_prot_io_swap_get32;
333 hp->ahi_put16 = i_ddi_prot_io_swap_put16;
334 hp->ahi_put32 = i_ddi_prot_io_swap_put32;
335 hp->ahi_rep_get16 =
336 i_ddi_prot_io_swap_rep_get16;
337 hp->ahi_rep_get32 =
338 i_ddi_prot_io_swap_rep_get32;
339 hp->ahi_rep_put16 =
340 i_ddi_prot_io_swap_rep_put16;
341 hp->ahi_rep_put32 =
342 i_ddi_prot_io_swap_rep_put32;
343 } else {
344 hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
345 hp->ahi_get16 = i_ddi_prot_io_get16;
346 hp->ahi_get32 = i_ddi_prot_io_get32;
347 hp->ahi_put16 = i_ddi_prot_io_put16;
348 hp->ahi_put32 = i_ddi_prot_io_put32;
349 hp->ahi_rep_get16 = i_ddi_prot_io_rep_get16;
350 hp->ahi_rep_get32 = i_ddi_prot_io_rep_get32;
351 hp->ahi_rep_put16 = i_ddi_prot_io_rep_put16;
352 hp->ahi_rep_put32 = i_ddi_prot_io_rep_put32;
353 }
354
355 } else if (hp->ahi_acc_attr & DDI_ACCATTR_CPU_VADDR) {
356
357 hp->ahi_get8 = i_ddi_prot_vaddr_get8;
358 hp->ahi_put8 = i_ddi_prot_vaddr_put8;
359 hp->ahi_rep_get8 = i_ddi_prot_vaddr_rep_get8;
360 hp->ahi_rep_put8 = i_ddi_prot_vaddr_rep_put8;
361
362 /*
363 * check for BIG endian access
364 */
365 if (handlep->ah_acc.devacc_attr_endian_flags ==
366 DDI_STRUCTURE_BE_ACC) {
367
368 hp->ahi_get16 = i_ddi_prot_vaddr_swap_get16;
369 hp->ahi_get32 = i_ddi_prot_vaddr_swap_get32;
370 hp->ahi_get64 = i_ddi_prot_vaddr_swap_get64;
371 hp->ahi_put16 = i_ddi_prot_vaddr_swap_put16;
372 hp->ahi_put32 = i_ddi_prot_vaddr_swap_put32;
373 hp->ahi_put64 = i_ddi_prot_vaddr_swap_put64;
374 hp->ahi_rep_get16 =
375 i_ddi_prot_vaddr_swap_rep_get16;
376 hp->ahi_rep_get32 =
377 i_ddi_prot_vaddr_swap_rep_get32;
378 hp->ahi_rep_get64 =
379 i_ddi_prot_vaddr_swap_rep_get64;
380 hp->ahi_rep_put16 =
381 i_ddi_prot_vaddr_swap_rep_put16;
382 hp->ahi_rep_put32 =
383 i_ddi_prot_vaddr_swap_rep_put32;
384 hp->ahi_rep_put64 =
385 i_ddi_prot_vaddr_swap_rep_put64;
386 } else {
387 hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
388 hp->ahi_get16 = i_ddi_prot_vaddr_get16;
389 hp->ahi_get32 = i_ddi_prot_vaddr_get32;
390 hp->ahi_get64 = i_ddi_prot_vaddr_get64;
391 hp->ahi_put16 = i_ddi_prot_vaddr_put16;
392 hp->ahi_put32 = i_ddi_prot_vaddr_put32;
393 hp->ahi_put64 = i_ddi_prot_vaddr_put64;
394 hp->ahi_rep_get16 = i_ddi_prot_vaddr_rep_get16;
395 hp->ahi_rep_get32 = i_ddi_prot_vaddr_rep_get32;
396 hp->ahi_rep_get64 = i_ddi_prot_vaddr_rep_get64;
397 hp->ahi_rep_put16 = i_ddi_prot_vaddr_rep_put16;
398 hp->ahi_rep_put32 = i_ddi_prot_vaddr_rep_put32;
399 hp->ahi_rep_put64 = i_ddi_prot_vaddr_rep_put64;
400 }
401 }
402 break;
403 case DDI_DEFAULT_ACC:
404 if (hp->ahi_acc_attr & DDI_ACCATTR_IO_SPACE) {
405 hp->ahi_get8 = i_ddi_io_get8;
406 hp->ahi_put8 = i_ddi_io_put8;
407 hp->ahi_rep_get8 = i_ddi_io_rep_get8;
408 hp->ahi_rep_put8 = i_ddi_io_rep_put8;
409
410 /* temporary set these 64 functions to no-ops */
411 hp->ahi_get64 = i_ddi_io_get64;
412 hp->ahi_put64 = i_ddi_io_put64;
413 hp->ahi_rep_get64 = i_ddi_io_rep_get64;
414 hp->ahi_rep_put64 = i_ddi_io_rep_put64;
415
416 /*
417 * check for BIG endian access
418 */
419 if (handlep->ah_acc.devacc_attr_endian_flags ==
420 DDI_STRUCTURE_BE_ACC) {
421 hp->ahi_get16 = i_ddi_io_swap_get16;
422 hp->ahi_get32 = i_ddi_io_swap_get32;
423 hp->ahi_put16 = i_ddi_io_swap_put16;
424 hp->ahi_put32 = i_ddi_io_swap_put32;
425 hp->ahi_rep_get16 = i_ddi_io_swap_rep_get16;
426 hp->ahi_rep_get32 = i_ddi_io_swap_rep_get32;
427 hp->ahi_rep_put16 = i_ddi_io_swap_rep_put16;
428 hp->ahi_rep_put32 = i_ddi_io_swap_rep_put32;
429 } else {
430 hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
431 hp->ahi_get16 = i_ddi_io_get16;
432 hp->ahi_get32 = i_ddi_io_get32;
433 hp->ahi_put16 = i_ddi_io_put16;
434 hp->ahi_put32 = i_ddi_io_put32;
435 hp->ahi_rep_get16 = i_ddi_io_rep_get16;
436 hp->ahi_rep_get32 = i_ddi_io_rep_get32;
437 hp->ahi_rep_put16 = i_ddi_io_rep_put16;
438 hp->ahi_rep_put32 = i_ddi_io_rep_put32;
439 }
440
441 } else if (hp->ahi_acc_attr & DDI_ACCATTR_CPU_VADDR) {
442
443 hp->ahi_get8 = i_ddi_vaddr_get8;
444 hp->ahi_put8 = i_ddi_vaddr_put8;
445 hp->ahi_rep_get8 = i_ddi_vaddr_rep_get8;
446 hp->ahi_rep_put8 = i_ddi_vaddr_rep_put8;
447
448 /*
449 * check for BIG endian access
450 */
451 if (handlep->ah_acc.devacc_attr_endian_flags ==
452 DDI_STRUCTURE_BE_ACC) {
453
454 hp->ahi_get16 = i_ddi_vaddr_swap_get16;
455 hp->ahi_get32 = i_ddi_vaddr_swap_get32;
456 hp->ahi_get64 = i_ddi_vaddr_swap_get64;
457 hp->ahi_put16 = i_ddi_vaddr_swap_put16;
458 hp->ahi_put32 = i_ddi_vaddr_swap_put32;
459 hp->ahi_put64 = i_ddi_vaddr_swap_put64;
460 hp->ahi_rep_get16 = i_ddi_vaddr_swap_rep_get16;
461 hp->ahi_rep_get32 = i_ddi_vaddr_swap_rep_get32;
462 hp->ahi_rep_get64 = i_ddi_vaddr_swap_rep_get64;
463 hp->ahi_rep_put16 = i_ddi_vaddr_swap_rep_put16;
464 hp->ahi_rep_put32 = i_ddi_vaddr_swap_rep_put32;
465 hp->ahi_rep_put64 = i_ddi_vaddr_swap_rep_put64;
466 } else {
467 hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
468 hp->ahi_get16 = i_ddi_vaddr_get16;
469 hp->ahi_get32 = i_ddi_vaddr_get32;
470 hp->ahi_get64 = i_ddi_vaddr_get64;
471 hp->ahi_put16 = i_ddi_vaddr_put16;
472 hp->ahi_put32 = i_ddi_vaddr_put32;
473 hp->ahi_put64 = i_ddi_vaddr_put64;
474 hp->ahi_rep_get16 = i_ddi_vaddr_rep_get16;
475 hp->ahi_rep_get32 = i_ddi_vaddr_rep_get32;
476 hp->ahi_rep_get64 = i_ddi_vaddr_rep_get64;
477 hp->ahi_rep_put16 = i_ddi_vaddr_rep_put16;
478 hp->ahi_rep_put32 = i_ddi_vaddr_rep_put32;
479 hp->ahi_rep_put64 = i_ddi_vaddr_rep_put64;
480 }
481 }
482 break;
483 }
484 hp->ahi_fault_check = i_ddi_acc_fault_check;
485 hp->ahi_fault_notify = i_ddi_acc_fault_notify;
486 hp->ahi_fault = 0;
487 impl_acc_err_init(handlep);
488 }
489
490 /*
491 * The followings are low-level routines for data access.
492 *
493 * All of these routines should be implemented in assembly. Those
494 * that have been rewritten be found in ~ml/ddi_i86_asm.s
495 */
496
497 /*ARGSUSED*/
498 uint16_t
i_ddi_vaddr_swap_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)499 i_ddi_vaddr_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
500 {
501 return (ddi_swap16(*addr));
502 }
503
504 /*ARGSUSED*/
505 uint16_t
i_ddi_io_swap_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)506 i_ddi_io_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
507 {
508 return (ddi_swap16(inw((uintptr_t)addr)));
509 }
510
511 /*ARGSUSED*/
512 uint32_t
i_ddi_vaddr_swap_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)513 i_ddi_vaddr_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
514 {
515 return (ddi_swap32(*addr));
516 }
517
518 /*ARGSUSED*/
519 uint32_t
i_ddi_io_swap_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)520 i_ddi_io_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
521 {
522 return (ddi_swap32(inl((uintptr_t)addr)));
523 }
524
525 /*ARGSUSED*/
526 uint64_t
i_ddi_vaddr_swap_get64(ddi_acc_impl_t * hdlp,uint64_t * addr)527 i_ddi_vaddr_swap_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
528 {
529 return (ddi_swap64(*addr));
530 }
531
532 /*ARGSUSED*/
533 void
i_ddi_vaddr_swap_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)534 i_ddi_vaddr_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
535 {
536 *addr = ddi_swap16(value);
537 }
538
539 /*ARGSUSED*/
540 void
i_ddi_io_swap_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)541 i_ddi_io_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
542 {
543 outw((uintptr_t)addr, ddi_swap16(value));
544 }
545
546 /*ARGSUSED*/
547 void
i_ddi_vaddr_swap_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)548 i_ddi_vaddr_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
549 {
550 *addr = ddi_swap32(value);
551 }
552
553 /*ARGSUSED*/
554 void
i_ddi_io_swap_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)555 i_ddi_io_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
556 {
557 outl((uintptr_t)addr, ddi_swap32(value));
558 }
559
560 /*ARGSUSED*/
561 void
i_ddi_vaddr_swap_put64(ddi_acc_impl_t * hdlp,uint64_t * addr,uint64_t value)562 i_ddi_vaddr_swap_put64(ddi_acc_impl_t *hdlp, uint64_t *addr, uint64_t value)
563 {
564 *addr = ddi_swap64(value);
565 }
566
567 /*ARGSUSED*/
568 void
i_ddi_vaddr_rep_get8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)569 i_ddi_vaddr_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
570 uint8_t *dev_addr, size_t repcount, uint_t flags)
571 {
572 uint8_t *h, *d;
573
574 h = host_addr;
575 d = dev_addr;
576
577 if (flags == DDI_DEV_AUTOINCR)
578 for (; repcount; repcount--)
579 *h++ = *d++;
580 else
581 for (; repcount; repcount--)
582 *h++ = *d;
583 }
584
585 /*ARGSUSED*/
586 void
i_ddi_vaddr_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)587 i_ddi_vaddr_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
588 uint16_t *dev_addr, size_t repcount, uint_t flags)
589 {
590 uint16_t *h, *d;
591
592 h = host_addr;
593 d = dev_addr;
594
595 if (flags == DDI_DEV_AUTOINCR)
596 for (; repcount; repcount--)
597 *h++ = *d++;
598 else
599 for (; repcount; repcount--)
600 *h++ = *d;
601 }
602
603 /*ARGSUSED*/
604 void
i_ddi_vaddr_swap_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)605 i_ddi_vaddr_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
606 uint16_t *dev_addr, size_t repcount, uint_t flags)
607 {
608 uint16_t *h, *d;
609
610 h = host_addr;
611 d = dev_addr;
612
613 if (flags == DDI_DEV_AUTOINCR)
614 for (; repcount; repcount--)
615 *h++ = ddi_swap16(*d++);
616 else
617 for (; repcount; repcount--)
618 *h++ = ddi_swap16(*d);
619 }
620
621 /*ARGSUSED*/
622 void
i_ddi_io_swap_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)623 i_ddi_io_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
624 uint16_t *dev_addr, size_t repcount, uint_t flags)
625 {
626 uint16_t *h;
627 uintptr_t port;
628
629 h = host_addr;
630 port = (uintptr_t)dev_addr;
631
632 if (flags == DDI_DEV_AUTOINCR)
633 for (; repcount; repcount--, port += 2)
634 *h++ = ddi_swap16(inw(port));
635 else
636 for (; repcount; repcount--)
637 *h++ = ddi_swap16(inw(port));
638 }
639
640 /*ARGSUSED*/
641 void
i_ddi_vaddr_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)642 i_ddi_vaddr_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
643 uint32_t *dev_addr, size_t repcount, uint_t flags)
644 {
645 uint32_t *h, *d;
646
647 h = host_addr;
648 d = dev_addr;
649
650 if (flags == DDI_DEV_AUTOINCR)
651 for (; repcount; repcount--)
652 *h++ = *d++;
653 else
654 for (; repcount; repcount--)
655 *h++ = *d;
656 }
657
658 /*ARGSUSED*/
659 void
i_ddi_vaddr_swap_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)660 i_ddi_vaddr_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
661 uint32_t *dev_addr, size_t repcount, uint_t flags)
662 {
663 uint32_t *h, *d;
664
665 h = host_addr;
666 d = dev_addr;
667
668 if (flags == DDI_DEV_AUTOINCR)
669 for (; repcount; repcount--)
670 *h++ = ddi_swap32(*d++);
671 else
672 for (; repcount; repcount--)
673 *h++ = ddi_swap32(*d);
674 }
675
676 /*ARGSUSED*/
677 void
i_ddi_io_swap_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)678 i_ddi_io_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
679 uint32_t *dev_addr, size_t repcount, uint_t flags)
680 {
681 uint32_t *h;
682 uintptr_t port;
683
684 h = host_addr;
685 port = (uintptr_t)dev_addr;
686
687 if (flags == DDI_DEV_AUTOINCR)
688 for (; repcount; repcount--, port += 4)
689 *h++ = ddi_swap32(inl(port));
690 else
691 for (; repcount; repcount--)
692 *h++ = ddi_swap32(inl(port));
693 }
694
695 /*ARGSUSED*/
696 void
i_ddi_vaddr_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)697 i_ddi_vaddr_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
698 uint64_t *dev_addr, size_t repcount, uint_t flags)
699 {
700 uint64_t *h, *d;
701
702 h = host_addr;
703 d = dev_addr;
704
705 if (flags == DDI_DEV_AUTOINCR)
706 for (; repcount; repcount--)
707 *h++ = *d++;
708 else
709 for (; repcount; repcount--)
710 *h++ = *d;
711 }
712
713 /*ARGSUSED*/
714 void
i_ddi_vaddr_swap_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)715 i_ddi_vaddr_swap_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
716 uint64_t *dev_addr, size_t repcount, uint_t flags)
717 {
718 uint64_t *h, *d;
719
720 h = host_addr;
721 d = dev_addr;
722
723 if (flags == DDI_DEV_AUTOINCR)
724 for (; repcount; repcount--)
725 *h++ = ddi_swap64(*d++);
726 else
727 for (; repcount; repcount--)
728 *h++ = ddi_swap64(*d);
729 }
730
731 /*ARGSUSED*/
732 void
i_ddi_vaddr_rep_put8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)733 i_ddi_vaddr_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
734 uint8_t *dev_addr, size_t repcount, uint_t flags)
735 {
736 uint8_t *h, *d;
737
738 h = host_addr;
739 d = dev_addr;
740
741 if (flags == DDI_DEV_AUTOINCR)
742 for (; repcount; repcount--)
743 *d++ = *h++;
744 else
745 for (; repcount; repcount--)
746 *d = *h++;
747 }
748
749 /*ARGSUSED*/
750 void
i_ddi_vaddr_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)751 i_ddi_vaddr_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
752 uint16_t *dev_addr, size_t repcount, uint_t flags)
753 {
754 uint16_t *h, *d;
755
756 h = host_addr;
757 d = dev_addr;
758
759 if (flags == DDI_DEV_AUTOINCR)
760 for (; repcount; repcount--)
761 *d++ = *h++;
762 else
763 for (; repcount; repcount--)
764 *d = *h++;
765 }
766
767 /*ARGSUSED*/
768 void
i_ddi_vaddr_swap_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)769 i_ddi_vaddr_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
770 uint16_t *dev_addr, size_t repcount, uint_t flags)
771 {
772 uint16_t *h, *d;
773
774 h = host_addr;
775 d = dev_addr;
776
777 if (flags == DDI_DEV_AUTOINCR)
778 for (; repcount; repcount--)
779 *d++ = ddi_swap16(*h++);
780 else
781 for (; repcount; repcount--)
782 *d = ddi_swap16(*h++);
783 }
784
785 /*ARGSUSED*/
786 void
i_ddi_io_swap_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)787 i_ddi_io_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
788 uint16_t *dev_addr, size_t repcount, uint_t flags)
789 {
790 uint16_t *h;
791 uintptr_t port;
792
793 h = host_addr;
794 port = (uintptr_t)dev_addr;
795
796 if (flags == DDI_DEV_AUTOINCR)
797 for (; repcount; repcount--, port += 2)
798 outw(port, ddi_swap16(*h++));
799 else
800 for (; repcount; repcount--)
801 outw(port, ddi_swap16(*h++));
802 }
803
804 /*ARGSUSED*/
805 void
i_ddi_vaddr_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)806 i_ddi_vaddr_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
807 uint32_t *dev_addr, size_t repcount, uint_t flags)
808 {
809 uint32_t *h, *d;
810
811 h = host_addr;
812 d = dev_addr;
813
814 if (flags == DDI_DEV_AUTOINCR)
815 for (; repcount; repcount--)
816 *d++ = *h++;
817 else
818 for (; repcount; repcount--)
819 *d = *h++;
820 }
821
822 /*ARGSUSED*/
823 void
i_ddi_vaddr_swap_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)824 i_ddi_vaddr_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
825 uint32_t *dev_addr, size_t repcount, uint_t flags)
826 {
827 uint32_t *h, *d;
828
829 h = host_addr;
830 d = dev_addr;
831
832 if (flags == DDI_DEV_AUTOINCR)
833 for (; repcount; repcount--)
834 *d++ = ddi_swap32(*h++);
835 else
836 for (; repcount; repcount--)
837 *d = ddi_swap32(*h++);
838 }
839
840 /*ARGSUSED*/
841 void
i_ddi_io_swap_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)842 i_ddi_io_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
843 uint32_t *dev_addr, size_t repcount, uint_t flags)
844 {
845 uint32_t *h;
846 uintptr_t port;
847
848 h = host_addr;
849 port = (uintptr_t)dev_addr;
850
851 if (flags == DDI_DEV_AUTOINCR)
852 for (; repcount; repcount--, port += 4)
853 outl(port, ddi_swap32(*h++));
854 else
855 for (; repcount; repcount--)
856 outl(port, ddi_swap32(*h++));
857 }
858
859 /*ARGSUSED*/
860 void
i_ddi_vaddr_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)861 i_ddi_vaddr_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
862 uint64_t *dev_addr, size_t repcount, uint_t flags)
863 {
864 uint64_t *h, *d;
865
866 h = host_addr;
867 d = dev_addr;
868
869 if (flags == DDI_DEV_AUTOINCR)
870 for (; repcount; repcount--)
871 *d++ = *h++;
872 else
873 for (; repcount; repcount--)
874 *d = *h++;
875 }
876
877 /*ARGSUSED*/
878 void
i_ddi_vaddr_swap_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)879 i_ddi_vaddr_swap_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
880 uint64_t *dev_addr, size_t repcount, uint_t flags)
881 {
882 uint64_t *h, *d;
883
884 h = host_addr;
885 d = dev_addr;
886
887 if (flags == DDI_DEV_AUTOINCR)
888 for (; repcount; repcount--)
889 *d++ = ddi_swap64(*h++);
890 else
891 for (; repcount; repcount--)
892 *d = ddi_swap64(*h++);
893 }
894
895 /*ARGSUSED*/
896 uint64_t
i_ddi_io_get64(ddi_acc_impl_t * hdlp,uint64_t * addr)897 i_ddi_io_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
898 {
899 panic("ddi_get64 from i/o space");
900 /*NOTREACHED*/
901 return (0);
902 }
903
904 /*ARGSUSED*/
905 void
i_ddi_io_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t value)906 i_ddi_io_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr, uint64_t value)
907 {
908 panic("ddi_put64 to i/o space");
909 /*NOTREACHED*/
910 }
911
912 void
do_scan(ddi_acc_impl_t * hdlp)913 do_scan(ddi_acc_impl_t *hdlp)
914 {
915 ddi_fm_error_t de;
916 ndi_err_t *errp = (ndi_err_t *)hdlp->ahi_err;
917
918 bzero(&de, sizeof (ddi_fm_error_t));
919 de.fme_version = DDI_FME_VERSION;
920 de.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
921 de.fme_flag = DDI_FM_ERR_UNEXPECTED;
922
923 mutex_enter(hdlp->ahi_err_mutexp);
924 hdlp->ahi_scan(hdlp->ahi_scan_dip, &de);
925 if (de.fme_status != DDI_FM_OK) {
926 errp->err_ena = de.fme_ena;
927 errp->err_expected = de.fme_flag;
928 errp->err_status = DDI_FM_NONFATAL;
929 }
930 mutex_exit(hdlp->ahi_err_mutexp);
931 }
932
933 /*ARGSUSED*/
934 uint8_t
i_ddi_prot_vaddr_get8(ddi_acc_impl_t * hdlp,uint8_t * addr)935 i_ddi_prot_vaddr_get8(ddi_acc_impl_t *hdlp, uint8_t *addr)
936 {
937 uint8_t val;
938
939 mutex_enter(hdlp->ahi_peekpoke_mutexp);
940 val = *addr;
941 if (val == 0xff)
942 do_scan(hdlp);
943 mutex_exit(hdlp->ahi_peekpoke_mutexp);
944
945 return (val);
946 }
947
948 /*ARGSUSED*/
949 uint16_t
i_ddi_prot_vaddr_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)950 i_ddi_prot_vaddr_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
951 {
952 uint16_t val;
953
954 mutex_enter(hdlp->ahi_peekpoke_mutexp);
955 val = *addr;
956 if (val == 0xffff)
957 do_scan(hdlp);
958 mutex_exit(hdlp->ahi_peekpoke_mutexp);
959
960 return (val);
961 }
962
963 /*ARGSUSED*/
964 uint32_t
i_ddi_prot_vaddr_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)965 i_ddi_prot_vaddr_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
966 {
967 uint32_t val;
968
969 mutex_enter(hdlp->ahi_peekpoke_mutexp);
970 val = *addr;
971 if (val == 0xffffffff)
972 do_scan(hdlp);
973 mutex_exit(hdlp->ahi_peekpoke_mutexp);
974
975 return (val);
976 }
977
978 /*ARGSUSED*/
979 uint64_t
i_ddi_prot_vaddr_get64(ddi_acc_impl_t * hdlp,uint64_t * addr)980 i_ddi_prot_vaddr_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
981 {
982 uint64_t val;
983
984 mutex_enter(hdlp->ahi_peekpoke_mutexp);
985 val = *addr;
986 if (val == 0xffffffffffffffff)
987 do_scan(hdlp);
988 mutex_exit(hdlp->ahi_peekpoke_mutexp);
989
990 return (val);
991 }
992
993 /*ARGSUSED*/
994 uint8_t
i_ddi_prot_io_get8(ddi_acc_impl_t * hdlp,uint8_t * addr)995 i_ddi_prot_io_get8(ddi_acc_impl_t *hdlp, uint8_t *addr)
996 {
997 uint8_t val;
998
999 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1000 val = inb((uintptr_t)addr);
1001 if (val == 0xff)
1002 do_scan(hdlp);
1003 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1004
1005 return (val);
1006 }
1007
1008 /*ARGSUSED*/
1009 uint16_t
i_ddi_prot_io_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)1010 i_ddi_prot_io_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
1011 {
1012 uint16_t val;
1013
1014 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1015 val = inw((uintptr_t)addr);
1016 if (val == 0xffff)
1017 do_scan(hdlp);
1018 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1019
1020 return (val);
1021 }
1022
1023 /*ARGSUSED*/
1024 uint32_t
i_ddi_prot_io_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)1025 i_ddi_prot_io_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
1026 {
1027 uint32_t val;
1028
1029 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1030 val = inl((uintptr_t)addr);
1031 if (val == 0xffffffff)
1032 do_scan(hdlp);
1033 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1034
1035 return (val);
1036 }
1037
1038 /*ARGSUSED*/
1039 uint16_t
i_ddi_prot_vaddr_swap_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)1040 i_ddi_prot_vaddr_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
1041 {
1042 uint16_t val;
1043
1044 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1045 val = ddi_swap16(*addr);
1046 if (val == 0xffff)
1047 do_scan(hdlp);
1048 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1049
1050 return (val);
1051 }
1052
1053 /*ARGSUSED*/
1054 uint16_t
i_ddi_prot_io_swap_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)1055 i_ddi_prot_io_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
1056 {
1057 uint16_t val;
1058
1059 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1060 val = ddi_swap16(inw((uintptr_t)addr));
1061 if (val == 0xffff)
1062 do_scan(hdlp);
1063 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1064
1065 return (val);
1066 }
1067
1068 /*ARGSUSED*/
1069 uint32_t
i_ddi_prot_vaddr_swap_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)1070 i_ddi_prot_vaddr_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
1071 {
1072 uint32_t val;
1073
1074 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1075 val = ddi_swap32(*addr);
1076 if (val == 0xffffffff)
1077 do_scan(hdlp);
1078 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1079
1080 return (val);
1081 }
1082
1083 /*ARGSUSED*/
1084 uint32_t
i_ddi_prot_io_swap_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)1085 i_ddi_prot_io_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
1086 {
1087 uint32_t val;
1088
1089 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1090 val = ddi_swap32(inl((uintptr_t)addr));
1091 if (val == 0xffffffff)
1092 do_scan(hdlp);
1093 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1094
1095 return (val);
1096 }
1097
1098 /*ARGSUSED*/
1099 uint64_t
i_ddi_prot_vaddr_swap_get64(ddi_acc_impl_t * hdlp,uint64_t * addr)1100 i_ddi_prot_vaddr_swap_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
1101 {
1102 uint64_t val;
1103
1104 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1105 val = ddi_swap64(*addr);
1106 if (val == 0xffffffffffffffff)
1107 do_scan(hdlp);
1108 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1109
1110 return (val);
1111 }
1112
1113 /*ARGSUSED*/
1114 void
i_ddi_prot_vaddr_put8(ddi_acc_impl_t * hdlp,uint8_t * addr,uint8_t value)1115 i_ddi_prot_vaddr_put8(ddi_acc_impl_t *hdlp, uint8_t *addr, uint8_t value)
1116 {
1117 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1118 *addr = value;
1119 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1120 }
1121
1122 /*ARGSUSED*/
1123 void
i_ddi_prot_io_put8(ddi_acc_impl_t * hdlp,uint8_t * addr,uint8_t value)1124 i_ddi_prot_io_put8(ddi_acc_impl_t *hdlp, uint8_t *addr, uint8_t value)
1125 {
1126 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1127 outb((uintptr_t)addr, value);
1128 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1129 }
1130
1131 /*ARGSUSED*/
1132 void
i_ddi_prot_vaddr_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)1133 i_ddi_prot_vaddr_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
1134 {
1135 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1136 *addr = value;
1137 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1138 }
1139
1140 /*ARGSUSED*/
1141 void
i_ddi_prot_io_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)1142 i_ddi_prot_io_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
1143 {
1144 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1145 outw((uintptr_t)addr, value);
1146 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1147 }
1148
1149 /*ARGSUSED*/
1150 void
i_ddi_prot_vaddr_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)1151 i_ddi_prot_vaddr_put32(ddi_acc_impl_t *hdlp, uint32_t *addr,
1152 uint32_t value)
1153 {
1154 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1155 *addr = value;
1156 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1157 }
1158
1159 /*ARGSUSED*/
1160 void
i_ddi_prot_io_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)1161 i_ddi_prot_io_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
1162 {
1163 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1164 outl((uintptr_t)addr, value);
1165 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1166 }
1167
1168 /*ARGSUSED*/
1169 void
i_ddi_prot_vaddr_put64(ddi_acc_impl_t * hdlp,uint64_t * addr,uint64_t value)1170 i_ddi_prot_vaddr_put64(ddi_acc_impl_t *hdlp, uint64_t *addr,
1171 uint64_t value)
1172 {
1173 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1174 *addr = value;
1175 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1176 }
1177
1178 /*ARGSUSED*/
1179 void
i_ddi_prot_vaddr_swap_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)1180 i_ddi_prot_vaddr_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr,
1181 uint16_t value)
1182 {
1183 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1184 *addr = ddi_swap16(value);
1185 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1186 }
1187
1188 /*ARGSUSED*/
1189 void
i_ddi_prot_io_swap_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)1190 i_ddi_prot_io_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
1191 {
1192 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1193 outw((uintptr_t)addr, ddi_swap16(value));
1194 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1195 }
1196
1197 /*ARGSUSED*/
1198 void
i_ddi_prot_vaddr_swap_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)1199 i_ddi_prot_vaddr_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr,
1200 uint32_t value)
1201 {
1202 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1203 *addr = ddi_swap32(value);
1204 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1205 }
1206
1207 /*ARGSUSED*/
1208 void
i_ddi_prot_io_swap_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)1209 i_ddi_prot_io_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
1210 {
1211 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1212 outl((uintptr_t)addr, ddi_swap32(value));
1213 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1214 }
1215
1216 /*ARGSUSED*/
1217 void
i_ddi_prot_vaddr_swap_put64(ddi_acc_impl_t * hdlp,uint64_t * addr,uint64_t value)1218 i_ddi_prot_vaddr_swap_put64(ddi_acc_impl_t *hdlp, uint64_t *addr,
1219 uint64_t value)
1220 {
1221 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1222 *addr = ddi_swap64(value);
1223 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1224 }
1225
1226 /*ARGSUSED*/
1227 void
i_ddi_prot_io_rep_get8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1228 i_ddi_prot_io_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1229 uint8_t *dev_addr, size_t repcount, uint_t flags)
1230 {
1231 int fail = 0;
1232 uint8_t *h;
1233 uintptr_t port;
1234
1235 h = host_addr;
1236 port = (uintptr_t)dev_addr;
1237
1238 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1239 if (flags == DDI_DEV_AUTOINCR) {
1240 for (; repcount; repcount--, port++)
1241 if ((*h++ = inb(port)) == 0xff)
1242 fail = 1;
1243 } else {
1244 for (; repcount; repcount--)
1245 if ((*h++ = inb(port)) == 0xff)
1246 fail = 1;
1247 }
1248 if (fail == 1)
1249 do_scan(hdlp);
1250 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1251 }
1252
1253 /*ARGSUSED*/
1254 void
i_ddi_prot_io_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1255 i_ddi_prot_io_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1256 uint16_t *dev_addr, size_t repcount, uint_t flags)
1257 {
1258 int fail = 0;
1259 uint16_t *h;
1260 uintptr_t port;
1261
1262 h = host_addr;
1263 port = (uintptr_t)dev_addr;
1264
1265 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1266 if (flags == DDI_DEV_AUTOINCR) {
1267 for (; repcount; repcount--, port += 2)
1268 if ((*h++ = inw(port)) == 0xffff)
1269 fail = 1;
1270 } else {
1271 for (; repcount; repcount--)
1272 if ((*h++ = inw(port)) == 0xffff)
1273 fail = 1;
1274 }
1275 if (fail == 1)
1276 do_scan(hdlp);
1277 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1278 }
1279
1280 /*ARGSUSED*/
1281 void
i_ddi_prot_io_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1282 i_ddi_prot_io_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1283 uint32_t *dev_addr, size_t repcount, uint_t flags)
1284 {
1285 int fail = 0;
1286 uint32_t *h;
1287 uintptr_t port;
1288
1289 h = host_addr;
1290 port = (uintptr_t)dev_addr;
1291
1292 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1293 if (flags == DDI_DEV_AUTOINCR) {
1294 for (; repcount; repcount--, port += 4)
1295 if ((*h++ = inl(port)) == 0xffffffff)
1296 fail = 1;
1297 } else {
1298 for (; repcount; repcount--)
1299 if ((*h++ = inl(port)) == 0xffffffff)
1300 fail = 1;
1301 }
1302 if (fail == 1)
1303 do_scan(hdlp);
1304 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1305 }
1306
1307 /*ARGSUSED*/
1308 void
i_ddi_prot_vaddr_rep_get8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1309 i_ddi_prot_vaddr_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1310 uint8_t *dev_addr, size_t repcount, uint_t flags)
1311 {
1312 int fail = 0;
1313 uint8_t *h, *d;
1314
1315 h = host_addr;
1316 d = dev_addr;
1317
1318 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1319 if (flags == DDI_DEV_AUTOINCR) {
1320 for (; repcount; repcount--)
1321 if ((*h++ = *d++) == 0xff)
1322 fail = 1;
1323 } else {
1324 for (; repcount; repcount--)
1325 if ((*h++ = *d) == 0xff)
1326 fail = 1;
1327 }
1328 if (fail == 1)
1329 do_scan(hdlp);
1330 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1331 }
1332
1333 /*ARGSUSED*/
1334 void
i_ddi_prot_vaddr_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1335 i_ddi_prot_vaddr_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1336 uint16_t *dev_addr, size_t repcount, uint_t flags)
1337 {
1338 int fail = 0;
1339 uint16_t *h, *d;
1340
1341 h = host_addr;
1342 d = dev_addr;
1343
1344 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1345 if (flags == DDI_DEV_AUTOINCR) {
1346 for (; repcount; repcount--)
1347 if ((*h++ = *d++) == 0xffff)
1348 fail = 1;
1349 } else {
1350 for (; repcount; repcount--)
1351 if ((*h++ = *d) == 0xffff)
1352 fail = 1;
1353 }
1354 if (fail == 1)
1355 do_scan(hdlp);
1356 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1357 }
1358
1359 /*ARGSUSED*/
1360 void
i_ddi_prot_vaddr_swap_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1361 i_ddi_prot_vaddr_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1362 uint16_t *dev_addr, size_t repcount, uint_t flags)
1363 {
1364 int fail = 0;
1365 uint16_t *h, *d;
1366
1367 h = host_addr;
1368 d = dev_addr;
1369
1370 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1371 if (flags == DDI_DEV_AUTOINCR) {
1372 for (; repcount; repcount--)
1373 if ((*h++ = ddi_swap16(*d++)) == 0xffff)
1374 fail = 1;
1375 } else {
1376 for (; repcount; repcount--)
1377 if ((*h++ = ddi_swap16(*d)) == 0xffff)
1378 fail = 1;
1379 }
1380 if (fail == 1)
1381 do_scan(hdlp);
1382 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1383 }
1384
1385 /*ARGSUSED*/
1386 void
i_ddi_prot_io_swap_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1387 i_ddi_prot_io_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1388 uint16_t *dev_addr, size_t repcount, uint_t flags)
1389 {
1390 int fail = 0;
1391 uint16_t *h;
1392 uintptr_t port;
1393
1394 h = host_addr;
1395 port = (uintptr_t)dev_addr;
1396
1397 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1398 if (flags == DDI_DEV_AUTOINCR) {
1399 for (; repcount; repcount--, port += 2)
1400 if ((*h++ = ddi_swap16(inw(port))) == 0xffff)
1401 fail = 1;
1402 } else {
1403 for (; repcount; repcount--)
1404 if ((*h++ = ddi_swap16(inw(port))) == 0xffff)
1405 fail = 1;
1406 }
1407 if (fail == 1)
1408 do_scan(hdlp);
1409 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1410 }
1411
1412 /*ARGSUSED*/
1413 void
i_ddi_prot_vaddr_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1414 i_ddi_prot_vaddr_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1415 uint32_t *dev_addr, size_t repcount, uint_t flags)
1416 {
1417 int fail = 0;
1418 uint32_t *h, *d;
1419
1420 h = host_addr;
1421 d = dev_addr;
1422
1423 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1424 if (flags == DDI_DEV_AUTOINCR) {
1425 for (; repcount; repcount--)
1426 if ((*h++ = *d++) == 0xffffffff)
1427 fail = 1;
1428 } else {
1429 for (; repcount; repcount--)
1430 if ((*h++ = *d) == 0xffffffff)
1431 fail = 1;
1432 }
1433 if (fail == 1)
1434 do_scan(hdlp);
1435 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1436 }
1437
1438 /*ARGSUSED*/
1439 void
i_ddi_prot_vaddr_swap_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1440 i_ddi_prot_vaddr_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1441 uint32_t *dev_addr, size_t repcount, uint_t flags)
1442 {
1443 int fail = 0;
1444 uint32_t *h, *d;
1445
1446 h = host_addr;
1447 d = dev_addr;
1448
1449 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1450 if (flags == DDI_DEV_AUTOINCR) {
1451 for (; repcount; repcount--)
1452 if ((*h++ = ddi_swap32(*d++)) == 0xffffffff)
1453 fail = 1;
1454 } else {
1455 for (; repcount; repcount--)
1456 if ((*h++ = ddi_swap32(*d)) == 0xffffffff)
1457 fail = 1;
1458 }
1459 if (fail == 1)
1460 do_scan(hdlp);
1461 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1462 }
1463
1464 /*ARGSUSED*/
1465 void
i_ddi_prot_io_swap_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1466 i_ddi_prot_io_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1467 uint32_t *dev_addr, size_t repcount, uint_t flags)
1468 {
1469 int fail = 0;
1470 uint32_t *h;
1471 uintptr_t port;
1472
1473 h = host_addr;
1474 port = (uintptr_t)dev_addr;
1475
1476 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1477 if (flags == DDI_DEV_AUTOINCR) {
1478 for (; repcount; repcount--, port += 4)
1479 if ((*h++ = ddi_swap32(inl(port))) == 0xffffffff)
1480 fail = 1;
1481 } else {
1482 for (; repcount; repcount--)
1483 if ((*h++ = ddi_swap32(inl(port))) == 0xffffffff)
1484 fail = 1;
1485 }
1486 if (fail == 1)
1487 do_scan(hdlp);
1488 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1489 }
1490
1491 /*ARGSUSED*/
1492 void
i_ddi_prot_vaddr_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1493 i_ddi_prot_vaddr_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1494 uint64_t *dev_addr, size_t repcount, uint_t flags)
1495 {
1496 int fail = 0;
1497 uint64_t *h, *d;
1498
1499 h = host_addr;
1500 d = dev_addr;
1501
1502 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1503 if (flags == DDI_DEV_AUTOINCR) {
1504 for (; repcount; repcount--)
1505 if ((*h++ = *d++) == 0xffffffffffffffff)
1506 fail = 1;
1507 } else {
1508 for (; repcount; repcount--)
1509 if ((*h++ = *d) == 0xffffffffffffffff)
1510 fail = 1;
1511 }
1512 if (fail == 1)
1513 do_scan(hdlp);
1514 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1515 }
1516
1517 /*ARGSUSED*/
1518 void
i_ddi_prot_vaddr_swap_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1519 i_ddi_prot_vaddr_swap_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1520 uint64_t *dev_addr, size_t repcount, uint_t flags)
1521 {
1522 int fail = 0;
1523 uint64_t *h, *d;
1524
1525 h = host_addr;
1526 d = dev_addr;
1527
1528 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1529 if (flags == DDI_DEV_AUTOINCR) {
1530 for (; repcount; repcount--)
1531 if ((*h++ = ddi_swap64(*d++)) == 0xffffffffffffffff)
1532 fail = 1;
1533 } else {
1534 for (; repcount; repcount--)
1535 if ((*h++ = ddi_swap64(*d)) == 0xffffffffffffffff)
1536 fail = 1;
1537 }
1538 if (fail == 1)
1539 do_scan(hdlp);
1540 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1541 }
1542
1543 /*ARGSUSED*/
1544 void
i_ddi_prot_vaddr_rep_put8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1545 i_ddi_prot_vaddr_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1546 uint8_t *dev_addr, size_t repcount, uint_t flags)
1547 {
1548 uint8_t *h, *d;
1549
1550 h = host_addr;
1551 d = dev_addr;
1552
1553 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1554 if (flags == DDI_DEV_AUTOINCR)
1555 for (; repcount; repcount--)
1556 *d++ = *h++;
1557 else
1558 for (; repcount; repcount--)
1559 *d = *h++;
1560 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1561 }
1562
1563 /*ARGSUSED*/
1564 void
i_ddi_prot_io_rep_put8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1565 i_ddi_prot_io_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1566 uint8_t *dev_addr, size_t repcount, uint_t flags)
1567 {
1568 uint8_t *h;
1569 uintptr_t port;
1570
1571 h = host_addr;
1572 port = (uintptr_t)dev_addr;
1573
1574 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1575 if (flags == DDI_DEV_AUTOINCR)
1576 for (; repcount; repcount--, port++)
1577 outb(port, *h++);
1578 else
1579 for (; repcount; repcount--)
1580 outb(port, *h++);
1581 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1582 }
1583
1584 /*ARGSUSED*/
1585 void
i_ddi_prot_vaddr_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1586 i_ddi_prot_vaddr_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1587 uint16_t *dev_addr, size_t repcount, uint_t flags)
1588 {
1589 uint16_t *h, *d;
1590
1591 h = host_addr;
1592 d = dev_addr;
1593
1594 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1595 if (flags == DDI_DEV_AUTOINCR)
1596 for (; repcount; repcount--)
1597 *d++ = *h++;
1598 else
1599 for (; repcount; repcount--)
1600 *d = *h++;
1601 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1602 }
1603
1604 /*ARGSUSED*/
1605 void
i_ddi_prot_io_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1606 i_ddi_prot_io_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1607 uint16_t *dev_addr, size_t repcount, uint_t flags)
1608 {
1609 uint16_t *h;
1610 uintptr_t port;
1611
1612 h = host_addr;
1613 port = (uintptr_t)dev_addr;
1614
1615 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1616 if (flags == DDI_DEV_AUTOINCR)
1617 for (; repcount; repcount--, port += 2)
1618 outw(port, *h++);
1619 else
1620 for (; repcount; repcount--)
1621 outw(port, *h++);
1622 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1623 }
1624
1625 /*ARGSUSED*/
1626 void
i_ddi_prot_vaddr_swap_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1627 i_ddi_prot_vaddr_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1628 uint16_t *dev_addr, size_t repcount, uint_t flags)
1629 {
1630 uint16_t *h, *d;
1631
1632 h = host_addr;
1633 d = dev_addr;
1634
1635 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1636 if (flags == DDI_DEV_AUTOINCR)
1637 for (; repcount; repcount--)
1638 *d++ = ddi_swap16(*h++);
1639 else
1640 for (; repcount; repcount--)
1641 *d = ddi_swap16(*h++);
1642 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1643 }
1644
1645 /*ARGSUSED*/
1646 void
i_ddi_prot_io_swap_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1647 i_ddi_prot_io_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1648 uint16_t *dev_addr, size_t repcount, uint_t flags)
1649 {
1650 uint16_t *h;
1651 uintptr_t port;
1652
1653 h = host_addr;
1654 port = (uintptr_t)dev_addr;
1655
1656 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1657 if (flags == DDI_DEV_AUTOINCR)
1658 for (; repcount; repcount--, port += 2)
1659 outw(port, ddi_swap16(*h++));
1660 else
1661 for (; repcount; repcount--)
1662 outw(port, ddi_swap16(*h++));
1663 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1664 }
1665
1666 /*ARGSUSED*/
1667 void
i_ddi_prot_vaddr_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1668 i_ddi_prot_vaddr_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1669 uint32_t *dev_addr, size_t repcount, uint_t flags)
1670 {
1671 uint32_t *h, *d;
1672
1673 h = host_addr;
1674 d = dev_addr;
1675
1676 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1677 if (flags == DDI_DEV_AUTOINCR)
1678 for (; repcount; repcount--)
1679 *d++ = *h++;
1680 else
1681 for (; repcount; repcount--)
1682 *d = *h++;
1683 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1684 }
1685
1686 /*ARGSUSED*/
1687 void
i_ddi_prot_io_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1688 i_ddi_prot_io_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1689 uint32_t *dev_addr, size_t repcount, uint_t flags)
1690 {
1691 uint32_t *h;
1692 uintptr_t port;
1693
1694 h = host_addr;
1695 port = (uintptr_t)dev_addr;
1696
1697 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1698 if (flags == DDI_DEV_AUTOINCR)
1699 for (; repcount; repcount--, port += 4)
1700 outl(port, *h++);
1701 else
1702 for (; repcount; repcount--)
1703 outl(port, *h++);
1704 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1705 }
1706
1707 /*ARGSUSED*/
1708 void
i_ddi_prot_vaddr_swap_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1709 i_ddi_prot_vaddr_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1710 uint32_t *dev_addr, size_t repcount, uint_t flags)
1711 {
1712 uint32_t *h, *d;
1713
1714 h = host_addr;
1715 d = dev_addr;
1716
1717 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1718 if (flags == DDI_DEV_AUTOINCR)
1719 for (; repcount; repcount--)
1720 *d++ = ddi_swap32(*h++);
1721 else
1722 for (; repcount; repcount--)
1723 *d = ddi_swap32(*h++);
1724 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1725 }
1726
1727 /*ARGSUSED*/
1728 void
i_ddi_prot_io_swap_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1729 i_ddi_prot_io_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1730 uint32_t *dev_addr, size_t repcount, uint_t flags)
1731 {
1732 uint32_t *h;
1733 uintptr_t port;
1734
1735 h = host_addr;
1736 port = (uintptr_t)dev_addr;
1737
1738 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1739 if (flags == DDI_DEV_AUTOINCR)
1740 for (; repcount; repcount--, port += 4)
1741 outl(port, ddi_swap32(*h++));
1742 else
1743 for (; repcount; repcount--)
1744 outl(port, ddi_swap32(*h++));
1745 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1746 }
1747
1748 /*ARGSUSED*/
1749 void
i_ddi_prot_vaddr_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1750 i_ddi_prot_vaddr_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1751 uint64_t *dev_addr, size_t repcount, uint_t flags)
1752 {
1753 uint64_t *h, *d;
1754
1755 h = host_addr;
1756 d = dev_addr;
1757
1758 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1759 if (flags == DDI_DEV_AUTOINCR)
1760 for (; repcount; repcount--)
1761 *d++ = *h++;
1762 else
1763 for (; repcount; repcount--)
1764 *d = *h++;
1765 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1766 }
1767
1768 /*ARGSUSED*/
1769 void
i_ddi_prot_vaddr_swap_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1770 i_ddi_prot_vaddr_swap_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1771 uint64_t *dev_addr, size_t repcount, uint_t flags)
1772 {
1773 uint64_t *h, *d;
1774
1775 h = host_addr;
1776 d = dev_addr;
1777
1778 mutex_enter(hdlp->ahi_peekpoke_mutexp);
1779 if (flags == DDI_DEV_AUTOINCR)
1780 for (; repcount; repcount--)
1781 *d++ = ddi_swap64(*h++);
1782 else
1783 for (; repcount; repcount--)
1784 *d = ddi_swap64(*h++);
1785 mutex_exit(hdlp->ahi_peekpoke_mutexp);
1786 }
1787
1788 void
ddi_io_rep_get8(ddi_acc_handle_t handle,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount)1789 ddi_io_rep_get8(ddi_acc_handle_t handle,
1790 uint8_t *host_addr, uint8_t *dev_addr, size_t repcount)
1791 {
1792 (((ddi_acc_impl_t *)handle)->ahi_rep_get8)
1793 ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1794 repcount, DDI_DEV_NO_AUTOINCR);
1795 }
1796
1797 void
ddi_io_rep_get16(ddi_acc_handle_t handle,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount)1798 ddi_io_rep_get16(ddi_acc_handle_t handle,
1799 uint16_t *host_addr, uint16_t *dev_addr, size_t repcount)
1800 {
1801 (((ddi_acc_impl_t *)handle)->ahi_rep_get16)
1802 ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1803 repcount, DDI_DEV_NO_AUTOINCR);
1804 }
1805
1806 void
ddi_io_rep_get32(ddi_acc_handle_t handle,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount)1807 ddi_io_rep_get32(ddi_acc_handle_t handle,
1808 uint32_t *host_addr, uint32_t *dev_addr, size_t repcount)
1809 {
1810 (((ddi_acc_impl_t *)handle)->ahi_rep_get32)
1811 ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1812 repcount, DDI_DEV_NO_AUTOINCR);
1813 }
1814
1815 /*ARGSUSED*/
1816 void
i_ddi_io_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1817 i_ddi_io_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1818 uint64_t *dev_addr, size_t repcount, uint_t flags)
1819 {
1820 cmn_err(CE_PANIC, "ddi_rep_get64 from i/o space");
1821 }
1822
1823 void
ddi_io_rep_put8(ddi_acc_handle_t handle,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount)1824 ddi_io_rep_put8(ddi_acc_handle_t handle,
1825 uint8_t *host_addr, uint8_t *dev_addr, size_t repcount)
1826 {
1827 (((ddi_acc_impl_t *)handle)->ahi_rep_put8)
1828 ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1829 repcount, DDI_DEV_NO_AUTOINCR);
1830 }
1831
1832 void
ddi_io_rep_put16(ddi_acc_handle_t handle,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount)1833 ddi_io_rep_put16(ddi_acc_handle_t handle,
1834 uint16_t *host_addr, uint16_t *dev_addr, size_t repcount)
1835 {
1836 (((ddi_acc_impl_t *)handle)->ahi_rep_put16)
1837 ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1838 repcount, DDI_DEV_NO_AUTOINCR);
1839 }
1840
1841 void
ddi_io_rep_put32(ddi_acc_handle_t handle,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount)1842 ddi_io_rep_put32(ddi_acc_handle_t handle,
1843 uint32_t *host_addr, uint32_t *dev_addr, size_t repcount)
1844 {
1845 (((ddi_acc_impl_t *)handle)->ahi_rep_put32)
1846 ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1847 repcount, DDI_DEV_NO_AUTOINCR);
1848 }
1849
1850 /*ARGSUSED*/
1851 void
i_ddi_io_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1852 i_ddi_io_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1853 uint64_t *dev_addr, size_t repcount, uint_t flags)
1854 {
1855 cmn_err(CE_PANIC, "ddi_rep_put64 to i/o space");
1856 }
1857
1858 /*
1859 * These next two functions could be translated into assembler someday
1860 */
1861 int
ddi_check_acc_handle(ddi_acc_handle_t handle)1862 ddi_check_acc_handle(ddi_acc_handle_t handle)
1863 {
1864 ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;
1865 return (((*hdlp->ahi_fault_check)(hdlp) == DDI_SUCCESS) ? DDI_SUCCESS :
1866 DDI_FAILURE);
1867 }
1868
1869 int
i_ddi_acc_fault_check(ddi_acc_impl_t * hdlp)1870 i_ddi_acc_fault_check(ddi_acc_impl_t *hdlp)
1871 {
1872 /* Default version, just returns flag value */
1873 return (hdlp->ahi_fault);
1874 }
1875
1876 /*ARGSUSED*/
1877 void
i_ddi_acc_fault_notify(ddi_acc_impl_t * hdlp)1878 i_ddi_acc_fault_notify(ddi_acc_impl_t *hdlp)
1879 {
1880 /* Default version, does nothing for now */
1881 }
1882
1883 void
i_ddi_acc_set_fault(ddi_acc_handle_t handle)1884 i_ddi_acc_set_fault(ddi_acc_handle_t handle)
1885 {
1886 ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;
1887
1888 if (!hdlp->ahi_fault) {
1889 hdlp->ahi_fault = 1;
1890 (*hdlp->ahi_fault_notify)(hdlp);
1891 }
1892 }
1893
1894 void
i_ddi_acc_clr_fault(ddi_acc_handle_t handle)1895 i_ddi_acc_clr_fault(ddi_acc_handle_t handle)
1896 {
1897 ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;
1898
1899 if (hdlp->ahi_fault) {
1900 hdlp->ahi_fault = 0;
1901 (*hdlp->ahi_fault_notify)(hdlp);
1902 }
1903 }
1904