1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-ClauseE
3 *
4 * Copyright (c) KATO Takenori, 1999.
5 *
6 * All rights reserved. Unpublished rights reserved under the copyright
7 * laws of Japan.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer as
15 * the first lines of this file unmodified.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 * 3. The name of the author may not be used to endorse or promote products
20 * derived from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
27 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
31 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 /* $NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $ */
35
36 /*-
37 * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc.
38 * All rights reserved.
39 *
40 * This code is derived from software contributed to The NetBSD Foundation
41 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
42 * NASA Ames Research Center.
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
63 * POSSIBILITY OF SUCH DAMAGE.
64 */
65
66 /*-
67 * Copyright (c) 1996 Charles M. Hannum. All rights reserved.
68 * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved.
69 *
70 * Redistribution and use in source and binary forms, with or without
71 * modification, are permitted provided that the following conditions
72 * are met:
73 * 1. Redistributions of source code must retain the above copyright
74 * notice, this list of conditions and the following disclaimer.
75 * 2. Redistributions in binary form must reproduce the above copyright
76 * notice, this list of conditions and the following disclaimer in the
77 * documentation and/or other materials provided with the distribution.
78 * 3. All advertising materials mentioning features or use of this software
79 * must display the following acknowledgement:
80 * This product includes software developed by Christopher G. Demetriou
81 * for the NetBSD Project.
82 * 4. The name of the author may not be used to endorse or promote products
83 * derived from this software without specific prior written permission
84 *
85 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
86 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
87 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
88 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
89 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
90 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
91 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
92 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
93 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
94 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
95 */
96
97 #ifndef _MACHINE_BUS_H_
98 #define _MACHINE_BUS_H_
99
100 #include <machine/_bus.h>
101 #include <machine/cpufunc.h>
102 #include <machine/bus_dma.h>
103
104 /*
105 * Values for the x86 bus space tag, not to be used directly by MI code.
106 */
107 #define X86_BUS_SPACE_IO 0 /* space is i/o space */
108 #define X86_BUS_SPACE_MEM 1 /* space is mem space */
109
110 #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF
111 #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF
112 #if defined(__amd64__)
113 #define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFULL
114 #else
115 #define BUS_SPACE_MAXSIZE 0xFFFFFFFF
116 #endif
117 #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF
118 #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF
119 #if defined(__amd64__) || defined(PAE)
120 #define BUS_SPACE_MAXADDR_48BIT 0xFFFFFFFFFFFFULL
121 #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL
122 #else
123 #define BUS_SPACE_MAXADDR 0xFFFFFFFF
124 #endif
125
126 #define BUS_SPACE_INVALID_DATA (~0)
127 #define BUS_SPACE_UNRESTRICTED (~0)
128
129 #define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */
130 #define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */
131
132 #if defined(SAN_NEEDS_INTERCEPTORS) && !defined(SAN_RUNTIME)
133 #include <sys/bus_san.h>
134 #else
135
136 /*
137 * Map a region of device bus space into CPU virtual address space.
138 */
139
140 int bus_space_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size,
141 int flags, bus_space_handle_t *bshp);
142
143 /*
144 * Unmap a region of device bus space.
145 */
146
147 void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t bsh,
148 bus_size_t size);
149
150 /*
151 * Get a new handle for a subregion of an already-mapped area of bus space.
152 */
153
154 static __inline int bus_space_subregion(bus_space_tag_t t,
155 bus_space_handle_t bsh,
156 bus_size_t offset, bus_size_t size,
157 bus_space_handle_t *nbshp);
158
159 static __inline int
bus_space_subregion(bus_space_tag_t t __unused,bus_space_handle_t bsh,bus_size_t offset,bus_size_t size __unused,bus_space_handle_t * nbshp)160 bus_space_subregion(bus_space_tag_t t __unused, bus_space_handle_t bsh,
161 bus_size_t offset, bus_size_t size __unused,
162 bus_space_handle_t *nbshp)
163 {
164
165 *nbshp = bsh + offset;
166 return (0);
167 }
168
169 /*
170 * Allocate a region of memory that is accessible to devices in bus space.
171 */
172
173 int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart,
174 bus_addr_t rend, bus_size_t size, bus_size_t align,
175 bus_size_t boundary, int flags, bus_addr_t *addrp,
176 bus_space_handle_t *bshp);
177
178 /*
179 * Free a region of bus space accessible memory.
180 */
181
182 static __inline void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh,
183 bus_size_t size);
184
185 static __inline void
bus_space_free(bus_space_tag_t t __unused,bus_space_handle_t bsh __unused,bus_size_t size __unused)186 bus_space_free(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused,
187 bus_size_t size __unused)
188 {
189 }
190
191 /*
192 * Read a 1, 2, 4, or 8 byte quantity from bus space
193 * described by tag/handle/offset.
194 */
195 static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag,
196 bus_space_handle_t handle,
197 bus_size_t offset);
198
199 static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag,
200 bus_space_handle_t handle,
201 bus_size_t offset);
202
203 static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag,
204 bus_space_handle_t handle,
205 bus_size_t offset);
206
207 #ifdef __amd64__
208 static __inline uint64_t bus_space_read_8(bus_space_tag_t tag,
209 bus_space_handle_t handle,
210 bus_size_t offset);
211 #endif
212
213 static __inline u_int8_t
bus_space_read_1(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)214 bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle,
215 bus_size_t offset)
216 {
217
218 if (tag == X86_BUS_SPACE_IO)
219 return (inb(handle + offset));
220 return (*(volatile u_int8_t *)(handle + offset));
221 }
222
223 static __inline u_int16_t
bus_space_read_2(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)224 bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle,
225 bus_size_t offset)
226 {
227
228 if (tag == X86_BUS_SPACE_IO)
229 return (inw(handle + offset));
230 return (*(volatile u_int16_t *)(handle + offset));
231 }
232
233 static __inline u_int32_t
bus_space_read_4(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)234 bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle,
235 bus_size_t offset)
236 {
237
238 if (tag == X86_BUS_SPACE_IO)
239 return (inl(handle + offset));
240 return (*(volatile u_int32_t *)(handle + offset));
241 }
242
243 #ifdef __amd64__
244 static __inline uint64_t
bus_space_read_8(bus_space_tag_t tag,bus_space_handle_t handle,bus_size_t offset)245 bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
246 bus_size_t offset)
247 {
248
249 if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */
250 return (BUS_SPACE_INVALID_DATA);
251 return (*(volatile uint64_t *)(handle + offset));
252 }
253 #endif
254
255 /*
256 * Read `count' 1, 2, 4, or 8 byte quantities from bus space
257 * described by tag/handle/offset and copy into buffer provided.
258 */
259 static __inline void bus_space_read_multi_1(bus_space_tag_t tag,
260 bus_space_handle_t bsh,
261 bus_size_t offset, u_int8_t *addr,
262 size_t count);
263
264 static __inline void bus_space_read_multi_2(bus_space_tag_t tag,
265 bus_space_handle_t bsh,
266 bus_size_t offset, u_int16_t *addr,
267 size_t count);
268
269 static __inline void bus_space_read_multi_4(bus_space_tag_t tag,
270 bus_space_handle_t bsh,
271 bus_size_t offset, u_int32_t *addr,
272 size_t count);
273
274 static __inline void
bus_space_read_multi_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t * addr,size_t count)275 bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
276 bus_size_t offset, u_int8_t *addr, size_t count)
277 {
278
279 if (tag == X86_BUS_SPACE_IO)
280 insb(bsh + offset, addr, count);
281 else {
282 __asm __volatile(" \n\
283 1: movb (%2),%%al \n\
284 stosb \n\
285 loop 1b" :
286 "=D" (addr), "=c" (count) :
287 "r" (bsh + offset), "0" (addr), "1" (count) :
288 "%eax", "memory");
289 }
290 }
291
292 static __inline void
bus_space_read_multi_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t * addr,size_t count)293 bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
294 bus_size_t offset, u_int16_t *addr, size_t count)
295 {
296
297 if (tag == X86_BUS_SPACE_IO)
298 insw(bsh + offset, addr, count);
299 else {
300 __asm __volatile(" \n\
301 1: movw (%2),%%ax \n\
302 stosw \n\
303 loop 1b" :
304 "=D" (addr), "=c" (count) :
305 "r" (bsh + offset), "0" (addr), "1" (count) :
306 "%eax", "memory");
307 }
308 }
309
310 static __inline void
bus_space_read_multi_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t * addr,size_t count)311 bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
312 bus_size_t offset, u_int32_t *addr, size_t count)
313 {
314
315 if (tag == X86_BUS_SPACE_IO)
316 insl(bsh + offset, addr, count);
317 else {
318 __asm __volatile(" \n\
319 1: movl (%2),%%eax \n\
320 stosl \n\
321 loop 1b" :
322 "=D" (addr), "=c" (count) :
323 "r" (bsh + offset), "0" (addr), "1" (count) :
324 "%eax", "memory");
325 }
326 }
327
328 #if 0 /* Cause a link error for bus_space_read_multi_8 */
329 #define bus_space_read_multi_8 !!! bus_space_read_multi_8 unimplemented !!!
330 #endif
331
332 /*
333 * Read `count' 1, 2, 4, or 8 byte quantities from bus space
334 * described by tag/handle and starting at `offset' and copy into
335 * buffer provided.
336 */
337 static __inline void bus_space_read_region_1(bus_space_tag_t tag,
338 bus_space_handle_t bsh,
339 bus_size_t offset, u_int8_t *addr,
340 size_t count);
341
342 static __inline void bus_space_read_region_2(bus_space_tag_t tag,
343 bus_space_handle_t bsh,
344 bus_size_t offset, u_int16_t *addr,
345 size_t count);
346
347 static __inline void bus_space_read_region_4(bus_space_tag_t tag,
348 bus_space_handle_t bsh,
349 bus_size_t offset, u_int32_t *addr,
350 size_t count);
351
352 static __inline void
bus_space_read_region_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t * addr,size_t count)353 bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
354 bus_size_t offset, u_int8_t *addr, size_t count)
355 {
356
357 if (tag == X86_BUS_SPACE_IO) {
358 int _port_ = bsh + offset;
359 __asm __volatile(" \n\
360 1: inb %w2,%%al \n\
361 stosb \n\
362 incl %2 \n\
363 loop 1b" :
364 "=D" (addr), "=c" (count), "=d" (_port_) :
365 "0" (addr), "1" (count), "2" (_port_) :
366 "%eax", "memory", "cc");
367 } else {
368 bus_space_handle_t _port_ = bsh + offset;
369 __asm __volatile(" \n\
370 repne \n\
371 movsb" :
372 "=D" (addr), "=c" (count), "=S" (_port_) :
373 "0" (addr), "1" (count), "2" (_port_) :
374 "memory", "cc");
375 }
376 }
377
378 static __inline void
bus_space_read_region_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t * addr,size_t count)379 bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
380 bus_size_t offset, u_int16_t *addr, size_t count)
381 {
382
383 if (tag == X86_BUS_SPACE_IO) {
384 int _port_ = bsh + offset;
385 __asm __volatile(" \n\
386 1: inw %w2,%%ax \n\
387 stosw \n\
388 addl $2,%2 \n\
389 loop 1b" :
390 "=D" (addr), "=c" (count), "=d" (_port_) :
391 "0" (addr), "1" (count), "2" (_port_) :
392 "%eax", "memory", "cc");
393 } else {
394 bus_space_handle_t _port_ = bsh + offset;
395 __asm __volatile(" \n\
396 repne \n\
397 movsw" :
398 "=D" (addr), "=c" (count), "=S" (_port_) :
399 "0" (addr), "1" (count), "2" (_port_) :
400 "memory", "cc");
401 }
402 }
403
404 static __inline void
bus_space_read_region_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t * addr,size_t count)405 bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
406 bus_size_t offset, u_int32_t *addr, size_t count)
407 {
408
409 if (tag == X86_BUS_SPACE_IO) {
410 int _port_ = bsh + offset;
411 __asm __volatile(" \n\
412 1: inl %w2,%%eax \n\
413 stosl \n\
414 addl $4,%2 \n\
415 loop 1b" :
416 "=D" (addr), "=c" (count), "=d" (_port_) :
417 "0" (addr), "1" (count), "2" (_port_) :
418 "%eax", "memory", "cc");
419 } else {
420 bus_space_handle_t _port_ = bsh + offset;
421 __asm __volatile(" \n\
422 repne \n\
423 movsl" :
424 "=D" (addr), "=c" (count), "=S" (_port_) :
425 "0" (addr), "1" (count), "2" (_port_) :
426 "memory", "cc");
427 }
428 }
429
430 #if 0 /* Cause a link error for bus_space_read_region_8 */
431 #define bus_space_read_region_8 !!! bus_space_read_region_8 unimplemented !!!
432 #endif
433
434 /*
435 * Write the 1, 2, 4, or 8 byte value `value' to bus space
436 * described by tag/handle/offset.
437 */
438
439 static __inline void bus_space_write_1(bus_space_tag_t tag,
440 bus_space_handle_t bsh,
441 bus_size_t offset, u_int8_t value);
442
443 static __inline void bus_space_write_2(bus_space_tag_t tag,
444 bus_space_handle_t bsh,
445 bus_size_t offset, u_int16_t value);
446
447 static __inline void bus_space_write_4(bus_space_tag_t tag,
448 bus_space_handle_t bsh,
449 bus_size_t offset, u_int32_t value);
450
451 #ifdef __amd64__
452 static __inline void bus_space_write_8(bus_space_tag_t tag,
453 bus_space_handle_t bsh,
454 bus_size_t offset, uint64_t value);
455 #endif
456
457 static __inline void
bus_space_write_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t value)458 bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh,
459 bus_size_t offset, u_int8_t value)
460 {
461
462 if (tag == X86_BUS_SPACE_IO)
463 outb(bsh + offset, value);
464 else
465 *(volatile u_int8_t *)(bsh + offset) = value;
466 }
467
468 static __inline void
bus_space_write_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t value)469 bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh,
470 bus_size_t offset, u_int16_t value)
471 {
472
473 if (tag == X86_BUS_SPACE_IO)
474 outw(bsh + offset, value);
475 else
476 *(volatile u_int16_t *)(bsh + offset) = value;
477 }
478
479 static __inline void
bus_space_write_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t value)480 bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh,
481 bus_size_t offset, u_int32_t value)
482 {
483
484 if (tag == X86_BUS_SPACE_IO)
485 outl(bsh + offset, value);
486 else
487 *(volatile u_int32_t *)(bsh + offset) = value;
488 }
489
490 #ifdef __amd64__
491 static __inline void
bus_space_write_8(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,uint64_t value)492 bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
493 bus_size_t offset, uint64_t value)
494 {
495
496 if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */
497 return;
498 else
499 *(volatile uint64_t *)(bsh + offset) = value;
500 }
501 #endif
502
503 /*
504 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer
505 * provided to bus space described by tag/handle/offset.
506 */
507
508 static __inline void bus_space_write_multi_1(bus_space_tag_t tag,
509 bus_space_handle_t bsh,
510 bus_size_t offset,
511 const u_int8_t *addr,
512 size_t count);
513 static __inline void bus_space_write_multi_2(bus_space_tag_t tag,
514 bus_space_handle_t bsh,
515 bus_size_t offset,
516 const u_int16_t *addr,
517 size_t count);
518
519 static __inline void bus_space_write_multi_4(bus_space_tag_t tag,
520 bus_space_handle_t bsh,
521 bus_size_t offset,
522 const u_int32_t *addr,
523 size_t count);
524
525 static __inline void
bus_space_write_multi_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int8_t * addr,size_t count)526 bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
527 bus_size_t offset, const u_int8_t *addr, size_t count)
528 {
529
530 if (tag == X86_BUS_SPACE_IO)
531 outsb(bsh + offset, addr, count);
532 else {
533 __asm __volatile(" \n\
534 1: lodsb \n\
535 movb %%al,(%2) \n\
536 loop 1b" :
537 "=S" (addr), "=c" (count) :
538 "r" (bsh + offset), "0" (addr), "1" (count) :
539 "%eax", "memory", "cc");
540 }
541 }
542
543 static __inline void
bus_space_write_multi_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int16_t * addr,size_t count)544 bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
545 bus_size_t offset, const u_int16_t *addr, size_t count)
546 {
547
548 if (tag == X86_BUS_SPACE_IO)
549 outsw(bsh + offset, addr, count);
550 else {
551 __asm __volatile(" \n\
552 1: lodsw \n\
553 movw %%ax,(%2) \n\
554 loop 1b" :
555 "=S" (addr), "=c" (count) :
556 "r" (bsh + offset), "0" (addr), "1" (count) :
557 "%eax", "memory", "cc");
558 }
559 }
560
561 static __inline void
bus_space_write_multi_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int32_t * addr,size_t count)562 bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
563 bus_size_t offset, const u_int32_t *addr, size_t count)
564 {
565
566 if (tag == X86_BUS_SPACE_IO)
567 outsl(bsh + offset, addr, count);
568 else {
569 __asm __volatile(" \n\
570 1: lodsl \n\
571 movl %%eax,(%2) \n\
572 loop 1b" :
573 "=S" (addr), "=c" (count) :
574 "r" (bsh + offset), "0" (addr), "1" (count) :
575 "%eax", "memory", "cc");
576 }
577 }
578
579 #if 0 /* Cause a link error for bus_space_write_multi_8 */
580 #define bus_space_write_multi_8(t, h, o, a, c) \
581 !!! bus_space_write_multi_8 unimplemented !!!
582 #endif
583
584 /*
585 * Write `count' 1, 2, 4, or 8 byte quantities from the buffer provided
586 * to bus space described by tag/handle starting at `offset'.
587 */
588
589 static __inline void bus_space_write_region_1(bus_space_tag_t tag,
590 bus_space_handle_t bsh,
591 bus_size_t offset,
592 const u_int8_t *addr,
593 size_t count);
594 static __inline void bus_space_write_region_2(bus_space_tag_t tag,
595 bus_space_handle_t bsh,
596 bus_size_t offset,
597 const u_int16_t *addr,
598 size_t count);
599 static __inline void bus_space_write_region_4(bus_space_tag_t tag,
600 bus_space_handle_t bsh,
601 bus_size_t offset,
602 const u_int32_t *addr,
603 size_t count);
604
605 static __inline void
bus_space_write_region_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int8_t * addr,size_t count)606 bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
607 bus_size_t offset, const u_int8_t *addr, size_t count)
608 {
609
610 if (tag == X86_BUS_SPACE_IO) {
611 int _port_ = bsh + offset;
612 __asm __volatile(" \n\
613 1: lodsb \n\
614 outb %%al,%w0 \n\
615 incl %0 \n\
616 loop 1b" :
617 "=d" (_port_), "=S" (addr), "=c" (count) :
618 "0" (_port_), "1" (addr), "2" (count) :
619 "%eax", "memory", "cc");
620 } else {
621 bus_space_handle_t _port_ = bsh + offset;
622 __asm __volatile(" \n\
623 repne \n\
624 movsb" :
625 "=D" (_port_), "=S" (addr), "=c" (count) :
626 "0" (_port_), "1" (addr), "2" (count) :
627 "memory", "cc");
628 }
629 }
630
631 static __inline void
bus_space_write_region_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int16_t * addr,size_t count)632 bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
633 bus_size_t offset, const u_int16_t *addr, size_t count)
634 {
635
636 if (tag == X86_BUS_SPACE_IO) {
637 int _port_ = bsh + offset;
638 __asm __volatile(" \n\
639 1: lodsw \n\
640 outw %%ax,%w0 \n\
641 addl $2,%0 \n\
642 loop 1b" :
643 "=d" (_port_), "=S" (addr), "=c" (count) :
644 "0" (_port_), "1" (addr), "2" (count) :
645 "%eax", "memory", "cc");
646 } else {
647 bus_space_handle_t _port_ = bsh + offset;
648 __asm __volatile(" \n\
649 repne \n\
650 movsw" :
651 "=D" (_port_), "=S" (addr), "=c" (count) :
652 "0" (_port_), "1" (addr), "2" (count) :
653 "memory", "cc");
654 }
655 }
656
657 static __inline void
bus_space_write_region_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,const u_int32_t * addr,size_t count)658 bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
659 bus_size_t offset, const u_int32_t *addr, size_t count)
660 {
661
662 if (tag == X86_BUS_SPACE_IO) {
663 int _port_ = bsh + offset;
664 __asm __volatile(" \n\
665 1: lodsl \n\
666 outl %%eax,%w0 \n\
667 addl $4,%0 \n\
668 loop 1b" :
669 "=d" (_port_), "=S" (addr), "=c" (count) :
670 "0" (_port_), "1" (addr), "2" (count) :
671 "%eax", "memory", "cc");
672 } else {
673 bus_space_handle_t _port_ = bsh + offset;
674 __asm __volatile(" \n\
675 repne \n\
676 movsl" :
677 "=D" (_port_), "=S" (addr), "=c" (count) :
678 "0" (_port_), "1" (addr), "2" (count) :
679 "memory", "cc");
680 }
681 }
682
683 #if 0 /* Cause a link error for bus_space_write_region_8 */
684 #define bus_space_write_region_8 \
685 !!! bus_space_write_region_8 unimplemented !!!
686 #endif
687
688 /*
689 * Write the 1, 2, 4, or 8 byte value `val' to bus space described
690 * by tag/handle/offset `count' times.
691 */
692
693 static __inline void bus_space_set_multi_1(bus_space_tag_t tag,
694 bus_space_handle_t bsh,
695 bus_size_t offset,
696 u_int8_t value, size_t count);
697 static __inline void bus_space_set_multi_2(bus_space_tag_t tag,
698 bus_space_handle_t bsh,
699 bus_size_t offset,
700 u_int16_t value, size_t count);
701 static __inline void bus_space_set_multi_4(bus_space_tag_t tag,
702 bus_space_handle_t bsh,
703 bus_size_t offset,
704 u_int32_t value, size_t count);
705
706 static __inline void
bus_space_set_multi_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t value,size_t count)707 bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh,
708 bus_size_t offset, u_int8_t value, size_t count)
709 {
710 bus_space_handle_t addr = bsh + offset;
711
712 if (tag == X86_BUS_SPACE_IO)
713 while (count--)
714 outb(addr, value);
715 else
716 while (count--)
717 *(volatile u_int8_t *)(addr) = value;
718 }
719
720 static __inline void
bus_space_set_multi_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t value,size_t count)721 bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh,
722 bus_size_t offset, u_int16_t value, size_t count)
723 {
724 bus_space_handle_t addr = bsh + offset;
725
726 if (tag == X86_BUS_SPACE_IO)
727 while (count--)
728 outw(addr, value);
729 else
730 while (count--)
731 *(volatile u_int16_t *)(addr) = value;
732 }
733
734 static __inline void
bus_space_set_multi_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t value,size_t count)735 bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh,
736 bus_size_t offset, u_int32_t value, size_t count)
737 {
738 bus_space_handle_t addr = bsh + offset;
739
740 if (tag == X86_BUS_SPACE_IO)
741 while (count--)
742 outl(addr, value);
743 else
744 while (count--)
745 *(volatile u_int32_t *)(addr) = value;
746 }
747
748 #if 0 /* Cause a link error for bus_space_set_multi_8 */
749 #define bus_space_set_multi_8 !!! bus_space_set_multi_8 unimplemented !!!
750 #endif
751
752 /*
753 * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described
754 * by tag/handle starting at `offset'.
755 */
756
757 static __inline void bus_space_set_region_1(bus_space_tag_t tag,
758 bus_space_handle_t bsh,
759 bus_size_t offset, u_int8_t value,
760 size_t count);
761 static __inline void bus_space_set_region_2(bus_space_tag_t tag,
762 bus_space_handle_t bsh,
763 bus_size_t offset, u_int16_t value,
764 size_t count);
765 static __inline void bus_space_set_region_4(bus_space_tag_t tag,
766 bus_space_handle_t bsh,
767 bus_size_t offset, u_int32_t value,
768 size_t count);
769
770 static __inline void
bus_space_set_region_1(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int8_t value,size_t count)771 bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh,
772 bus_size_t offset, u_int8_t value, size_t count)
773 {
774 bus_space_handle_t addr = bsh + offset;
775
776 if (tag == X86_BUS_SPACE_IO)
777 for (; count != 0; count--, addr++)
778 outb(addr, value);
779 else
780 for (; count != 0; count--, addr++)
781 *(volatile u_int8_t *)(addr) = value;
782 }
783
784 static __inline void
bus_space_set_region_2(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int16_t value,size_t count)785 bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh,
786 bus_size_t offset, u_int16_t value, size_t count)
787 {
788 bus_space_handle_t addr = bsh + offset;
789
790 if (tag == X86_BUS_SPACE_IO)
791 for (; count != 0; count--, addr += 2)
792 outw(addr, value);
793 else
794 for (; count != 0; count--, addr += 2)
795 *(volatile u_int16_t *)(addr) = value;
796 }
797
798 static __inline void
bus_space_set_region_4(bus_space_tag_t tag,bus_space_handle_t bsh,bus_size_t offset,u_int32_t value,size_t count)799 bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh,
800 bus_size_t offset, u_int32_t value, size_t count)
801 {
802 bus_space_handle_t addr = bsh + offset;
803
804 if (tag == X86_BUS_SPACE_IO)
805 for (; count != 0; count--, addr += 4)
806 outl(addr, value);
807 else
808 for (; count != 0; count--, addr += 4)
809 *(volatile u_int32_t *)(addr) = value;
810 }
811
812 #if 0 /* Cause a link error for bus_space_set_region_8 */
813 #define bus_space_set_region_8 !!! bus_space_set_region_8 unimplemented !!!
814 #endif
815
816 /*
817 * Copy `count' 1, 2, 4, or 8 byte values from bus space starting
818 * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2.
819 */
820
821 static __inline void bus_space_copy_region_1(bus_space_tag_t tag,
822 bus_space_handle_t bsh1,
823 bus_size_t off1,
824 bus_space_handle_t bsh2,
825 bus_size_t off2, size_t count);
826
827 static __inline void bus_space_copy_region_2(bus_space_tag_t tag,
828 bus_space_handle_t bsh1,
829 bus_size_t off1,
830 bus_space_handle_t bsh2,
831 bus_size_t off2, size_t count);
832
833 static __inline void bus_space_copy_region_4(bus_space_tag_t tag,
834 bus_space_handle_t bsh1,
835 bus_size_t off1,
836 bus_space_handle_t bsh2,
837 bus_size_t off2, size_t count);
838
839 static __inline void
bus_space_copy_region_1(bus_space_tag_t tag,bus_space_handle_t bsh1,bus_size_t off1,bus_space_handle_t bsh2,bus_size_t off2,size_t count)840 bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1,
841 bus_size_t off1, bus_space_handle_t bsh2,
842 bus_size_t off2, size_t count)
843 {
844 bus_space_handle_t addr1 = bsh1 + off1;
845 bus_space_handle_t addr2 = bsh2 + off2;
846
847 if (tag == X86_BUS_SPACE_IO) {
848 if (addr1 >= addr2) {
849 /* src after dest: copy forward */
850 for (; count != 0; count--, addr1++, addr2++)
851 outb(addr2, inb(addr1));
852 } else {
853 /* dest after src: copy backwards */
854 for (addr1 += (count - 1), addr2 += (count - 1);
855 count != 0; count--, addr1--, addr2--)
856 outb(addr2, inb(addr1));
857 }
858 } else {
859 if (addr1 >= addr2) {
860 /* src after dest: copy forward */
861 for (; count != 0; count--, addr1++, addr2++)
862 *(volatile u_int8_t *)(addr2) =
863 *(volatile u_int8_t *)(addr1);
864 } else {
865 /* dest after src: copy backwards */
866 for (addr1 += (count - 1), addr2 += (count - 1);
867 count != 0; count--, addr1--, addr2--)
868 *(volatile u_int8_t *)(addr2) =
869 *(volatile u_int8_t *)(addr1);
870 }
871 }
872 }
873
874 static __inline void
bus_space_copy_region_2(bus_space_tag_t tag,bus_space_handle_t bsh1,bus_size_t off1,bus_space_handle_t bsh2,bus_size_t off2,size_t count)875 bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1,
876 bus_size_t off1, bus_space_handle_t bsh2,
877 bus_size_t off2, size_t count)
878 {
879 bus_space_handle_t addr1 = bsh1 + off1;
880 bus_space_handle_t addr2 = bsh2 + off2;
881
882 if (tag == X86_BUS_SPACE_IO) {
883 if (addr1 >= addr2) {
884 /* src after dest: copy forward */
885 for (; count != 0; count--, addr1 += 2, addr2 += 2)
886 outw(addr2, inw(addr1));
887 } else {
888 /* dest after src: copy backwards */
889 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1);
890 count != 0; count--, addr1 -= 2, addr2 -= 2)
891 outw(addr2, inw(addr1));
892 }
893 } else {
894 if (addr1 >= addr2) {
895 /* src after dest: copy forward */
896 for (; count != 0; count--, addr1 += 2, addr2 += 2)
897 *(volatile u_int16_t *)(addr2) =
898 *(volatile u_int16_t *)(addr1);
899 } else {
900 /* dest after src: copy backwards */
901 for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1);
902 count != 0; count--, addr1 -= 2, addr2 -= 2)
903 *(volatile u_int16_t *)(addr2) =
904 *(volatile u_int16_t *)(addr1);
905 }
906 }
907 }
908
909 static __inline void
bus_space_copy_region_4(bus_space_tag_t tag,bus_space_handle_t bsh1,bus_size_t off1,bus_space_handle_t bsh2,bus_size_t off2,size_t count)910 bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1,
911 bus_size_t off1, bus_space_handle_t bsh2,
912 bus_size_t off2, size_t count)
913 {
914 bus_space_handle_t addr1 = bsh1 + off1;
915 bus_space_handle_t addr2 = bsh2 + off2;
916
917 if (tag == X86_BUS_SPACE_IO) {
918 if (addr1 >= addr2) {
919 /* src after dest: copy forward */
920 for (; count != 0; count--, addr1 += 4, addr2 += 4)
921 outl(addr2, inl(addr1));
922 } else {
923 /* dest after src: copy backwards */
924 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1);
925 count != 0; count--, addr1 -= 4, addr2 -= 4)
926 outl(addr2, inl(addr1));
927 }
928 } else {
929 if (addr1 >= addr2) {
930 /* src after dest: copy forward */
931 for (; count != 0; count--, addr1 += 4, addr2 += 4)
932 *(volatile u_int32_t *)(addr2) =
933 *(volatile u_int32_t *)(addr1);
934 } else {
935 /* dest after src: copy backwards */
936 for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1);
937 count != 0; count--, addr1 -= 4, addr2 -= 4)
938 *(volatile u_int32_t *)(addr2) =
939 *(volatile u_int32_t *)(addr1);
940 }
941 }
942 }
943
944 #if 0 /* Cause a link error for bus_space_copy_8 */
945 #define bus_space_copy_region_8 !!! bus_space_copy_region_8 unimplemented !!!
946 #endif
947
948 /*
949 * Bus read/write barrier methods.
950 *
951 * void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh,
952 * bus_size_t offset, bus_size_t len, int flags);
953 *
954 *
955 * Note that BUS_SPACE_BARRIER_WRITE doesn't do anything other than
956 * prevent reordering by the compiler; all Intel x86 processors currently
957 * retire operations outside the CPU in program order.
958 */
959 static __inline void
bus_space_barrier(bus_space_tag_t tag __unused,bus_space_handle_t bsh __unused,bus_size_t offset __unused,bus_size_t len __unused,int flags)960 bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused,
961 bus_size_t offset __unused, bus_size_t len __unused, int flags)
962 {
963 if (flags & BUS_SPACE_BARRIER_READ)
964 #ifdef __amd64__
965 __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory");
966 #else
967 __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory");
968 #endif
969 else
970 __compiler_membar();
971 }
972
973 #ifdef BUS_SPACE_NO_LEGACY
974 #undef inb
975 #undef outb
976 #define inb(a) compiler_error
977 #define inw(a) compiler_error
978 #define inl(a) compiler_error
979 #define outb(a, b) compiler_error
980 #define outw(a, b) compiler_error
981 #define outl(a, b) compiler_error
982 #endif
983
984 /*
985 * Stream accesses are the same as normal accesses on x86; there are no
986 * supported bus systems with an endianess different from the host one.
987 */
988 #define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o))
989 #define bus_space_read_stream_2(t, h, o) bus_space_read_2((t), (h), (o))
990 #define bus_space_read_stream_4(t, h, o) bus_space_read_4((t), (h), (o))
991
992 #define bus_space_read_multi_stream_1(t, h, o, a, c) \
993 bus_space_read_multi_1((t), (h), (o), (a), (c))
994 #define bus_space_read_multi_stream_2(t, h, o, a, c) \
995 bus_space_read_multi_2((t), (h), (o), (a), (c))
996 #define bus_space_read_multi_stream_4(t, h, o, a, c) \
997 bus_space_read_multi_4((t), (h), (o), (a), (c))
998
999 #define bus_space_write_stream_1(t, h, o, v) \
1000 bus_space_write_1((t), (h), (o), (v))
1001 #define bus_space_write_stream_2(t, h, o, v) \
1002 bus_space_write_2((t), (h), (o), (v))
1003 #define bus_space_write_stream_4(t, h, o, v) \
1004 bus_space_write_4((t), (h), (o), (v))
1005
1006 #define bus_space_write_multi_stream_1(t, h, o, a, c) \
1007 bus_space_write_multi_1((t), (h), (o), (a), (c))
1008 #define bus_space_write_multi_stream_2(t, h, o, a, c) \
1009 bus_space_write_multi_2((t), (h), (o), (a), (c))
1010 #define bus_space_write_multi_stream_4(t, h, o, a, c) \
1011 bus_space_write_multi_4((t), (h), (o), (a), (c))
1012
1013 #define bus_space_set_multi_stream_1(t, h, o, v, c) \
1014 bus_space_set_multi_1((t), (h), (o), (v), (c))
1015 #define bus_space_set_multi_stream_2(t, h, o, v, c) \
1016 bus_space_set_multi_2((t), (h), (o), (v), (c))
1017 #define bus_space_set_multi_stream_4(t, h, o, v, c) \
1018 bus_space_set_multi_4((t), (h), (o), (v), (c))
1019
1020 #define bus_space_read_region_stream_1(t, h, o, a, c) \
1021 bus_space_read_region_1((t), (h), (o), (a), (c))
1022 #define bus_space_read_region_stream_2(t, h, o, a, c) \
1023 bus_space_read_region_2((t), (h), (o), (a), (c))
1024 #define bus_space_read_region_stream_4(t, h, o, a, c) \
1025 bus_space_read_region_4((t), (h), (o), (a), (c))
1026
1027 #define bus_space_write_region_stream_1(t, h, o, a, c) \
1028 bus_space_write_region_1((t), (h), (o), (a), (c))
1029 #define bus_space_write_region_stream_2(t, h, o, a, c) \
1030 bus_space_write_region_2((t), (h), (o), (a), (c))
1031 #define bus_space_write_region_stream_4(t, h, o, a, c) \
1032 bus_space_write_region_4((t), (h), (o), (a), (c))
1033
1034 #define bus_space_set_region_stream_1(t, h, o, v, c) \
1035 bus_space_set_region_1((t), (h), (o), (v), (c))
1036 #define bus_space_set_region_stream_2(t, h, o, v, c) \
1037 bus_space_set_region_2((t), (h), (o), (v), (c))
1038 #define bus_space_set_region_stream_4(t, h, o, v, c) \
1039 bus_space_set_region_4((t), (h), (o), (v), (c))
1040
1041 #define bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c) \
1042 bus_space_copy_region_1((t), (h1), (o1), (h2), (o2), (c))
1043 #define bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c) \
1044 bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c))
1045 #define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \
1046 bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c))
1047
1048 #define BUS_PEEK_FUNC(width, type) \
1049 static inline int \
1050 bus_space_peek_##width(bus_space_tag_t tag, \
1051 bus_space_handle_t hnd, bus_size_t offset, type *value) \
1052 { \
1053 type tmp; \
1054 tmp = bus_space_read_##width(tag, hnd, offset); \
1055 *value = (type)tmp; \
1056 return (0); \
1057 }
1058 BUS_PEEK_FUNC(1, uint8_t)
1059 BUS_PEEK_FUNC(2, uint16_t)
1060 BUS_PEEK_FUNC(4, uint32_t)
1061 #ifdef __amd64__
1062 BUS_PEEK_FUNC(8, uint64_t)
1063 #endif
1064
1065 #define BUS_POKE_FUNC(width, type) \
1066 static inline int \
1067 bus_space_poke_##width(bus_space_tag_t tag, \
1068 bus_space_handle_t hnd, bus_size_t offset, type value) \
1069 { \
1070 bus_space_write_##width(tag, hnd, offset, value); \
1071 return (0); \
1072 }
1073 BUS_POKE_FUNC(1, uint8_t)
1074 BUS_POKE_FUNC(2, uint16_t)
1075 BUS_POKE_FUNC(4, uint32_t)
1076 #ifdef __amd64__
1077 BUS_POKE_FUNC(8, uint64_t)
1078 #endif
1079
1080 #endif /* !SAN_NEEDS_INTERCEPTORS && SAN_RUNTIME */
1081
1082 #endif /* !_MACHINE_BUS_H_ */
1083