1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #ifndef _SYS_EFSYS_H 33 #define _SYS_EFSYS_H 34 35 #ifdef __cplusplus 36 extern "C" { 37 #endif 38 39 #include <sys/param.h> 40 #include <sys/bus.h> 41 #include <sys/endian.h> 42 #include <sys/lock.h> 43 #include <sys/malloc.h> 44 #include <sys/mbuf.h> 45 #include <sys/mutex.h> 46 #include <sys/rwlock.h> 47 #include <sys/sdt.h> 48 #include <sys/systm.h> 49 50 #include <machine/bus.h> 51 #include <machine/endian.h> 52 53 #define EFSYS_HAS_UINT64 1 54 #define EFSYS_USE_UINT64 0 55 #if _BYTE_ORDER == _BIG_ENDIAN 56 #define EFSYS_IS_BIG_ENDIAN 1 57 #define EFSYS_IS_LITTLE_ENDIAN 0 58 #elif _BYTE_ORDER == _LITTLE_ENDIAN 59 #define EFSYS_IS_BIG_ENDIAN 0 60 #define EFSYS_IS_LITTLE_ENDIAN 1 61 #endif 62 #include "efx_types.h" 63 64 /* Common code requires this */ 65 #if __FreeBSD_version < 800068 66 #define memmove(d, s, l) bcopy(s, d, l) 67 #endif 68 69 /* FreeBSD equivalents of Solaris things */ 70 #ifndef _NOTE 71 #define _NOTE(s) 72 #endif 73 74 #ifndef B_FALSE 75 #define B_FALSE FALSE 76 #endif 77 #ifndef B_TRUE 78 #define B_TRUE TRUE 79 #endif 80 81 #ifndef IS_P2ALIGNED 82 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 83 #endif 84 85 #ifndef P2ROUNDUP 86 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 87 #endif 88 89 #ifndef IS2P 90 #define ISP2(x) (((x) & ((x) - 1)) == 0) 91 #endif 92 93 #define ENOTACTIVE EINVAL 94 95 /* Memory type to use on FreeBSD */ 96 MALLOC_DECLARE(M_SFXGE); 97 98 /* Machine dependend prefetch wrappers */ 99 #if defined(__i386__) || defined(__amd64__) 100 static __inline void 101 prefetch_read_many(void *addr) 102 { 103 104 __asm__( 105 "prefetcht0 (%0)" 106 : 107 : "r" (addr)); 108 } 109 110 static __inline void 111 prefetch_read_once(void *addr) 112 { 113 114 __asm__( 115 "prefetchnta (%0)" 116 : 117 : "r" (addr)); 118 } 119 #elif defined(__sparc64__) 120 static __inline void 121 prefetch_read_many(void *addr) 122 { 123 124 __asm__( 125 "prefetch [%0], 0" 126 : 127 : "r" (addr)); 128 } 129 130 static __inline void 131 prefetch_read_once(void *addr) 132 { 133 134 __asm__( 135 "prefetch [%0], 1" 136 : 137 : "r" (addr)); 138 } 139 #else 140 static __inline void 141 prefetch_read_many(void *addr) 142 { 143 144 } 145 146 static __inline void 147 prefetch_read_once(void *addr) 148 { 149 150 } 151 #endif 152 153 #if defined(__i386__) || defined(__amd64__) 154 #include <vm/vm.h> 155 #include <vm/pmap.h> 156 #endif 157 static __inline void 158 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 159 struct mbuf *m, bus_dma_segment_t *seg) 160 { 161 #if defined(__i386__) || defined(__amd64__) 162 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 163 seg->ds_len = m->m_len; 164 #else 165 int nsegstmp; 166 167 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 168 #endif 169 } 170 171 /* Modifiers used for DOS builds */ 172 #define __cs 173 #define __far 174 175 /* Modifiers used for Windows builds */ 176 #define __in 177 #define __in_opt 178 #define __in_ecount(_n) 179 #define __in_ecount_opt(_n) 180 #define __in_bcount(_n) 181 #define __in_bcount_opt(_n) 182 183 #define __out 184 #define __out_opt 185 #define __out_ecount(_n) 186 #define __out_ecount_opt(_n) 187 #define __out_bcount(_n) 188 #define __out_bcount_opt(_n) 189 190 #define __deref_out 191 192 #define __inout 193 #define __inout_opt 194 #define __inout_ecount(_n) 195 #define __inout_ecount_opt(_n) 196 #define __inout_bcount(_n) 197 #define __inout_bcount_opt(_n) 198 #define __inout_bcount_full_opt(_n) 199 200 #define __deref_out_bcount_opt(n) 201 202 #define __checkReturn 203 204 #define __drv_when(_p, _c) 205 206 /* Code inclusion options */ 207 208 209 #define EFSYS_OPT_NAMES 1 210 211 #define EFSYS_OPT_FALCON 0 212 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 213 #define EFSYS_OPT_SIENA 1 214 #ifdef DEBUG 215 #define EFSYS_OPT_CHECK_REG 1 216 #else 217 #define EFSYS_OPT_CHECK_REG 0 218 #endif 219 220 #define EFSYS_OPT_MCDI 1 221 222 #define EFSYS_OPT_MAC_FALCON_GMAC 0 223 #define EFSYS_OPT_MAC_FALCON_XMAC 0 224 #define EFSYS_OPT_MAC_STATS 1 225 226 #define EFSYS_OPT_LOOPBACK 0 227 228 #define EFSYS_OPT_MON_NULL 0 229 #define EFSYS_OPT_MON_LM87 0 230 #define EFSYS_OPT_MON_MAX6647 0 231 #define EFSYS_OPT_MON_SIENA 0 232 #define EFSYS_OPT_MON_STATS 0 233 234 #define EFSYS_OPT_PHY_NULL 0 235 #define EFSYS_OPT_PHY_QT2022C2 0 236 #define EFSYS_OPT_PHY_SFX7101 0 237 #define EFSYS_OPT_PHY_TXC43128 0 238 #define EFSYS_OPT_PHY_PM8358 0 239 #define EFSYS_OPT_PHY_SFT9001 0 240 #define EFSYS_OPT_PHY_QT2025C 0 241 #define EFSYS_OPT_PHY_STATS 1 242 #define EFSYS_OPT_PHY_PROPS 0 243 #define EFSYS_OPT_PHY_BIST 1 244 #define EFSYS_OPT_PHY_LED_CONTROL 1 245 #define EFSYS_OPT_PHY_FLAGS 0 246 247 #define EFSYS_OPT_VPD 1 248 #define EFSYS_OPT_NVRAM 1 249 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 250 #define EFSYS_OPT_NVRAM_SFT9001 0 251 #define EFSYS_OPT_NVRAM_SFX7101 0 252 #define EFSYS_OPT_BOOTCFG 0 253 254 #define EFSYS_OPT_PCIE_TUNE 0 255 #define EFSYS_OPT_DIAG 0 256 #define EFSYS_OPT_WOL 1 257 #define EFSYS_OPT_RX_SCALE 1 258 #define EFSYS_OPT_QSTATS 1 259 #define EFSYS_OPT_FILTER 0 260 #define EFSYS_OPT_RX_SCATTER 0 261 #define EFSYS_OPT_RX_HDR_SPLIT 0 262 263 #define EFSYS_OPT_EV_PREFETCH 0 264 265 #define EFSYS_OPT_DECODE_INTR_FATAL 1 266 267 /* ID */ 268 269 typedef struct __efsys_identifier_s efsys_identifier_t; 270 271 /* PROBE */ 272 273 #ifndef KDTRACE_HOOKS 274 275 #define EFSYS_PROBE(_name) 276 277 #define EFSYS_PROBE1(_name, _type1, _arg1) 278 279 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 280 281 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 282 _type3, _arg3) 283 284 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 285 _type3, _arg3, _type4, _arg4) 286 287 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 288 _type3, _arg3, _type4, _arg4, _type5, _arg5) 289 290 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 291 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 292 _type6, _arg6) 293 294 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 295 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 296 _type6, _arg6, _type7, _arg7) 297 298 #else /* KDTRACE_HOOKS */ 299 300 #define EFSYS_PROBE(_name) \ 301 DTRACE_PROBE(_name) 302 303 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 304 DTRACE_PROBE1(_name, _type1, _arg1) 305 306 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 307 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 308 309 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 310 _type3, _arg3) \ 311 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 312 _type3, _arg3) 313 314 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 315 _type3, _arg3, _type4, _arg4) \ 316 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 317 _type3, _arg3, _type4, _arg4) 318 319 #ifdef DTRACE_PROBE5 320 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 321 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 322 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 323 _type3, _arg3, _type4, _arg4, _type5, _arg5) 324 #else 325 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 326 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 327 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 328 _type3, _arg3, _type4, _arg4) 329 #endif 330 331 #ifdef DTRACE_PROBE6 332 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 333 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 334 _type6, _arg6) \ 335 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 336 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 337 _type6, _arg6) 338 #else 339 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 340 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 341 _type6, _arg6) \ 342 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 343 _type3, _arg3, _type4, _arg4, _type5, _arg5) 344 #endif 345 346 #ifdef DTRACE_PROBE7 347 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 348 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 349 _type6, _arg6, _type7, _arg7) \ 350 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 351 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 352 _type6, _arg6, _type7, _arg7) 353 #else 354 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 355 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 356 _type6, _arg6, _type7, _arg7) \ 357 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 358 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 359 _type6, _arg6) 360 #endif 361 362 #endif /* KDTRACE_HOOKS */ 363 364 /* DMA */ 365 366 typedef uint64_t efsys_dma_addr_t; 367 368 typedef struct efsys_mem_s { 369 bus_dma_tag_t esm_tag; 370 bus_dmamap_t esm_map; 371 caddr_t esm_base; 372 efsys_dma_addr_t esm_addr; 373 size_t esm_size; 374 } efsys_mem_t; 375 376 377 #define EFSYS_MEM_ZERO(_esmp, _size) \ 378 do { \ 379 (void) memset((_esmp)->esm_base, 0, (_size)); \ 380 \ 381 _NOTE(CONSTANTCONDITION) \ 382 } while (B_FALSE) 383 384 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 385 do { \ 386 uint32_t *addr; \ 387 \ 388 _NOTE(CONSTANTCONDITION) \ 389 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 390 ("not power of 2 aligned")); \ 391 \ 392 addr = (void *)((_esmp)->esm_base + (_offset)); \ 393 \ 394 (_edp)->ed_u32[0] = *addr; \ 395 \ 396 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 397 uint32_t, (_edp)->ed_u32[0]); \ 398 \ 399 _NOTE(CONSTANTCONDITION) \ 400 } while (B_FALSE) 401 402 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 403 do { \ 404 uint32_t *addr; \ 405 \ 406 _NOTE(CONSTANTCONDITION) \ 407 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 408 ("not power of 2 aligned")); \ 409 \ 410 addr = (void *)((_esmp)->esm_base + (_offset)); \ 411 \ 412 (_eqp)->eq_u32[0] = *addr++; \ 413 (_eqp)->eq_u32[1] = *addr; \ 414 \ 415 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 416 uint32_t, (_eqp)->eq_u32[1], \ 417 uint32_t, (_eqp)->eq_u32[0]); \ 418 \ 419 _NOTE(CONSTANTCONDITION) \ 420 } while (B_FALSE) 421 422 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 423 do { \ 424 uint32_t *addr; \ 425 \ 426 _NOTE(CONSTANTCONDITION) \ 427 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 428 ("not power of 2 aligned")); \ 429 \ 430 addr = (void *)((_esmp)->esm_base + (_offset)); \ 431 \ 432 (_eop)->eo_u32[0] = *addr++; \ 433 (_eop)->eo_u32[1] = *addr++; \ 434 (_eop)->eo_u32[2] = *addr++; \ 435 (_eop)->eo_u32[3] = *addr; \ 436 \ 437 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 438 uint32_t, (_eop)->eo_u32[3], \ 439 uint32_t, (_eop)->eo_u32[2], \ 440 uint32_t, (_eop)->eo_u32[1], \ 441 uint32_t, (_eop)->eo_u32[0]); \ 442 \ 443 _NOTE(CONSTANTCONDITION) \ 444 } while (B_FALSE) 445 446 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 447 do { \ 448 uint32_t *addr; \ 449 \ 450 _NOTE(CONSTANTCONDITION) \ 451 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 452 ("not power of 2 aligned")); \ 453 \ 454 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 455 uint32_t, (_edp)->ed_u32[0]); \ 456 \ 457 addr = (void *)((_esmp)->esm_base + (_offset)); \ 458 \ 459 *addr = (_edp)->ed_u32[0]; \ 460 \ 461 _NOTE(CONSTANTCONDITION) \ 462 } while (B_FALSE) 463 464 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 465 do { \ 466 uint32_t *addr; \ 467 \ 468 _NOTE(CONSTANTCONDITION) \ 469 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 470 ("not power of 2 aligned")); \ 471 \ 472 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 473 uint32_t, (_eqp)->eq_u32[1], \ 474 uint32_t, (_eqp)->eq_u32[0]); \ 475 \ 476 addr = (void *)((_esmp)->esm_base + (_offset)); \ 477 \ 478 *addr++ = (_eqp)->eq_u32[0]; \ 479 *addr = (_eqp)->eq_u32[1]; \ 480 \ 481 _NOTE(CONSTANTCONDITION) \ 482 } while (B_FALSE) 483 484 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 485 do { \ 486 uint32_t *addr; \ 487 \ 488 _NOTE(CONSTANTCONDITION) \ 489 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 490 ("not power of 2 aligned")); \ 491 \ 492 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 493 uint32_t, (_eop)->eo_u32[3], \ 494 uint32_t, (_eop)->eo_u32[2], \ 495 uint32_t, (_eop)->eo_u32[1], \ 496 uint32_t, (_eop)->eo_u32[0]); \ 497 \ 498 addr = (void *)((_esmp)->esm_base + (_offset)); \ 499 \ 500 *addr++ = (_eop)->eo_u32[0]; \ 501 *addr++ = (_eop)->eo_u32[1]; \ 502 *addr++ = (_eop)->eo_u32[2]; \ 503 *addr = (_eop)->eo_u32[3]; \ 504 \ 505 _NOTE(CONSTANTCONDITION) \ 506 } while (B_FALSE) 507 508 #define EFSYS_MEM_ADDR(_esmp) \ 509 ((_esmp)->esm_addr) 510 511 /* BAR */ 512 513 typedef struct efsys_bar_s { 514 struct mtx esb_lock; 515 bus_space_tag_t esb_tag; 516 bus_space_handle_t esb_handle; 517 int esb_rid; 518 struct resource *esb_res; 519 } efsys_bar_t; 520 521 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 522 do { \ 523 _NOTE(CONSTANTCONDITION) \ 524 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 525 ("not power of 2 aligned")); \ 526 \ 527 _NOTE(CONSTANTCONDITION) \ 528 if (_lock) \ 529 mtx_lock(&((_esbp)->esb_lock)); \ 530 \ 531 (_edp)->ed_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 532 (_esbp)->esb_handle, (_offset)); \ 533 \ 534 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 535 uint32_t, (_edp)->ed_u32[0]); \ 536 \ 537 _NOTE(CONSTANTCONDITION) \ 538 if (_lock) \ 539 mtx_unlock(&((_esbp)->esb_lock)); \ 540 _NOTE(CONSTANTCONDITION) \ 541 } while (B_FALSE) 542 543 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 544 do { \ 545 _NOTE(CONSTANTCONDITION) \ 546 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 547 ("not power of 2 aligned")); \ 548 \ 549 mtx_lock(&((_esbp)->esb_lock)); \ 550 \ 551 (_eqp)->eq_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 552 (_esbp)->esb_handle, (_offset)); \ 553 (_eqp)->eq_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ 554 (_esbp)->esb_handle, (_offset+4)); \ 555 \ 556 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 557 uint32_t, (_eqp)->eq_u32[1], \ 558 uint32_t, (_eqp)->eq_u32[0]); \ 559 \ 560 mtx_unlock(&((_esbp)->esb_lock)); \ 561 _NOTE(CONSTANTCONDITION) \ 562 } while (B_FALSE) 563 564 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 565 do { \ 566 _NOTE(CONSTANTCONDITION) \ 567 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 568 ("not power of 2 aligned")); \ 569 \ 570 _NOTE(CONSTANTCONDITION) \ 571 if (_lock) \ 572 mtx_lock(&((_esbp)->esb_lock)); \ 573 \ 574 (_eop)->eo_u32[0] = bus_space_read_4((_esbp)->esb_tag, \ 575 (_esbp)->esb_handle, (_offset)); \ 576 (_eop)->eo_u32[1] = bus_space_read_4((_esbp)->esb_tag, \ 577 (_esbp)->esb_handle, (_offset+4)); \ 578 (_eop)->eo_u32[2] = bus_space_read_4((_esbp)->esb_tag, \ 579 (_esbp)->esb_handle, (_offset+8)); \ 580 (_eop)->eo_u32[3] = bus_space_read_4((_esbp)->esb_tag, \ 581 (_esbp)->esb_handle, (_offset+12)); \ 582 \ 583 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 584 uint32_t, (_eop)->eo_u32[3], \ 585 uint32_t, (_eop)->eo_u32[2], \ 586 uint32_t, (_eop)->eo_u32[1], \ 587 uint32_t, (_eop)->eo_u32[0]); \ 588 \ 589 _NOTE(CONSTANTCONDITION) \ 590 if (_lock) \ 591 mtx_unlock(&((_esbp)->esb_lock)); \ 592 _NOTE(CONSTANTCONDITION) \ 593 } while (B_FALSE) 594 595 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 596 do { \ 597 _NOTE(CONSTANTCONDITION) \ 598 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 599 ("not power of 2 aligned")); \ 600 \ 601 _NOTE(CONSTANTCONDITION) \ 602 if (_lock) \ 603 mtx_lock(&((_esbp)->esb_lock)); \ 604 \ 605 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 606 uint32_t, (_edp)->ed_u32[0]); \ 607 \ 608 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 609 (_offset), (_edp)->ed_u32[0]); \ 610 \ 611 _NOTE(CONSTANTCONDITION) \ 612 if (_lock) \ 613 mtx_unlock(&((_esbp)->esb_lock)); \ 614 _NOTE(CONSTANTCONDITION) \ 615 } while (B_FALSE) 616 617 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 618 do { \ 619 _NOTE(CONSTANTCONDITION) \ 620 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 621 ("not power of 2 aligned")); \ 622 \ 623 mtx_lock(&((_esbp)->esb_lock)); \ 624 \ 625 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 626 uint32_t, (_eqp)->eq_u32[1], \ 627 uint32_t, (_eqp)->eq_u32[0]); \ 628 \ 629 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 630 (_offset), (_eqp)->eq_u32[0]); \ 631 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 632 (_offset+4), (_eqp)->eq_u32[1]); \ 633 \ 634 mtx_unlock(&((_esbp)->esb_lock)); \ 635 _NOTE(CONSTANTCONDITION) \ 636 } while (B_FALSE) 637 638 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 639 do { \ 640 _NOTE(CONSTANTCONDITION) \ 641 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 642 ("not power of 2 aligned")); \ 643 \ 644 _NOTE(CONSTANTCONDITION) \ 645 if (_lock) \ 646 mtx_lock(&((_esbp)->esb_lock)); \ 647 \ 648 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 649 uint32_t, (_eop)->eo_u32[3], \ 650 uint32_t, (_eop)->eo_u32[2], \ 651 uint32_t, (_eop)->eo_u32[1], \ 652 uint32_t, (_eop)->eo_u32[0]); \ 653 \ 654 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 655 (_offset), (_eop)->eo_u32[0]); \ 656 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 657 (_offset+4), (_eop)->eo_u32[1]); \ 658 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 659 (_offset+8), (_eop)->eo_u32[2]); \ 660 bus_space_write_4((_esbp)->esb_tag, (_esbp)->esb_handle,\ 661 (_offset+12), (_eop)->eo_u32[3]); \ 662 \ 663 _NOTE(CONSTANTCONDITION) \ 664 if (_lock) \ 665 mtx_unlock(&((_esbp)->esb_lock)); \ 666 _NOTE(CONSTANTCONDITION) \ 667 } while (B_FALSE) 668 669 /* SPIN */ 670 671 #define EFSYS_SPIN(_us) \ 672 do { \ 673 DELAY(_us); \ 674 _NOTE(CONSTANTCONDITION) \ 675 } while (B_FALSE) 676 677 #define EFSYS_SLEEP EFSYS_SPIN 678 679 /* BARRIERS */ 680 681 /* Strict ordering guaranteed by devacc.devacc_attr_dataorder */ 682 #define EFSYS_MEM_READ_BARRIER() 683 #define EFSYS_PIO_WRITE_BARRIER() 684 685 /* TIMESTAMP */ 686 687 typedef clock_t efsys_timestamp_t; 688 689 #define EFSYS_TIMESTAMP(_usp) \ 690 do { \ 691 clock_t now; \ 692 \ 693 now = ticks; \ 694 *(_usp) = now * hz / 1000000; \ 695 _NOTE(CONSTANTCONDITION) \ 696 } while (B_FALSE) 697 698 /* KMEM */ 699 700 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 701 do { \ 702 (_esip) = (_esip); \ 703 (_p) = malloc((_size), M_SFXGE, M_WAITOK|M_ZERO); \ 704 _NOTE(CONSTANTCONDITION) \ 705 } while (B_FALSE) 706 707 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 708 do { \ 709 (void) (_esip); \ 710 (void) (_size); \ 711 free((_p), M_SFXGE); \ 712 _NOTE(CONSTANTCONDITION) \ 713 } while (B_FALSE) 714 715 /* LOCK */ 716 717 typedef struct mtx efsys_lock_t; 718 719 #define EFSYS_LOCK_MAGIC 0x000010c4 720 721 #define EFSYS_LOCK(_lockp, _state) \ 722 do { \ 723 mtx_lock(_lockp); \ 724 (_state) = EFSYS_LOCK_MAGIC; \ 725 _NOTE(CONSTANTCONDITION) \ 726 } while (B_FALSE) 727 728 #define EFSYS_UNLOCK(_lockp, _state) \ 729 do { \ 730 if ((_state) != EFSYS_LOCK_MAGIC) \ 731 KASSERT(B_FALSE, ("not locked")); \ 732 mtx_unlock(_lockp); \ 733 _NOTE(CONSTANTCONDITION) \ 734 } while (B_FALSE) 735 736 /* PREEMPT */ 737 738 #define EFSYS_PREEMPT_DISABLE(_state) \ 739 do { \ 740 (_state) = (_state); \ 741 critical_enter(); \ 742 _NOTE(CONSTANTCONDITION) \ 743 } while (B_FALSE) 744 745 #define EFSYS_PREEMPT_ENABLE(_state) \ 746 do { \ 747 (_state) = (_state); \ 748 critical_exit(_state); \ 749 _NOTE(CONSTANTCONDITION) \ 750 } while (B_FALSE) 751 752 /* STAT */ 753 754 typedef uint64_t efsys_stat_t; 755 756 #define EFSYS_STAT_INCR(_knp, _delta) \ 757 do { \ 758 *(_knp) += (_delta); \ 759 _NOTE(CONSTANTCONDITION) \ 760 } while (B_FALSE) 761 762 #define EFSYS_STAT_DECR(_knp, _delta) \ 763 do { \ 764 *(_knp) -= (_delta); \ 765 _NOTE(CONSTANTCONDITION) \ 766 } while (B_FALSE) 767 768 #define EFSYS_STAT_SET(_knp, _val) \ 769 do { \ 770 *(_knp) = (_val); \ 771 _NOTE(CONSTANTCONDITION) \ 772 } while (B_FALSE) 773 774 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 775 do { \ 776 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 777 _NOTE(CONSTANTCONDITION) \ 778 } while (B_FALSE) 779 780 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 781 do { \ 782 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 783 _NOTE(CONSTANTCONDITION) \ 784 } while (B_FALSE) 785 786 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 787 do { \ 788 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 789 _NOTE(CONSTANTCONDITION) \ 790 } while (B_FALSE) 791 792 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 793 do { \ 794 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 795 _NOTE(CONSTANTCONDITION) \ 796 } while (B_FALSE) 797 798 /* ERR */ 799 800 extern void sfxge_err(efsys_identifier_t *, unsigned int, 801 uint32_t, uint32_t); 802 803 #if EFSYS_OPT_DECODE_INTR_FATAL 804 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 805 do { \ 806 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 807 _NOTE(CONSTANTCONDITION) \ 808 } while (B_FALSE) 809 #endif 810 811 /* ASSERT */ 812 813 #define EFSYS_ASSERT(_exp) do { \ 814 if (!(_exp)) \ 815 panic(#_exp); \ 816 } while (0) 817 818 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 819 const _t __x = (_t)(_x); \ 820 const _t __y = (_t)(_y); \ 821 if (!(__x _op __y)) \ 822 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 823 } while(0) 824 825 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 826 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 827 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 828 829 #ifdef __cplusplus 830 } 831 #endif 832 833 #endif /* _SYS_EFSYS_H */ 834