1 /*- 2 * Copyright (c) 2010-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 * 33 * $FreeBSD$ 34 */ 35 36 #ifndef _SYS_EFSYS_H 37 #define _SYS_EFSYS_H 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/bus.h> 45 #include <sys/endian.h> 46 #include <sys/lock.h> 47 #include <sys/malloc.h> 48 #include <sys/mbuf.h> 49 #include <sys/mutex.h> 50 #include <sys/rwlock.h> 51 #include <sys/sdt.h> 52 #include <sys/systm.h> 53 54 #include <machine/bus.h> 55 #include <machine/endian.h> 56 57 #define EFSYS_HAS_UINT64 1 58 #if defined(__x86_64__) 59 #define EFSYS_USE_UINT64 1 60 #else 61 #define EFSYS_USE_UINT64 0 62 #endif 63 #define EFSYS_HAS_SSE2_M128 0 64 #if _BYTE_ORDER == _BIG_ENDIAN 65 #define EFSYS_IS_BIG_ENDIAN 1 66 #define EFSYS_IS_LITTLE_ENDIAN 0 67 #elif _BYTE_ORDER == _LITTLE_ENDIAN 68 #define EFSYS_IS_BIG_ENDIAN 0 69 #define EFSYS_IS_LITTLE_ENDIAN 1 70 #endif 71 #include "efx_types.h" 72 73 /* Common code requires this */ 74 #if __FreeBSD_version < 800068 75 #define memmove(d, s, l) bcopy(s, d, l) 76 #endif 77 78 /* FreeBSD equivalents of Solaris things */ 79 #ifndef _NOTE 80 #define _NOTE(s) 81 #endif 82 83 #ifndef B_FALSE 84 #define B_FALSE FALSE 85 #endif 86 #ifndef B_TRUE 87 #define B_TRUE TRUE 88 #endif 89 90 #ifndef IS_P2ALIGNED 91 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 92 #endif 93 94 #ifndef P2ROUNDUP 95 #define P2ROUNDUP(x, align) (-(-(x) & -(align))) 96 #endif 97 98 #ifndef P2ALIGN 99 #define P2ALIGN(_x, _a) ((_x) & -(_a)) 100 #endif 101 102 #ifndef IS2P 103 #define ISP2(x) (((x) & ((x) - 1)) == 0) 104 #endif 105 106 #if defined(__x86_64__) && __FreeBSD_version >= 1000000 107 108 #define SFXGE_USE_BUS_SPACE_8 1 109 110 #if !defined(bus_space_read_stream_8) 111 112 #define bus_space_read_stream_8(t, h, o) \ 113 bus_space_read_8((t), (h), (o)) 114 115 #define bus_space_write_stream_8(t, h, o, v) \ 116 bus_space_write_8((t), (h), (o), (v)) 117 118 #endif 119 120 #endif 121 122 #define ENOTACTIVE EINVAL 123 124 /* Memory type to use on FreeBSD */ 125 MALLOC_DECLARE(M_SFXGE); 126 127 /* Machine dependend prefetch wrappers */ 128 #if defined(__i386__) || defined(__amd64__) 129 static __inline void 130 prefetch_read_many(void *addr) 131 { 132 133 __asm__( 134 "prefetcht0 (%0)" 135 : 136 : "r" (addr)); 137 } 138 139 static __inline void 140 prefetch_read_once(void *addr) 141 { 142 143 __asm__( 144 "prefetchnta (%0)" 145 : 146 : "r" (addr)); 147 } 148 #elif defined(__sparc64__) 149 static __inline void 150 prefetch_read_many(void *addr) 151 { 152 153 __asm__( 154 "prefetch [%0], 0" 155 : 156 : "r" (addr)); 157 } 158 159 static __inline void 160 prefetch_read_once(void *addr) 161 { 162 163 __asm__( 164 "prefetch [%0], 1" 165 : 166 : "r" (addr)); 167 } 168 #else 169 static __inline void 170 prefetch_read_many(void *addr) 171 { 172 173 } 174 175 static __inline void 176 prefetch_read_once(void *addr) 177 { 178 179 } 180 #endif 181 182 #if defined(__i386__) || defined(__amd64__) 183 #include <vm/vm.h> 184 #include <vm/pmap.h> 185 #endif 186 static __inline void 187 sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t map, 188 struct mbuf *m, bus_dma_segment_t *seg) 189 { 190 #if defined(__i386__) || defined(__amd64__) 191 seg->ds_addr = pmap_kextract(mtod(m, vm_offset_t)); 192 seg->ds_len = m->m_len; 193 #else 194 int nsegstmp; 195 196 bus_dmamap_load_mbuf_sg(tag, map, m, seg, &nsegstmp, 0); 197 #endif 198 } 199 200 /* Modifiers used for Windows builds */ 201 #define __in 202 #define __in_opt 203 #define __in_ecount(_n) 204 #define __in_ecount_opt(_n) 205 #define __in_bcount(_n) 206 #define __in_bcount_opt(_n) 207 208 #define __out 209 #define __out_opt 210 #define __out_ecount(_n) 211 #define __out_ecount_opt(_n) 212 #define __out_bcount(_n) 213 #define __out_bcount_opt(_n) 214 215 #define __deref_out 216 217 #define __inout 218 #define __inout_opt 219 #define __inout_ecount(_n) 220 #define __inout_ecount_opt(_n) 221 #define __inout_bcount(_n) 222 #define __inout_bcount_opt(_n) 223 #define __inout_bcount_full_opt(_n) 224 225 #define __deref_out_bcount_opt(n) 226 227 #define __checkReturn 228 #define __success(_x) 229 230 #define __drv_when(_p, _c) 231 232 /* Code inclusion options */ 233 234 235 #define EFSYS_OPT_NAMES 1 236 237 #define EFSYS_OPT_FALCON 0 238 #define EFSYS_OPT_FALCON_NIC_CFG_OVERRIDE 0 239 #define EFSYS_OPT_SIENA 1 240 #define EFSYS_OPT_HUNTINGTON 1 241 #define EFSYS_OPT_MEDFORD 0 242 #ifdef DEBUG 243 #define EFSYS_OPT_CHECK_REG 1 244 #else 245 #define EFSYS_OPT_CHECK_REG 0 246 #endif 247 248 #define EFSYS_OPT_MCDI 1 249 #define EFSYS_OPT_MCDI_LOGGING 0 250 #define EFSYS_OPT_MCDI_PROXY_AUTH 0 251 252 #define EFSYS_OPT_MAC_FALCON_GMAC 0 253 #define EFSYS_OPT_MAC_FALCON_XMAC 0 254 #define EFSYS_OPT_MAC_STATS 1 255 256 #define EFSYS_OPT_LOOPBACK 0 257 258 #define EFSYS_OPT_MON_NULL 0 259 #define EFSYS_OPT_MON_LM87 0 260 #define EFSYS_OPT_MON_MAX6647 0 261 #define EFSYS_OPT_MON_MCDI 0 262 #define EFSYS_OPT_MON_STATS 0 263 264 #define EFSYS_OPT_PHY_NULL 0 265 #define EFSYS_OPT_PHY_QT2022C2 0 266 #define EFSYS_OPT_PHY_SFX7101 0 267 #define EFSYS_OPT_PHY_TXC43128 0 268 #define EFSYS_OPT_PHY_SFT9001 0 269 #define EFSYS_OPT_PHY_QT2025C 0 270 #define EFSYS_OPT_PHY_STATS 1 271 #define EFSYS_OPT_PHY_PROPS 0 272 #define EFSYS_OPT_PHY_BIST 0 273 #define EFSYS_OPT_BIST 1 274 #define EFSYS_OPT_PHY_LED_CONTROL 1 275 #define EFSYS_OPT_PHY_FLAGS 0 276 277 #define EFSYS_OPT_VPD 1 278 #define EFSYS_OPT_NVRAM 1 279 #define EFSYS_OPT_NVRAM_FALCON_BOOTROM 0 280 #define EFSYS_OPT_NVRAM_SFT9001 0 281 #define EFSYS_OPT_NVRAM_SFX7101 0 282 #define EFSYS_OPT_BOOTCFG 0 283 284 #define EFSYS_OPT_PCIE_TUNE 0 285 #define EFSYS_OPT_DIAG 0 286 #define EFSYS_OPT_WOL 1 287 #define EFSYS_OPT_RX_SCALE 1 288 #define EFSYS_OPT_QSTATS 1 289 #define EFSYS_OPT_FILTER 1 290 #define EFSYS_OPT_RX_SCATTER 0 291 292 #define EFSYS_OPT_EV_PREFETCH 0 293 294 #define EFSYS_OPT_DECODE_INTR_FATAL 1 295 296 #define EFSYS_OPT_LICENSING 0 297 298 /* ID */ 299 300 typedef struct __efsys_identifier_s efsys_identifier_t; 301 302 /* PROBE */ 303 304 #ifndef DTRACE_PROBE 305 306 #define EFSYS_PROBE(_name) 307 308 #define EFSYS_PROBE1(_name, _type1, _arg1) 309 310 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) 311 312 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 313 _type3, _arg3) 314 315 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 316 _type3, _arg3, _type4, _arg4) 317 318 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 319 _type3, _arg3, _type4, _arg4, _type5, _arg5) 320 321 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 322 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 323 _type6, _arg6) 324 325 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 326 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 327 _type6, _arg6, _type7, _arg7) 328 329 #else /* DTRACE_PROBE */ 330 331 #define EFSYS_PROBE(_name) \ 332 DTRACE_PROBE(_name) 333 334 #define EFSYS_PROBE1(_name, _type1, _arg1) \ 335 DTRACE_PROBE1(_name, _type1, _arg1) 336 337 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2) \ 338 DTRACE_PROBE2(_name, _type1, _arg1, _type2, _arg2) 339 340 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 341 _type3, _arg3) \ 342 DTRACE_PROBE3(_name, _type1, _arg1, _type2, _arg2, \ 343 _type3, _arg3) 344 345 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 346 _type3, _arg3, _type4, _arg4) \ 347 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 348 _type3, _arg3, _type4, _arg4) 349 350 #ifdef DTRACE_PROBE5 351 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 352 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 353 DTRACE_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 354 _type3, _arg3, _type4, _arg4, _type5, _arg5) 355 #else 356 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 357 _type3, _arg3, _type4, _arg4, _type5, _arg5) \ 358 DTRACE_PROBE4(_name, _type1, _arg1, _type2, _arg2, \ 359 _type3, _arg3, _type4, _arg4) 360 #endif 361 362 #ifdef DTRACE_PROBE6 363 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 364 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 365 _type6, _arg6) \ 366 DTRACE_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 367 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 368 _type6, _arg6) 369 #else 370 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 371 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 372 _type6, _arg6) \ 373 EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2, \ 374 _type3, _arg3, _type4, _arg4, _type5, _arg5) 375 #endif 376 377 #ifdef DTRACE_PROBE7 378 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 379 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 380 _type6, _arg6, _type7, _arg7) \ 381 DTRACE_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 382 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 383 _type6, _arg6, _type7, _arg7) 384 #else 385 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2, \ 386 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 387 _type6, _arg6, _type7, _arg7) \ 388 EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2, \ 389 _type3, _arg3, _type4, _arg4, _type5, _arg5, \ 390 _type6, _arg6) 391 #endif 392 393 #endif /* DTRACE_PROBE */ 394 395 /* DMA */ 396 397 typedef uint64_t efsys_dma_addr_t; 398 399 typedef struct efsys_mem_s { 400 bus_dma_tag_t esm_tag; 401 bus_dmamap_t esm_map; 402 caddr_t esm_base; 403 efsys_dma_addr_t esm_addr; 404 } efsys_mem_t; 405 406 407 #define EFSYS_MEM_ZERO(_esmp, _size) \ 408 do { \ 409 (void) memset((_esmp)->esm_base, 0, (_size)); \ 410 \ 411 _NOTE(CONSTANTCONDITION) \ 412 } while (B_FALSE) 413 414 #define EFSYS_MEM_READD(_esmp, _offset, _edp) \ 415 do { \ 416 uint32_t *addr; \ 417 \ 418 _NOTE(CONSTANTCONDITION) \ 419 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 420 ("not power of 2 aligned")); \ 421 \ 422 addr = (void *)((_esmp)->esm_base + (_offset)); \ 423 \ 424 (_edp)->ed_u32[0] = *addr; \ 425 \ 426 EFSYS_PROBE2(mem_readd, unsigned int, (_offset), \ 427 uint32_t, (_edp)->ed_u32[0]); \ 428 \ 429 _NOTE(CONSTANTCONDITION) \ 430 } while (B_FALSE) 431 432 #if defined(__x86_64__) 433 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 434 do { \ 435 uint64_t *addr; \ 436 \ 437 _NOTE(CONSTANTCONDITION) \ 438 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 439 ("not power of 2 aligned")); \ 440 \ 441 addr = (void *)((_esmp)->esm_base + (_offset)); \ 442 \ 443 (_eqp)->eq_u64[0] = *addr; \ 444 \ 445 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 446 uint32_t, (_eqp)->eq_u32[1], \ 447 uint32_t, (_eqp)->eq_u32[0]); \ 448 \ 449 _NOTE(CONSTANTCONDITION) \ 450 } while (B_FALSE) 451 #else 452 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp) \ 453 do { \ 454 uint32_t *addr; \ 455 \ 456 _NOTE(CONSTANTCONDITION) \ 457 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 458 ("not power of 2 aligned")); \ 459 \ 460 addr = (void *)((_esmp)->esm_base + (_offset)); \ 461 \ 462 (_eqp)->eq_u32[0] = *addr++; \ 463 (_eqp)->eq_u32[1] = *addr; \ 464 \ 465 EFSYS_PROBE3(mem_readq, unsigned int, (_offset), \ 466 uint32_t, (_eqp)->eq_u32[1], \ 467 uint32_t, (_eqp)->eq_u32[0]); \ 468 \ 469 _NOTE(CONSTANTCONDITION) \ 470 } while (B_FALSE) 471 #endif 472 473 #if defined(__x86_64__) 474 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 475 do { \ 476 uint64_t *addr; \ 477 \ 478 _NOTE(CONSTANTCONDITION) \ 479 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 480 ("not power of 2 aligned")); \ 481 \ 482 addr = (void *)((_esmp)->esm_base + (_offset)); \ 483 \ 484 (_eop)->eo_u64[0] = *addr++; \ 485 (_eop)->eo_u64[1] = *addr; \ 486 \ 487 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 488 uint32_t, (_eop)->eo_u32[3], \ 489 uint32_t, (_eop)->eo_u32[2], \ 490 uint32_t, (_eop)->eo_u32[1], \ 491 uint32_t, (_eop)->eo_u32[0]); \ 492 \ 493 _NOTE(CONSTANTCONDITION) \ 494 } while (B_FALSE) 495 #else 496 #define EFSYS_MEM_READO(_esmp, _offset, _eop) \ 497 do { \ 498 uint32_t *addr; \ 499 \ 500 _NOTE(CONSTANTCONDITION) \ 501 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 502 ("not power of 2 aligned")); \ 503 \ 504 addr = (void *)((_esmp)->esm_base + (_offset)); \ 505 \ 506 (_eop)->eo_u32[0] = *addr++; \ 507 (_eop)->eo_u32[1] = *addr++; \ 508 (_eop)->eo_u32[2] = *addr++; \ 509 (_eop)->eo_u32[3] = *addr; \ 510 \ 511 EFSYS_PROBE5(mem_reado, unsigned int, (_offset), \ 512 uint32_t, (_eop)->eo_u32[3], \ 513 uint32_t, (_eop)->eo_u32[2], \ 514 uint32_t, (_eop)->eo_u32[1], \ 515 uint32_t, (_eop)->eo_u32[0]); \ 516 \ 517 _NOTE(CONSTANTCONDITION) \ 518 } while (B_FALSE) 519 #endif 520 521 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp) \ 522 do { \ 523 uint32_t *addr; \ 524 \ 525 _NOTE(CONSTANTCONDITION) \ 526 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 527 ("not power of 2 aligned")); \ 528 \ 529 EFSYS_PROBE2(mem_writed, unsigned int, (_offset), \ 530 uint32_t, (_edp)->ed_u32[0]); \ 531 \ 532 addr = (void *)((_esmp)->esm_base + (_offset)); \ 533 \ 534 *addr = (_edp)->ed_u32[0]; \ 535 \ 536 _NOTE(CONSTANTCONDITION) \ 537 } while (B_FALSE) 538 539 #if defined(__x86_64__) 540 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 541 do { \ 542 uint64_t *addr; \ 543 \ 544 _NOTE(CONSTANTCONDITION) \ 545 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 546 ("not power of 2 aligned")); \ 547 \ 548 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 549 uint32_t, (_eqp)->eq_u32[1], \ 550 uint32_t, (_eqp)->eq_u32[0]); \ 551 \ 552 addr = (void *)((_esmp)->esm_base + (_offset)); \ 553 \ 554 *addr = (_eqp)->eq_u64[0]; \ 555 \ 556 _NOTE(CONSTANTCONDITION) \ 557 } while (B_FALSE) 558 559 #else 560 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp) \ 561 do { \ 562 uint32_t *addr; \ 563 \ 564 _NOTE(CONSTANTCONDITION) \ 565 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 566 ("not power of 2 aligned")); \ 567 \ 568 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset), \ 569 uint32_t, (_eqp)->eq_u32[1], \ 570 uint32_t, (_eqp)->eq_u32[0]); \ 571 \ 572 addr = (void *)((_esmp)->esm_base + (_offset)); \ 573 \ 574 *addr++ = (_eqp)->eq_u32[0]; \ 575 *addr = (_eqp)->eq_u32[1]; \ 576 \ 577 _NOTE(CONSTANTCONDITION) \ 578 } while (B_FALSE) 579 #endif 580 581 #if defined(__x86_64__) 582 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 583 do { \ 584 uint64_t *addr; \ 585 \ 586 _NOTE(CONSTANTCONDITION) \ 587 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 588 ("not power of 2 aligned")); \ 589 \ 590 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 591 uint32_t, (_eop)->eo_u32[3], \ 592 uint32_t, (_eop)->eo_u32[2], \ 593 uint32_t, (_eop)->eo_u32[1], \ 594 uint32_t, (_eop)->eo_u32[0]); \ 595 \ 596 addr = (void *)((_esmp)->esm_base + (_offset)); \ 597 \ 598 *addr++ = (_eop)->eo_u64[0]; \ 599 *addr = (_eop)->eo_u64[1]; \ 600 \ 601 _NOTE(CONSTANTCONDITION) \ 602 } while (B_FALSE) 603 #else 604 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop) \ 605 do { \ 606 uint32_t *addr; \ 607 \ 608 _NOTE(CONSTANTCONDITION) \ 609 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 610 ("not power of 2 aligned")); \ 611 \ 612 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset), \ 613 uint32_t, (_eop)->eo_u32[3], \ 614 uint32_t, (_eop)->eo_u32[2], \ 615 uint32_t, (_eop)->eo_u32[1], \ 616 uint32_t, (_eop)->eo_u32[0]); \ 617 \ 618 addr = (void *)((_esmp)->esm_base + (_offset)); \ 619 \ 620 *addr++ = (_eop)->eo_u32[0]; \ 621 *addr++ = (_eop)->eo_u32[1]; \ 622 *addr++ = (_eop)->eo_u32[2]; \ 623 *addr = (_eop)->eo_u32[3]; \ 624 \ 625 _NOTE(CONSTANTCONDITION) \ 626 } while (B_FALSE) 627 #endif 628 629 #define EFSYS_MEM_ADDR(_esmp) \ 630 ((_esmp)->esm_addr) 631 632 #define EFSYS_MEM_IS_NULL(_esmp) \ 633 ((_esmp)->esm_base == NULL) 634 635 /* BAR */ 636 637 #define SFXGE_LOCK_NAME_MAX 16 638 639 typedef struct efsys_bar_s { 640 struct mtx esb_lock; 641 char esb_lock_name[SFXGE_LOCK_NAME_MAX]; 642 bus_space_tag_t esb_tag; 643 bus_space_handle_t esb_handle; 644 int esb_rid; 645 struct resource *esb_res; 646 } efsys_bar_t; 647 648 #define SFXGE_BAR_LOCK_INIT(_esbp, _ifname) \ 649 do { \ 650 snprintf((_esbp)->esb_lock_name, \ 651 sizeof((_esbp)->esb_lock_name), \ 652 "%s:bar", (_ifname)); \ 653 mtx_init(&(_esbp)->esb_lock, (_esbp)->esb_lock_name, \ 654 NULL, MTX_DEF); \ 655 _NOTE(CONSTANTCONDITION) \ 656 } while (B_FALSE) 657 #define SFXGE_BAR_LOCK_DESTROY(_esbp) \ 658 mtx_destroy(&(_esbp)->esb_lock) 659 #define SFXGE_BAR_LOCK(_esbp) \ 660 mtx_lock(&(_esbp)->esb_lock) 661 #define SFXGE_BAR_UNLOCK(_esbp) \ 662 mtx_unlock(&(_esbp)->esb_lock) 663 664 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock) \ 665 do { \ 666 _NOTE(CONSTANTCONDITION) \ 667 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 668 ("not power of 2 aligned")); \ 669 \ 670 _NOTE(CONSTANTCONDITION) \ 671 if (_lock) \ 672 SFXGE_BAR_LOCK(_esbp); \ 673 \ 674 (_edp)->ed_u32[0] = bus_space_read_stream_4( \ 675 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 676 (_offset)); \ 677 \ 678 EFSYS_PROBE2(bar_readd, unsigned int, (_offset), \ 679 uint32_t, (_edp)->ed_u32[0]); \ 680 \ 681 _NOTE(CONSTANTCONDITION) \ 682 if (_lock) \ 683 SFXGE_BAR_UNLOCK(_esbp); \ 684 _NOTE(CONSTANTCONDITION) \ 685 } while (B_FALSE) 686 687 #if defined(SFXGE_USE_BUS_SPACE_8) 688 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 689 do { \ 690 _NOTE(CONSTANTCONDITION) \ 691 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 692 ("not power of 2 aligned")); \ 693 \ 694 SFXGE_BAR_LOCK(_esbp); \ 695 \ 696 (_eqp)->eq_u64[0] = bus_space_read_stream_8( \ 697 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 698 (_offset)); \ 699 \ 700 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 701 uint32_t, (_eqp)->eq_u32[1], \ 702 uint32_t, (_eqp)->eq_u32[0]); \ 703 \ 704 SFXGE_BAR_UNLOCK(_esbp); \ 705 _NOTE(CONSTANTCONDITION) \ 706 } while (B_FALSE) 707 708 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 709 do { \ 710 _NOTE(CONSTANTCONDITION) \ 711 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 712 ("not power of 2 aligned")); \ 713 \ 714 _NOTE(CONSTANTCONDITION) \ 715 if (_lock) \ 716 SFXGE_BAR_LOCK(_esbp); \ 717 \ 718 (_eop)->eo_u64[0] = bus_space_read_stream_8( \ 719 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 720 (_offset)); \ 721 (_eop)->eo_u64[1] = bus_space_read_stream_8( \ 722 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 723 (_offset) + 8); \ 724 \ 725 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 726 uint32_t, (_eop)->eo_u32[3], \ 727 uint32_t, (_eop)->eo_u32[2], \ 728 uint32_t, (_eop)->eo_u32[1], \ 729 uint32_t, (_eop)->eo_u32[0]); \ 730 \ 731 _NOTE(CONSTANTCONDITION) \ 732 if (_lock) \ 733 SFXGE_BAR_UNLOCK(_esbp); \ 734 _NOTE(CONSTANTCONDITION) \ 735 } while (B_FALSE) 736 737 #else 738 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp) \ 739 do { \ 740 _NOTE(CONSTANTCONDITION) \ 741 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 742 ("not power of 2 aligned")); \ 743 \ 744 SFXGE_BAR_LOCK(_esbp); \ 745 \ 746 (_eqp)->eq_u32[0] = bus_space_read_stream_4( \ 747 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 748 (_offset)); \ 749 (_eqp)->eq_u32[1] = bus_space_read_stream_4( \ 750 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 751 (_offset) + 4); \ 752 \ 753 EFSYS_PROBE3(bar_readq, unsigned int, (_offset), \ 754 uint32_t, (_eqp)->eq_u32[1], \ 755 uint32_t, (_eqp)->eq_u32[0]); \ 756 \ 757 SFXGE_BAR_UNLOCK(_esbp); \ 758 _NOTE(CONSTANTCONDITION) \ 759 } while (B_FALSE) 760 761 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock) \ 762 do { \ 763 _NOTE(CONSTANTCONDITION) \ 764 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 765 ("not power of 2 aligned")); \ 766 \ 767 _NOTE(CONSTANTCONDITION) \ 768 if (_lock) \ 769 SFXGE_BAR_LOCK(_esbp); \ 770 \ 771 (_eop)->eo_u32[0] = bus_space_read_stream_4( \ 772 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 773 (_offset)); \ 774 (_eop)->eo_u32[1] = bus_space_read_stream_4( \ 775 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 776 (_offset) + 4); \ 777 (_eop)->eo_u32[2] = bus_space_read_stream_4( \ 778 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 779 (_offset) + 8); \ 780 (_eop)->eo_u32[3] = bus_space_read_stream_4( \ 781 (_esbp)->esb_tag, (_esbp)->esb_handle, \ 782 (_offset) + 12); \ 783 \ 784 EFSYS_PROBE5(bar_reado, unsigned int, (_offset), \ 785 uint32_t, (_eop)->eo_u32[3], \ 786 uint32_t, (_eop)->eo_u32[2], \ 787 uint32_t, (_eop)->eo_u32[1], \ 788 uint32_t, (_eop)->eo_u32[0]); \ 789 \ 790 _NOTE(CONSTANTCONDITION) \ 791 if (_lock) \ 792 SFXGE_BAR_UNLOCK(_esbp); \ 793 _NOTE(CONSTANTCONDITION) \ 794 } while (B_FALSE) 795 #endif 796 797 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock) \ 798 do { \ 799 _NOTE(CONSTANTCONDITION) \ 800 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_dword_t)), \ 801 ("not power of 2 aligned")); \ 802 \ 803 _NOTE(CONSTANTCONDITION) \ 804 if (_lock) \ 805 SFXGE_BAR_LOCK(_esbp); \ 806 \ 807 EFSYS_PROBE2(bar_writed, unsigned int, (_offset), \ 808 uint32_t, (_edp)->ed_u32[0]); \ 809 \ 810 /* \ 811 * Make sure that previous writes to the dword have \ 812 * been done. It should be cheaper than barrier just \ 813 * after the write below. \ 814 */ \ 815 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 816 (_offset), sizeof (efx_dword_t), \ 817 BUS_SPACE_BARRIER_WRITE); \ 818 bus_space_write_stream_4((_esbp)->esb_tag, \ 819 (_esbp)->esb_handle, \ 820 (_offset), (_edp)->ed_u32[0]); \ 821 \ 822 _NOTE(CONSTANTCONDITION) \ 823 if (_lock) \ 824 SFXGE_BAR_UNLOCK(_esbp); \ 825 _NOTE(CONSTANTCONDITION) \ 826 } while (B_FALSE) 827 828 #if defined(SFXGE_USE_BUS_SPACE_8) 829 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 830 do { \ 831 _NOTE(CONSTANTCONDITION) \ 832 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 833 ("not power of 2 aligned")); \ 834 \ 835 SFXGE_BAR_LOCK(_esbp); \ 836 \ 837 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 838 uint32_t, (_eqp)->eq_u32[1], \ 839 uint32_t, (_eqp)->eq_u32[0]); \ 840 \ 841 /* \ 842 * Make sure that previous writes to the qword have \ 843 * been done. It should be cheaper than barrier just \ 844 * after the write below. \ 845 */ \ 846 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 847 (_offset), sizeof (efx_qword_t), \ 848 BUS_SPACE_BARRIER_WRITE); \ 849 bus_space_write_stream_8((_esbp)->esb_tag, \ 850 (_esbp)->esb_handle, \ 851 (_offset), (_eqp)->eq_u64[0]); \ 852 \ 853 SFXGE_BAR_UNLOCK(_esbp); \ 854 _NOTE(CONSTANTCONDITION) \ 855 } while (B_FALSE) 856 #else 857 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp) \ 858 do { \ 859 _NOTE(CONSTANTCONDITION) \ 860 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 861 ("not power of 2 aligned")); \ 862 \ 863 SFXGE_BAR_LOCK(_esbp); \ 864 \ 865 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset), \ 866 uint32_t, (_eqp)->eq_u32[1], \ 867 uint32_t, (_eqp)->eq_u32[0]); \ 868 \ 869 /* \ 870 * Make sure that previous writes to the qword have \ 871 * been done. It should be cheaper than barrier just \ 872 * after the last write below. \ 873 */ \ 874 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 875 (_offset), sizeof (efx_qword_t), \ 876 BUS_SPACE_BARRIER_WRITE); \ 877 bus_space_write_stream_4((_esbp)->esb_tag, \ 878 (_esbp)->esb_handle, \ 879 (_offset), (_eqp)->eq_u32[0]); \ 880 /* \ 881 * It should be guaranteed that the last dword comes \ 882 * the last, so barrier entire qword to be sure that \ 883 * neither above nor below writes are reordered. \ 884 */ \ 885 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 886 (_offset), sizeof (efx_qword_t), \ 887 BUS_SPACE_BARRIER_WRITE); \ 888 bus_space_write_stream_4((_esbp)->esb_tag, \ 889 (_esbp)->esb_handle, \ 890 (_offset) + 4, (_eqp)->eq_u32[1]); \ 891 \ 892 SFXGE_BAR_UNLOCK(_esbp); \ 893 _NOTE(CONSTANTCONDITION) \ 894 } while (B_FALSE) 895 #endif 896 897 /* 898 * Guarantees 64bit aligned 64bit writes to write combined BAR mapping 899 * (required by PIO hardware) 900 */ 901 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp) \ 902 do { \ 903 _NOTE(CONSTANTCONDITION) \ 904 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_qword_t)), \ 905 ("not power of 2 aligned")); \ 906 \ 907 (void) (_esbp); \ 908 \ 909 /* FIXME: Perform a 64-bit write */ \ 910 KASSERT(0, ("not implemented")); \ 911 \ 912 _NOTE(CONSTANTCONDITION) \ 913 } while (B_FALSE) 914 915 #if defined(SFXGE_USE_BUS_SPACE_8) 916 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 917 do { \ 918 _NOTE(CONSTANTCONDITION) \ 919 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 920 ("not power of 2 aligned")); \ 921 \ 922 _NOTE(CONSTANTCONDITION) \ 923 if (_lock) \ 924 SFXGE_BAR_LOCK(_esbp); \ 925 \ 926 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 927 uint32_t, (_eop)->eo_u32[3], \ 928 uint32_t, (_eop)->eo_u32[2], \ 929 uint32_t, (_eop)->eo_u32[1], \ 930 uint32_t, (_eop)->eo_u32[0]); \ 931 \ 932 /* \ 933 * Make sure that previous writes to the oword have \ 934 * been done. It should be cheaper than barrier just \ 935 * after the last write below. \ 936 */ \ 937 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 938 (_offset), sizeof (efx_oword_t), \ 939 BUS_SPACE_BARRIER_WRITE); \ 940 bus_space_write_stream_8((_esbp)->esb_tag, \ 941 (_esbp)->esb_handle, \ 942 (_offset), (_eop)->eo_u64[0]); \ 943 /* \ 944 * It should be guaranteed that the last qword comes \ 945 * the last, so barrier entire oword to be sure that \ 946 * neither above nor below writes are reordered. \ 947 */ \ 948 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 949 (_offset), sizeof (efx_oword_t), \ 950 BUS_SPACE_BARRIER_WRITE); \ 951 bus_space_write_stream_8((_esbp)->esb_tag, \ 952 (_esbp)->esb_handle, \ 953 (_offset) + 8, (_eop)->eo_u64[1]); \ 954 \ 955 _NOTE(CONSTANTCONDITION) \ 956 if (_lock) \ 957 SFXGE_BAR_UNLOCK(_esbp); \ 958 _NOTE(CONSTANTCONDITION) \ 959 } while (B_FALSE) 960 961 #else 962 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock) \ 963 do { \ 964 _NOTE(CONSTANTCONDITION) \ 965 KASSERT(IS_P2ALIGNED(_offset, sizeof (efx_oword_t)), \ 966 ("not power of 2 aligned")); \ 967 \ 968 _NOTE(CONSTANTCONDITION) \ 969 if (_lock) \ 970 SFXGE_BAR_LOCK(_esbp); \ 971 \ 972 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset), \ 973 uint32_t, (_eop)->eo_u32[3], \ 974 uint32_t, (_eop)->eo_u32[2], \ 975 uint32_t, (_eop)->eo_u32[1], \ 976 uint32_t, (_eop)->eo_u32[0]); \ 977 \ 978 /* \ 979 * Make sure that previous writes to the oword have \ 980 * been done. It should be cheaper than barrier just \ 981 * after the last write below. \ 982 */ \ 983 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 984 (_offset), sizeof (efx_oword_t), \ 985 BUS_SPACE_BARRIER_WRITE); \ 986 bus_space_write_stream_4((_esbp)->esb_tag, \ 987 (_esbp)->esb_handle, \ 988 (_offset), (_eop)->eo_u32[0]); \ 989 bus_space_write_stream_4((_esbp)->esb_tag, \ 990 (_esbp)->esb_handle, \ 991 (_offset) + 4, (_eop)->eo_u32[1]); \ 992 bus_space_write_stream_4((_esbp)->esb_tag, \ 993 (_esbp)->esb_handle, \ 994 (_offset) + 8, (_eop)->eo_u32[2]); \ 995 /* \ 996 * It should be guaranteed that the last dword comes \ 997 * the last, so barrier entire oword to be sure that \ 998 * neither above nor below writes are reordered. \ 999 */ \ 1000 bus_space_barrier((_esbp)->esb_tag, (_esbp)->esb_handle,\ 1001 (_offset), sizeof (efx_oword_t), \ 1002 BUS_SPACE_BARRIER_WRITE); \ 1003 bus_space_write_stream_4((_esbp)->esb_tag, \ 1004 (_esbp)->esb_handle, \ 1005 (_offset) + 12, (_eop)->eo_u32[3]); \ 1006 \ 1007 _NOTE(CONSTANTCONDITION) \ 1008 if (_lock) \ 1009 SFXGE_BAR_UNLOCK(_esbp); \ 1010 _NOTE(CONSTANTCONDITION) \ 1011 } while (B_FALSE) 1012 #endif 1013 1014 /* Use the standard octo-word write for doorbell writes */ 1015 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop) \ 1016 do { \ 1017 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE); \ 1018 _NOTE(CONSTANTCONDITION) \ 1019 } while (B_FALSE) 1020 1021 /* SPIN */ 1022 1023 #define EFSYS_SPIN(_us) \ 1024 do { \ 1025 DELAY(_us); \ 1026 _NOTE(CONSTANTCONDITION) \ 1027 } while (B_FALSE) 1028 1029 #define EFSYS_SLEEP EFSYS_SPIN 1030 1031 /* BARRIERS */ 1032 1033 #define EFSYS_MEM_READ_BARRIER() rmb() 1034 #define EFSYS_PIO_WRITE_BARRIER() 1035 1036 /* DMA SYNC */ 1037 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size) \ 1038 do { \ 1039 bus_dmamap_sync((_esmp)->esm_tag, \ 1040 (_esmp)->esm_map, \ 1041 BUS_DMASYNC_POSTREAD); \ 1042 _NOTE(CONSTANTCONDITION) \ 1043 } while (B_FALSE) 1044 1045 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size) \ 1046 do { \ 1047 bus_dmamap_sync((_esmp)->esm_tag, \ 1048 (_esmp)->esm_map, \ 1049 BUS_DMASYNC_PREWRITE); \ 1050 _NOTE(CONSTANTCONDITION) \ 1051 } while (B_FALSE) 1052 1053 /* TIMESTAMP */ 1054 1055 typedef clock_t efsys_timestamp_t; 1056 1057 #define EFSYS_TIMESTAMP(_usp) \ 1058 do { \ 1059 clock_t now; \ 1060 \ 1061 now = ticks; \ 1062 *(_usp) = now * hz / 1000000; \ 1063 _NOTE(CONSTANTCONDITION) \ 1064 } while (B_FALSE) 1065 1066 /* KMEM */ 1067 1068 #define EFSYS_KMEM_ALLOC(_esip, _size, _p) \ 1069 do { \ 1070 (_esip) = (_esip); \ 1071 /* \ 1072 * The macro is used in non-sleepable contexts, for \ 1073 * example, holding a mutex. \ 1074 */ \ 1075 (_p) = malloc((_size), M_SFXGE, M_NOWAIT|M_ZERO); \ 1076 _NOTE(CONSTANTCONDITION) \ 1077 } while (B_FALSE) 1078 1079 #define EFSYS_KMEM_FREE(_esip, _size, _p) \ 1080 do { \ 1081 (void) (_esip); \ 1082 (void) (_size); \ 1083 free((_p), M_SFXGE); \ 1084 _NOTE(CONSTANTCONDITION) \ 1085 } while (B_FALSE) 1086 1087 /* LOCK */ 1088 1089 typedef struct efsys_lock_s { 1090 struct mtx lock; 1091 char lock_name[SFXGE_LOCK_NAME_MAX]; 1092 } efsys_lock_t; 1093 1094 #define SFXGE_EFSYS_LOCK_INIT(_eslp, _ifname, _label) \ 1095 do { \ 1096 efsys_lock_t *__eslp = (_eslp); \ 1097 \ 1098 snprintf((__eslp)->lock_name, \ 1099 sizeof((__eslp)->lock_name), \ 1100 "%s:%s", (_ifname), (_label)); \ 1101 mtx_init(&(__eslp)->lock, (__eslp)->lock_name, \ 1102 NULL, MTX_DEF); \ 1103 } while (B_FALSE) 1104 #define SFXGE_EFSYS_LOCK_DESTROY(_eslp) \ 1105 mtx_destroy(&(_eslp)->lock) 1106 #define SFXGE_EFSYS_LOCK(_eslp) \ 1107 mtx_lock(&(_eslp)->lock) 1108 #define SFXGE_EFSYS_UNLOCK(_eslp) \ 1109 mtx_unlock(&(_eslp)->lock) 1110 #define SFXGE_EFSYS_LOCK_ASSERT_OWNED(_eslp) \ 1111 mtx_assert(&(_eslp)->lock, MA_OWNED) 1112 1113 #define EFSYS_LOCK_MAGIC 0x000010c4 1114 1115 #define EFSYS_LOCK(_lockp, _state) \ 1116 do { \ 1117 SFXGE_EFSYS_LOCK(_lockp); \ 1118 (_state) = EFSYS_LOCK_MAGIC; \ 1119 _NOTE(CONSTANTCONDITION) \ 1120 } while (B_FALSE) 1121 1122 #define EFSYS_UNLOCK(_lockp, _state) \ 1123 do { \ 1124 if ((_state) != EFSYS_LOCK_MAGIC) \ 1125 KASSERT(B_FALSE, ("not locked")); \ 1126 SFXGE_EFSYS_UNLOCK(_lockp); \ 1127 _NOTE(CONSTANTCONDITION) \ 1128 } while (B_FALSE) 1129 1130 /* PREEMPT */ 1131 1132 #define EFSYS_PREEMPT_DISABLE(_state) \ 1133 do { \ 1134 (_state) = (_state); \ 1135 critical_enter(); \ 1136 _NOTE(CONSTANTCONDITION) \ 1137 } while (B_FALSE) 1138 1139 #define EFSYS_PREEMPT_ENABLE(_state) \ 1140 do { \ 1141 (_state) = (_state); \ 1142 critical_exit(_state); \ 1143 _NOTE(CONSTANTCONDITION) \ 1144 } while (B_FALSE) 1145 1146 /* STAT */ 1147 1148 typedef uint64_t efsys_stat_t; 1149 1150 #define EFSYS_STAT_INCR(_knp, _delta) \ 1151 do { \ 1152 *(_knp) += (_delta); \ 1153 _NOTE(CONSTANTCONDITION) \ 1154 } while (B_FALSE) 1155 1156 #define EFSYS_STAT_DECR(_knp, _delta) \ 1157 do { \ 1158 *(_knp) -= (_delta); \ 1159 _NOTE(CONSTANTCONDITION) \ 1160 } while (B_FALSE) 1161 1162 #define EFSYS_STAT_SET(_knp, _val) \ 1163 do { \ 1164 *(_knp) = (_val); \ 1165 _NOTE(CONSTANTCONDITION) \ 1166 } while (B_FALSE) 1167 1168 #define EFSYS_STAT_SET_QWORD(_knp, _valp) \ 1169 do { \ 1170 *(_knp) = le64toh((_valp)->eq_u64[0]); \ 1171 _NOTE(CONSTANTCONDITION) \ 1172 } while (B_FALSE) 1173 1174 #define EFSYS_STAT_SET_DWORD(_knp, _valp) \ 1175 do { \ 1176 *(_knp) = le32toh((_valp)->ed_u32[0]); \ 1177 _NOTE(CONSTANTCONDITION) \ 1178 } while (B_FALSE) 1179 1180 #define EFSYS_STAT_INCR_QWORD(_knp, _valp) \ 1181 do { \ 1182 *(_knp) += le64toh((_valp)->eq_u64[0]); \ 1183 _NOTE(CONSTANTCONDITION) \ 1184 } while (B_FALSE) 1185 1186 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp) \ 1187 do { \ 1188 *(_knp) -= le64toh((_valp)->eq_u64[0]); \ 1189 _NOTE(CONSTANTCONDITION) \ 1190 } while (B_FALSE) 1191 1192 /* ERR */ 1193 1194 extern void sfxge_err(efsys_identifier_t *, unsigned int, 1195 uint32_t, uint32_t); 1196 1197 #if EFSYS_OPT_DECODE_INTR_FATAL 1198 #define EFSYS_ERR(_esip, _code, _dword0, _dword1) \ 1199 do { \ 1200 sfxge_err((_esip), (_code), (_dword0), (_dword1)); \ 1201 _NOTE(CONSTANTCONDITION) \ 1202 } while (B_FALSE) 1203 #endif 1204 1205 /* ASSERT */ 1206 1207 #define EFSYS_ASSERT(_exp) do { \ 1208 if (!(_exp)) \ 1209 panic("%s", #_exp); \ 1210 } while (0) 1211 1212 #define EFSYS_ASSERT3(_x, _op, _y, _t) do { \ 1213 const _t __x = (_t)(_x); \ 1214 const _t __y = (_t)(_y); \ 1215 if (!(__x _op __y)) \ 1216 panic("assertion failed at %s:%u", __FILE__, __LINE__); \ 1217 } while(0) 1218 1219 #define EFSYS_ASSERT3U(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uint64_t) 1220 #define EFSYS_ASSERT3S(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, int64_t) 1221 #define EFSYS_ASSERT3P(_x, _op, _y) EFSYS_ASSERT3(_x, _op, _y, uintptr_t) 1222 1223 /* ROTATE */ 1224 1225 #define EFSYS_HAS_ROTL_DWORD 0 1226 1227 #ifdef __cplusplus 1228 } 1229 #endif 1230 1231 #endif /* _SYS_EFSYS_H */ 1232