1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "efx.h" 37 #include "efx_impl.h" 38 39 #if EFSYS_OPT_QSTATS 40 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 41 do { \ 42 (_etp)->et_stat[_stat]++; \ 43 _NOTE(CONSTANTCONDITION) \ 44 } while (B_FALSE) 45 #else 46 #define EFX_TX_QSTAT_INCR(_etp, _stat) 47 #endif 48 49 #if EFSYS_OPT_SIENA 50 51 static __checkReturn efx_rc_t 52 siena_tx_init( 53 __in efx_nic_t *enp); 54 55 static void 56 siena_tx_fini( 57 __in efx_nic_t *enp); 58 59 static __checkReturn efx_rc_t 60 siena_tx_qcreate( 61 __in efx_nic_t *enp, 62 __in unsigned int index, 63 __in unsigned int label, 64 __in efsys_mem_t *esmp, 65 __in size_t ndescs, 66 __in uint32_t id, 67 __in uint16_t flags, 68 __in efx_evq_t *eep, 69 __in efx_txq_t *etp, 70 __out unsigned int *addedp); 71 72 static void 73 siena_tx_qdestroy( 74 __in efx_txq_t *etp); 75 76 static __checkReturn efx_rc_t 77 siena_tx_qpost( 78 __in efx_txq_t *etp, 79 __in_ecount(ndescs) efx_buffer_t *eb, 80 __in unsigned int ndescs, 81 __in unsigned int completed, 82 __inout unsigned int *addedp); 83 84 static void 85 siena_tx_qpush( 86 __in efx_txq_t *etp, 87 __in unsigned int added, 88 __in unsigned int pushed); 89 90 static __checkReturn efx_rc_t 91 siena_tx_qpace( 92 __in efx_txq_t *etp, 93 __in unsigned int ns); 94 95 static __checkReturn efx_rc_t 96 siena_tx_qflush( 97 __in efx_txq_t *etp); 98 99 static void 100 siena_tx_qenable( 101 __in efx_txq_t *etp); 102 103 __checkReturn efx_rc_t 104 siena_tx_qdesc_post( 105 __in efx_txq_t *etp, 106 __in_ecount(ndescs) efx_desc_t *ed, 107 __in unsigned int ndescs, 108 __in unsigned int completed, 109 __inout unsigned int *addedp); 110 111 void 112 siena_tx_qdesc_dma_create( 113 __in efx_txq_t *etp, 114 __in efsys_dma_addr_t addr, 115 __in size_t size, 116 __in boolean_t eop, 117 __out efx_desc_t *edp); 118 119 #if EFSYS_OPT_QSTATS 120 static void 121 siena_tx_qstats_update( 122 __in efx_txq_t *etp, 123 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 124 #endif 125 126 #endif /* EFSYS_OPT_SIENA */ 127 128 #if EFSYS_OPT_SIENA 129 static const efx_tx_ops_t __efx_tx_siena_ops = { 130 siena_tx_init, /* etxo_init */ 131 siena_tx_fini, /* etxo_fini */ 132 siena_tx_qcreate, /* etxo_qcreate */ 133 siena_tx_qdestroy, /* etxo_qdestroy */ 134 siena_tx_qpost, /* etxo_qpost */ 135 siena_tx_qpush, /* etxo_qpush */ 136 siena_tx_qpace, /* etxo_qpace */ 137 siena_tx_qflush, /* etxo_qflush */ 138 siena_tx_qenable, /* etxo_qenable */ 139 NULL, /* etxo_qpio_enable */ 140 NULL, /* etxo_qpio_disable */ 141 NULL, /* etxo_qpio_write */ 142 NULL, /* etxo_qpio_post */ 143 siena_tx_qdesc_post, /* etxo_qdesc_post */ 144 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 145 NULL, /* etxo_qdesc_tso_create */ 146 NULL, /* etxo_qdesc_tso2_create */ 147 NULL, /* etxo_qdesc_vlantci_create */ 148 NULL, /* etxo_qdesc_checksum_create */ 149 #if EFSYS_OPT_QSTATS 150 siena_tx_qstats_update, /* etxo_qstats_update */ 151 #endif 152 }; 153 #endif /* EFSYS_OPT_SIENA */ 154 155 #if EFSYS_OPT_HUNTINGTON 156 static const efx_tx_ops_t __efx_tx_hunt_ops = { 157 ef10_tx_init, /* etxo_init */ 158 ef10_tx_fini, /* etxo_fini */ 159 ef10_tx_qcreate, /* etxo_qcreate */ 160 ef10_tx_qdestroy, /* etxo_qdestroy */ 161 ef10_tx_qpost, /* etxo_qpost */ 162 ef10_tx_qpush, /* etxo_qpush */ 163 ef10_tx_qpace, /* etxo_qpace */ 164 ef10_tx_qflush, /* etxo_qflush */ 165 ef10_tx_qenable, /* etxo_qenable */ 166 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 167 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 168 ef10_tx_qpio_write, /* etxo_qpio_write */ 169 ef10_tx_qpio_post, /* etxo_qpio_post */ 170 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 171 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 172 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 173 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 174 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 175 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 176 #if EFSYS_OPT_QSTATS 177 ef10_tx_qstats_update, /* etxo_qstats_update */ 178 #endif 179 }; 180 #endif /* EFSYS_OPT_HUNTINGTON */ 181 182 #if EFSYS_OPT_MEDFORD 183 static const efx_tx_ops_t __efx_tx_medford_ops = { 184 ef10_tx_init, /* etxo_init */ 185 ef10_tx_fini, /* etxo_fini */ 186 ef10_tx_qcreate, /* etxo_qcreate */ 187 ef10_tx_qdestroy, /* etxo_qdestroy */ 188 ef10_tx_qpost, /* etxo_qpost */ 189 ef10_tx_qpush, /* etxo_qpush */ 190 ef10_tx_qpace, /* etxo_qpace */ 191 ef10_tx_qflush, /* etxo_qflush */ 192 ef10_tx_qenable, /* etxo_qenable */ 193 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 194 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 195 ef10_tx_qpio_write, /* etxo_qpio_write */ 196 ef10_tx_qpio_post, /* etxo_qpio_post */ 197 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 198 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 199 NULL, /* etxo_qdesc_tso_create */ 200 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 201 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 202 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 203 #if EFSYS_OPT_QSTATS 204 ef10_tx_qstats_update, /* etxo_qstats_update */ 205 #endif 206 }; 207 #endif /* EFSYS_OPT_MEDFORD */ 208 209 #if EFSYS_OPT_MEDFORD2 210 static const efx_tx_ops_t __efx_tx_medford2_ops = { 211 ef10_tx_init, /* etxo_init */ 212 ef10_tx_fini, /* etxo_fini */ 213 ef10_tx_qcreate, /* etxo_qcreate */ 214 ef10_tx_qdestroy, /* etxo_qdestroy */ 215 ef10_tx_qpost, /* etxo_qpost */ 216 ef10_tx_qpush, /* etxo_qpush */ 217 ef10_tx_qpace, /* etxo_qpace */ 218 ef10_tx_qflush, /* etxo_qflush */ 219 ef10_tx_qenable, /* etxo_qenable */ 220 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 221 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 222 ef10_tx_qpio_write, /* etxo_qpio_write */ 223 ef10_tx_qpio_post, /* etxo_qpio_post */ 224 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 225 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 226 NULL, /* etxo_qdesc_tso_create */ 227 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 228 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 229 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 230 #if EFSYS_OPT_QSTATS 231 ef10_tx_qstats_update, /* etxo_qstats_update */ 232 #endif 233 }; 234 #endif /* EFSYS_OPT_MEDFORD2 */ 235 236 __checkReturn efx_rc_t 237 efx_tx_init( 238 __in efx_nic_t *enp) 239 { 240 const efx_tx_ops_t *etxop; 241 efx_rc_t rc; 242 243 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 244 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 245 246 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 247 rc = EINVAL; 248 goto fail1; 249 } 250 251 if (enp->en_mod_flags & EFX_MOD_TX) { 252 rc = EINVAL; 253 goto fail2; 254 } 255 256 switch (enp->en_family) { 257 #if EFSYS_OPT_SIENA 258 case EFX_FAMILY_SIENA: 259 etxop = &__efx_tx_siena_ops; 260 break; 261 #endif /* EFSYS_OPT_SIENA */ 262 263 #if EFSYS_OPT_HUNTINGTON 264 case EFX_FAMILY_HUNTINGTON: 265 etxop = &__efx_tx_hunt_ops; 266 break; 267 #endif /* EFSYS_OPT_HUNTINGTON */ 268 269 #if EFSYS_OPT_MEDFORD 270 case EFX_FAMILY_MEDFORD: 271 etxop = &__efx_tx_medford_ops; 272 break; 273 #endif /* EFSYS_OPT_MEDFORD */ 274 275 #if EFSYS_OPT_MEDFORD2 276 case EFX_FAMILY_MEDFORD2: 277 etxop = &__efx_tx_medford2_ops; 278 break; 279 #endif /* EFSYS_OPT_MEDFORD2 */ 280 281 default: 282 EFSYS_ASSERT(0); 283 rc = ENOTSUP; 284 goto fail3; 285 } 286 287 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 288 289 if ((rc = etxop->etxo_init(enp)) != 0) 290 goto fail4; 291 292 enp->en_etxop = etxop; 293 enp->en_mod_flags |= EFX_MOD_TX; 294 return (0); 295 296 fail4: 297 EFSYS_PROBE(fail4); 298 fail3: 299 EFSYS_PROBE(fail3); 300 fail2: 301 EFSYS_PROBE(fail2); 302 fail1: 303 EFSYS_PROBE1(fail1, efx_rc_t, rc); 304 305 enp->en_etxop = NULL; 306 enp->en_mod_flags &= ~EFX_MOD_TX; 307 return (rc); 308 } 309 310 void 311 efx_tx_fini( 312 __in efx_nic_t *enp) 313 { 314 const efx_tx_ops_t *etxop = enp->en_etxop; 315 316 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 317 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 319 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 320 321 etxop->etxo_fini(enp); 322 323 enp->en_etxop = NULL; 324 enp->en_mod_flags &= ~EFX_MOD_TX; 325 } 326 327 __checkReturn efx_rc_t 328 efx_tx_qcreate( 329 __in efx_nic_t *enp, 330 __in unsigned int index, 331 __in unsigned int label, 332 __in efsys_mem_t *esmp, 333 __in size_t ndescs, 334 __in uint32_t id, 335 __in uint16_t flags, 336 __in efx_evq_t *eep, 337 __deref_out efx_txq_t **etpp, 338 __out unsigned int *addedp) 339 { 340 const efx_tx_ops_t *etxop = enp->en_etxop; 341 efx_txq_t *etp; 342 efx_rc_t rc; 343 344 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 345 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 346 347 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, 348 enp->en_nic_cfg.enc_txq_limit); 349 350 /* Allocate an TXQ object */ 351 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 352 353 if (etp == NULL) { 354 rc = ENOMEM; 355 goto fail1; 356 } 357 358 etp->et_magic = EFX_TXQ_MAGIC; 359 etp->et_enp = enp; 360 etp->et_index = index; 361 etp->et_mask = ndescs - 1; 362 etp->et_esmp = esmp; 363 364 /* Initial descriptor index may be modified by etxo_qcreate */ 365 *addedp = 0; 366 367 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 368 ndescs, id, flags, eep, etp, addedp)) != 0) 369 goto fail2; 370 371 enp->en_tx_qcount++; 372 *etpp = etp; 373 374 return (0); 375 376 fail2: 377 EFSYS_PROBE(fail2); 378 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 379 fail1: 380 EFSYS_PROBE1(fail1, efx_rc_t, rc); 381 return (rc); 382 } 383 384 void 385 efx_tx_qdestroy( 386 __in efx_txq_t *etp) 387 { 388 efx_nic_t *enp = etp->et_enp; 389 const efx_tx_ops_t *etxop = enp->en_etxop; 390 391 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 392 393 EFSYS_ASSERT(enp->en_tx_qcount != 0); 394 --enp->en_tx_qcount; 395 396 etxop->etxo_qdestroy(etp); 397 398 /* Free the TXQ object */ 399 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 400 } 401 402 __checkReturn efx_rc_t 403 efx_tx_qpost( 404 __in efx_txq_t *etp, 405 __in_ecount(ndescs) efx_buffer_t *eb, 406 __in unsigned int ndescs, 407 __in unsigned int completed, 408 __inout unsigned int *addedp) 409 { 410 efx_nic_t *enp = etp->et_enp; 411 const efx_tx_ops_t *etxop = enp->en_etxop; 412 efx_rc_t rc; 413 414 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 415 416 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0) 417 goto fail1; 418 419 return (0); 420 421 fail1: 422 EFSYS_PROBE1(fail1, efx_rc_t, rc); 423 return (rc); 424 } 425 426 void 427 efx_tx_qpush( 428 __in efx_txq_t *etp, 429 __in unsigned int added, 430 __in unsigned int pushed) 431 { 432 efx_nic_t *enp = etp->et_enp; 433 const efx_tx_ops_t *etxop = enp->en_etxop; 434 435 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 436 437 etxop->etxo_qpush(etp, added, pushed); 438 } 439 440 __checkReturn efx_rc_t 441 efx_tx_qpace( 442 __in efx_txq_t *etp, 443 __in unsigned int ns) 444 { 445 efx_nic_t *enp = etp->et_enp; 446 const efx_tx_ops_t *etxop = enp->en_etxop; 447 efx_rc_t rc; 448 449 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 450 451 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 452 goto fail1; 453 454 return (0); 455 456 fail1: 457 EFSYS_PROBE1(fail1, efx_rc_t, rc); 458 return (rc); 459 } 460 461 __checkReturn efx_rc_t 462 efx_tx_qflush( 463 __in efx_txq_t *etp) 464 { 465 efx_nic_t *enp = etp->et_enp; 466 const efx_tx_ops_t *etxop = enp->en_etxop; 467 efx_rc_t rc; 468 469 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 470 471 if ((rc = etxop->etxo_qflush(etp)) != 0) 472 goto fail1; 473 474 return (0); 475 476 fail1: 477 EFSYS_PROBE1(fail1, efx_rc_t, rc); 478 return (rc); 479 } 480 481 void 482 efx_tx_qenable( 483 __in efx_txq_t *etp) 484 { 485 efx_nic_t *enp = etp->et_enp; 486 const efx_tx_ops_t *etxop = enp->en_etxop; 487 488 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 489 490 etxop->etxo_qenable(etp); 491 } 492 493 __checkReturn efx_rc_t 494 efx_tx_qpio_enable( 495 __in efx_txq_t *etp) 496 { 497 efx_nic_t *enp = etp->et_enp; 498 const efx_tx_ops_t *etxop = enp->en_etxop; 499 efx_rc_t rc; 500 501 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 502 503 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 504 rc = ENOTSUP; 505 goto fail1; 506 } 507 if (etxop->etxo_qpio_enable == NULL) { 508 rc = ENOTSUP; 509 goto fail2; 510 } 511 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 512 goto fail3; 513 514 return (0); 515 516 fail3: 517 EFSYS_PROBE(fail3); 518 fail2: 519 EFSYS_PROBE(fail2); 520 fail1: 521 EFSYS_PROBE1(fail1, efx_rc_t, rc); 522 return (rc); 523 } 524 525 void 526 efx_tx_qpio_disable( 527 __in efx_txq_t *etp) 528 { 529 efx_nic_t *enp = etp->et_enp; 530 const efx_tx_ops_t *etxop = enp->en_etxop; 531 532 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 533 534 if (etxop->etxo_qpio_disable != NULL) 535 etxop->etxo_qpio_disable(etp); 536 } 537 538 __checkReturn efx_rc_t 539 efx_tx_qpio_write( 540 __in efx_txq_t *etp, 541 __in_ecount(buf_length) uint8_t *buffer, 542 __in size_t buf_length, 543 __in size_t pio_buf_offset) 544 { 545 efx_nic_t *enp = etp->et_enp; 546 const efx_tx_ops_t *etxop = enp->en_etxop; 547 efx_rc_t rc; 548 549 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 550 551 if (etxop->etxo_qpio_write != NULL) { 552 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 553 pio_buf_offset)) != 0) 554 goto fail1; 555 return (0); 556 } 557 558 return (ENOTSUP); 559 560 fail1: 561 EFSYS_PROBE1(fail1, efx_rc_t, rc); 562 return (rc); 563 } 564 565 __checkReturn efx_rc_t 566 efx_tx_qpio_post( 567 __in efx_txq_t *etp, 568 __in size_t pkt_length, 569 __in unsigned int completed, 570 __inout unsigned int *addedp) 571 { 572 efx_nic_t *enp = etp->et_enp; 573 const efx_tx_ops_t *etxop = enp->en_etxop; 574 efx_rc_t rc; 575 576 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 577 578 if (etxop->etxo_qpio_post != NULL) { 579 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 580 addedp)) != 0) 581 goto fail1; 582 return (0); 583 } 584 585 return (ENOTSUP); 586 587 fail1: 588 EFSYS_PROBE1(fail1, efx_rc_t, rc); 589 return (rc); 590 } 591 592 __checkReturn efx_rc_t 593 efx_tx_qdesc_post( 594 __in efx_txq_t *etp, 595 __in_ecount(ndescs) efx_desc_t *ed, 596 __in unsigned int ndescs, 597 __in unsigned int completed, 598 __inout unsigned int *addedp) 599 { 600 efx_nic_t *enp = etp->et_enp; 601 const efx_tx_ops_t *etxop = enp->en_etxop; 602 603 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 604 605 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp)); 606 } 607 608 void 609 efx_tx_qdesc_dma_create( 610 __in efx_txq_t *etp, 611 __in efsys_dma_addr_t addr, 612 __in size_t size, 613 __in boolean_t eop, 614 __out efx_desc_t *edp) 615 { 616 efx_nic_t *enp = etp->et_enp; 617 const efx_tx_ops_t *etxop = enp->en_etxop; 618 619 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 620 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 621 622 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 623 } 624 625 void 626 efx_tx_qdesc_tso_create( 627 __in efx_txq_t *etp, 628 __in uint16_t ipv4_id, 629 __in uint32_t tcp_seq, 630 __in uint8_t tcp_flags, 631 __out efx_desc_t *edp) 632 { 633 efx_nic_t *enp = etp->et_enp; 634 const efx_tx_ops_t *etxop = enp->en_etxop; 635 636 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 637 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 638 639 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 640 } 641 642 void 643 efx_tx_qdesc_tso2_create( 644 __in efx_txq_t *etp, 645 __in uint16_t ipv4_id, 646 __in uint16_t outer_ipv4_id, 647 __in uint32_t tcp_seq, 648 __in uint16_t mss, 649 __out_ecount(count) efx_desc_t *edp, 650 __in int count) 651 { 652 efx_nic_t *enp = etp->et_enp; 653 const efx_tx_ops_t *etxop = enp->en_etxop; 654 655 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 656 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 657 658 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id, 659 tcp_seq, mss, edp, count); 660 } 661 662 void 663 efx_tx_qdesc_vlantci_create( 664 __in efx_txq_t *etp, 665 __in uint16_t tci, 666 __out efx_desc_t *edp) 667 { 668 efx_nic_t *enp = etp->et_enp; 669 const efx_tx_ops_t *etxop = enp->en_etxop; 670 671 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 672 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 673 674 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 675 } 676 677 void 678 efx_tx_qdesc_checksum_create( 679 __in efx_txq_t *etp, 680 __in uint16_t flags, 681 __out efx_desc_t *edp) 682 { 683 efx_nic_t *enp = etp->et_enp; 684 const efx_tx_ops_t *etxop = enp->en_etxop; 685 686 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 687 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL); 688 689 etxop->etxo_qdesc_checksum_create(etp, flags, edp); 690 } 691 692 #if EFSYS_OPT_QSTATS 693 void 694 efx_tx_qstats_update( 695 __in efx_txq_t *etp, 696 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 697 { 698 efx_nic_t *enp = etp->et_enp; 699 const efx_tx_ops_t *etxop = enp->en_etxop; 700 701 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 702 703 etxop->etxo_qstats_update(etp, stat); 704 } 705 #endif 706 707 #if EFSYS_OPT_SIENA 708 709 static __checkReturn efx_rc_t 710 siena_tx_init( 711 __in efx_nic_t *enp) 712 { 713 efx_oword_t oword; 714 715 /* 716 * Disable the timer-based TX DMA backoff and allow TX DMA to be 717 * controlled by the RX FIFO fill level (although always allow a 718 * minimal trickle). 719 */ 720 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 721 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 722 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 723 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 724 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 725 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 726 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 727 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 728 729 /* 730 * Filter all packets less than 14 bytes to avoid parsing 731 * errors. 732 */ 733 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 734 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 735 736 /* 737 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 738 * descriptors (which is bad). 739 */ 740 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 741 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 742 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 743 744 return (0); 745 } 746 747 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 748 do { \ 749 unsigned int id; \ 750 size_t offset; \ 751 efx_qword_t qword; \ 752 \ 753 id = (_added)++ & (_etp)->et_mask; \ 754 offset = id * sizeof (efx_qword_t); \ 755 \ 756 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 757 unsigned int, id, efsys_dma_addr_t, (_addr), \ 758 size_t, (_size), boolean_t, (_eop)); \ 759 \ 760 EFX_POPULATE_QWORD_4(qword, \ 761 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 762 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 763 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 764 (uint32_t)((_addr) & 0xffffffff), \ 765 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 766 (uint32_t)((_addr) >> 32)); \ 767 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 768 \ 769 _NOTE(CONSTANTCONDITION) \ 770 } while (B_FALSE) 771 772 static __checkReturn efx_rc_t 773 siena_tx_qpost( 774 __in efx_txq_t *etp, 775 __in_ecount(ndescs) efx_buffer_t *eb, 776 __in unsigned int ndescs, 777 __in unsigned int completed, 778 __inout unsigned int *addedp) 779 { 780 unsigned int added = *addedp; 781 unsigned int i; 782 783 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) 784 return (ENOSPC); 785 786 for (i = 0; i < ndescs; i++) { 787 efx_buffer_t *ebp = &eb[i]; 788 efsys_dma_addr_t start = ebp->eb_addr; 789 size_t size = ebp->eb_size; 790 efsys_dma_addr_t end = start + size; 791 792 /* 793 * Fragments must not span 4k boundaries. 794 * Here it is a stricter requirement than the maximum length. 795 */ 796 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1, 797 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); 798 799 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 800 } 801 802 EFX_TX_QSTAT_INCR(etp, TX_POST); 803 804 *addedp = added; 805 return (0); 806 } 807 808 static void 809 siena_tx_qpush( 810 __in efx_txq_t *etp, 811 __in unsigned int added, 812 __in unsigned int pushed) 813 { 814 efx_nic_t *enp = etp->et_enp; 815 uint32_t wptr; 816 efx_dword_t dword; 817 efx_oword_t oword; 818 819 /* Push the populated descriptors out */ 820 wptr = added & etp->et_mask; 821 822 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 823 824 /* Only write the third DWORD */ 825 EFX_POPULATE_DWORD_1(dword, 826 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 827 828 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 829 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 830 wptr, pushed & etp->et_mask); 831 EFSYS_PIO_WRITE_BARRIER(); 832 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 833 etp->et_index, &dword, B_FALSE); 834 } 835 836 #define EFX_MAX_PACE_VALUE 20 837 #define EFX_TX_PACE_CLOCK_BASE 104 838 839 static __checkReturn efx_rc_t 840 siena_tx_qpace( 841 __in efx_txq_t *etp, 842 __in unsigned int ns) 843 { 844 efx_nic_t *enp = etp->et_enp; 845 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 846 efx_oword_t oword; 847 unsigned int pace_val; 848 unsigned int timer_period; 849 efx_rc_t rc; 850 851 if (ns == 0) { 852 pace_val = 0; 853 } else { 854 /* 855 * The pace_val to write into the table is s.t 856 * ns <= timer_period * (2 ^ pace_val) 857 */ 858 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 859 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 860 if ((timer_period << pace_val) >= ns) 861 break; 862 } 863 } 864 if (pace_val > EFX_MAX_PACE_VALUE) { 865 rc = EINVAL; 866 goto fail1; 867 } 868 869 /* Update the pacing table */ 870 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 871 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 872 &oword, B_TRUE); 873 874 return (0); 875 876 fail1: 877 EFSYS_PROBE1(fail1, efx_rc_t, rc); 878 879 return (rc); 880 } 881 882 static __checkReturn efx_rc_t 883 siena_tx_qflush( 884 __in efx_txq_t *etp) 885 { 886 efx_nic_t *enp = etp->et_enp; 887 efx_oword_t oword; 888 uint32_t label; 889 890 efx_tx_qpace(etp, 0); 891 892 label = etp->et_index; 893 894 /* Flush the queue */ 895 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 896 FRF_AZ_TX_FLUSH_DESCQ, label); 897 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 898 899 return (0); 900 } 901 902 static void 903 siena_tx_qenable( 904 __in efx_txq_t *etp) 905 { 906 efx_nic_t *enp = etp->et_enp; 907 efx_oword_t oword; 908 909 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 910 etp->et_index, &oword, B_TRUE); 911 912 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 913 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 914 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 915 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 916 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 917 918 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 919 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 920 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 921 922 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 923 etp->et_index, &oword, B_TRUE); 924 } 925 926 static __checkReturn efx_rc_t 927 siena_tx_qcreate( 928 __in efx_nic_t *enp, 929 __in unsigned int index, 930 __in unsigned int label, 931 __in efsys_mem_t *esmp, 932 __in size_t ndescs, 933 __in uint32_t id, 934 __in uint16_t flags, 935 __in efx_evq_t *eep, 936 __in efx_txq_t *etp, 937 __out unsigned int *addedp) 938 { 939 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 940 efx_oword_t oword; 941 uint32_t size; 942 uint16_t inner_csum; 943 efx_rc_t rc; 944 945 _NOTE(ARGUNUSED(esmp)) 946 947 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 948 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 949 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 950 951 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs)); 952 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 953 954 if (!ISP2(ndescs) || 955 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) { 956 rc = EINVAL; 957 goto fail1; 958 } 959 if (index >= encp->enc_txq_limit) { 960 rc = EINVAL; 961 goto fail2; 962 } 963 for (size = 0; 964 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS); 965 size++) 966 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS)) 967 break; 968 if (id + (1 << size) >= encp->enc_buftbl_limit) { 969 rc = EINVAL; 970 goto fail3; 971 } 972 973 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; 974 if ((flags & inner_csum) != 0) { 975 rc = EINVAL; 976 goto fail4; 977 } 978 979 /* Set up the new descriptor queue */ 980 *addedp = 0; 981 982 EFX_POPULATE_OWORD_6(oword, 983 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 984 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 985 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 986 FRF_AZ_TX_DESCQ_LABEL, label, 987 FRF_AZ_TX_DESCQ_SIZE, size, 988 FRF_AZ_TX_DESCQ_TYPE, 0); 989 990 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 991 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 992 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 993 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 994 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 995 996 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 997 etp->et_index, &oword, B_TRUE); 998 999 return (0); 1000 1001 fail4: 1002 EFSYS_PROBE(fail4); 1003 fail3: 1004 EFSYS_PROBE(fail3); 1005 fail2: 1006 EFSYS_PROBE(fail2); 1007 fail1: 1008 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1009 1010 return (rc); 1011 } 1012 1013 __checkReturn efx_rc_t 1014 siena_tx_qdesc_post( 1015 __in efx_txq_t *etp, 1016 __in_ecount(ndescs) efx_desc_t *ed, 1017 __in unsigned int ndescs, 1018 __in unsigned int completed, 1019 __inout unsigned int *addedp) 1020 { 1021 unsigned int added = *addedp; 1022 unsigned int i; 1023 efx_rc_t rc; 1024 1025 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 1026 rc = ENOSPC; 1027 goto fail1; 1028 } 1029 1030 for (i = 0; i < ndescs; i++) { 1031 efx_desc_t *edp = &ed[i]; 1032 unsigned int id; 1033 size_t offset; 1034 1035 id = added++ & etp->et_mask; 1036 offset = id * sizeof (efx_desc_t); 1037 1038 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 1039 } 1040 1041 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 1042 unsigned int, added, unsigned int, ndescs); 1043 1044 EFX_TX_QSTAT_INCR(etp, TX_POST); 1045 1046 *addedp = added; 1047 return (0); 1048 1049 fail1: 1050 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1051 return (rc); 1052 } 1053 1054 void 1055 siena_tx_qdesc_dma_create( 1056 __in efx_txq_t *etp, 1057 __in efsys_dma_addr_t addr, 1058 __in size_t size, 1059 __in boolean_t eop, 1060 __out efx_desc_t *edp) 1061 { 1062 /* 1063 * Fragments must not span 4k boundaries. 1064 * Here it is a stricter requirement than the maximum length. 1065 */ 1066 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1, 1067 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); 1068 1069 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1070 efsys_dma_addr_t, addr, 1071 size_t, size, boolean_t, eop); 1072 1073 EFX_POPULATE_QWORD_4(edp->ed_eq, 1074 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1075 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1076 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1077 (uint32_t)(addr & 0xffffffff), 1078 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1079 (uint32_t)(addr >> 32)); 1080 } 1081 1082 #endif /* EFSYS_OPT_SIENA */ 1083 1084 #if EFSYS_OPT_QSTATS 1085 #if EFSYS_OPT_NAMES 1086 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ 1087 static const char * const __efx_tx_qstat_name[] = { 1088 "post", 1089 "post_pio", 1090 }; 1091 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1092 1093 const char * 1094 efx_tx_qstat_name( 1095 __in efx_nic_t *enp, 1096 __in unsigned int id) 1097 { 1098 _NOTE(ARGUNUSED(enp)) 1099 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1100 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1101 1102 return (__efx_tx_qstat_name[id]); 1103 } 1104 #endif /* EFSYS_OPT_NAMES */ 1105 #endif /* EFSYS_OPT_QSTATS */ 1106 1107 #if EFSYS_OPT_SIENA 1108 1109 #if EFSYS_OPT_QSTATS 1110 static void 1111 siena_tx_qstats_update( 1112 __in efx_txq_t *etp, 1113 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1114 { 1115 unsigned int id; 1116 1117 for (id = 0; id < TX_NQSTATS; id++) { 1118 efsys_stat_t *essp = &stat[id]; 1119 1120 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1121 etp->et_stat[id] = 0; 1122 } 1123 } 1124 #endif /* EFSYS_OPT_QSTATS */ 1125 1126 static void 1127 siena_tx_qdestroy( 1128 __in efx_txq_t *etp) 1129 { 1130 efx_nic_t *enp = etp->et_enp; 1131 efx_oword_t oword; 1132 1133 /* Purge descriptor queue */ 1134 EFX_ZERO_OWORD(oword); 1135 1136 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1137 etp->et_index, &oword, B_TRUE); 1138 } 1139 1140 static void 1141 siena_tx_fini( 1142 __in efx_nic_t *enp) 1143 { 1144 _NOTE(ARGUNUSED(enp)) 1145 } 1146 1147 #endif /* EFSYS_OPT_SIENA */ 1148