1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2007-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * The views and conclusions contained in the software and documentation are 29 * those of the authors and should not be interpreted as representing official 30 * policies, either expressed or implied, of the FreeBSD Project. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "efx.h" 37 #include "efx_impl.h" 38 39 #if EFSYS_OPT_QSTATS 40 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 41 do { \ 42 (_etp)->et_stat[_stat]++; \ 43 _NOTE(CONSTANTCONDITION) \ 44 } while (B_FALSE) 45 #else 46 #define EFX_TX_QSTAT_INCR(_etp, _stat) 47 #endif 48 49 #if EFSYS_OPT_SIENA 50 51 static __checkReturn efx_rc_t 52 siena_tx_init( 53 __in efx_nic_t *enp); 54 55 static void 56 siena_tx_fini( 57 __in efx_nic_t *enp); 58 59 static __checkReturn efx_rc_t 60 siena_tx_qcreate( 61 __in efx_nic_t *enp, 62 __in unsigned int index, 63 __in unsigned int label, 64 __in efsys_mem_t *esmp, 65 __in size_t ndescs, 66 __in uint32_t id, 67 __in uint16_t flags, 68 __in efx_evq_t *eep, 69 __in efx_txq_t *etp, 70 __out unsigned int *addedp); 71 72 static void 73 siena_tx_qdestroy( 74 __in efx_txq_t *etp); 75 76 static __checkReturn efx_rc_t 77 siena_tx_qpost( 78 __in efx_txq_t *etp, 79 __in_ecount(ndescs) efx_buffer_t *eb, 80 __in unsigned int ndescs, 81 __in unsigned int completed, 82 __inout unsigned int *addedp); 83 84 static void 85 siena_tx_qpush( 86 __in efx_txq_t *etp, 87 __in unsigned int added, 88 __in unsigned int pushed); 89 90 static __checkReturn efx_rc_t 91 siena_tx_qpace( 92 __in efx_txq_t *etp, 93 __in unsigned int ns); 94 95 static __checkReturn efx_rc_t 96 siena_tx_qflush( 97 __in efx_txq_t *etp); 98 99 static void 100 siena_tx_qenable( 101 __in efx_txq_t *etp); 102 103 __checkReturn efx_rc_t 104 siena_tx_qdesc_post( 105 __in efx_txq_t *etp, 106 __in_ecount(ndescs) efx_desc_t *ed, 107 __in unsigned int ndescs, 108 __in unsigned int completed, 109 __inout unsigned int *addedp); 110 111 void 112 siena_tx_qdesc_dma_create( 113 __in efx_txq_t *etp, 114 __in efsys_dma_addr_t addr, 115 __in size_t size, 116 __in boolean_t eop, 117 __out efx_desc_t *edp); 118 119 #if EFSYS_OPT_QSTATS 120 static void 121 siena_tx_qstats_update( 122 __in efx_txq_t *etp, 123 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 124 #endif 125 126 #endif /* EFSYS_OPT_SIENA */ 127 128 129 #if EFSYS_OPT_SIENA 130 static const efx_tx_ops_t __efx_tx_siena_ops = { 131 siena_tx_init, /* etxo_init */ 132 siena_tx_fini, /* etxo_fini */ 133 siena_tx_qcreate, /* etxo_qcreate */ 134 siena_tx_qdestroy, /* etxo_qdestroy */ 135 siena_tx_qpost, /* etxo_qpost */ 136 siena_tx_qpush, /* etxo_qpush */ 137 siena_tx_qpace, /* etxo_qpace */ 138 siena_tx_qflush, /* etxo_qflush */ 139 siena_tx_qenable, /* etxo_qenable */ 140 NULL, /* etxo_qpio_enable */ 141 NULL, /* etxo_qpio_disable */ 142 NULL, /* etxo_qpio_write */ 143 NULL, /* etxo_qpio_post */ 144 siena_tx_qdesc_post, /* etxo_qdesc_post */ 145 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 146 NULL, /* etxo_qdesc_tso_create */ 147 NULL, /* etxo_qdesc_tso2_create */ 148 NULL, /* etxo_qdesc_vlantci_create */ 149 NULL, /* etxo_qdesc_checksum_create */ 150 #if EFSYS_OPT_QSTATS 151 siena_tx_qstats_update, /* etxo_qstats_update */ 152 #endif 153 }; 154 #endif /* EFSYS_OPT_SIENA */ 155 156 #if EFSYS_OPT_HUNTINGTON 157 static const efx_tx_ops_t __efx_tx_hunt_ops = { 158 ef10_tx_init, /* etxo_init */ 159 ef10_tx_fini, /* etxo_fini */ 160 ef10_tx_qcreate, /* etxo_qcreate */ 161 ef10_tx_qdestroy, /* etxo_qdestroy */ 162 ef10_tx_qpost, /* etxo_qpost */ 163 ef10_tx_qpush, /* etxo_qpush */ 164 ef10_tx_qpace, /* etxo_qpace */ 165 ef10_tx_qflush, /* etxo_qflush */ 166 ef10_tx_qenable, /* etxo_qenable */ 167 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 168 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 169 ef10_tx_qpio_write, /* etxo_qpio_write */ 170 ef10_tx_qpio_post, /* etxo_qpio_post */ 171 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 172 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 173 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 174 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 175 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 176 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 177 #if EFSYS_OPT_QSTATS 178 ef10_tx_qstats_update, /* etxo_qstats_update */ 179 #endif 180 }; 181 #endif /* EFSYS_OPT_HUNTINGTON */ 182 183 #if EFSYS_OPT_MEDFORD 184 static const efx_tx_ops_t __efx_tx_medford_ops = { 185 ef10_tx_init, /* etxo_init */ 186 ef10_tx_fini, /* etxo_fini */ 187 ef10_tx_qcreate, /* etxo_qcreate */ 188 ef10_tx_qdestroy, /* etxo_qdestroy */ 189 ef10_tx_qpost, /* etxo_qpost */ 190 ef10_tx_qpush, /* etxo_qpush */ 191 ef10_tx_qpace, /* etxo_qpace */ 192 ef10_tx_qflush, /* etxo_qflush */ 193 ef10_tx_qenable, /* etxo_qenable */ 194 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 195 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 196 ef10_tx_qpio_write, /* etxo_qpio_write */ 197 ef10_tx_qpio_post, /* etxo_qpio_post */ 198 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 199 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 200 NULL, /* etxo_qdesc_tso_create */ 201 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 202 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 203 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 204 #if EFSYS_OPT_QSTATS 205 ef10_tx_qstats_update, /* etxo_qstats_update */ 206 #endif 207 }; 208 #endif /* EFSYS_OPT_MEDFORD */ 209 210 #if EFSYS_OPT_MEDFORD2 211 static const efx_tx_ops_t __efx_tx_medford2_ops = { 212 ef10_tx_init, /* etxo_init */ 213 ef10_tx_fini, /* etxo_fini */ 214 ef10_tx_qcreate, /* etxo_qcreate */ 215 ef10_tx_qdestroy, /* etxo_qdestroy */ 216 ef10_tx_qpost, /* etxo_qpost */ 217 ef10_tx_qpush, /* etxo_qpush */ 218 ef10_tx_qpace, /* etxo_qpace */ 219 ef10_tx_qflush, /* etxo_qflush */ 220 ef10_tx_qenable, /* etxo_qenable */ 221 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 222 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 223 ef10_tx_qpio_write, /* etxo_qpio_write */ 224 ef10_tx_qpio_post, /* etxo_qpio_post */ 225 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 226 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 227 NULL, /* etxo_qdesc_tso_create */ 228 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 229 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 230 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 231 #if EFSYS_OPT_QSTATS 232 ef10_tx_qstats_update, /* etxo_qstats_update */ 233 #endif 234 }; 235 #endif /* EFSYS_OPT_MEDFORD2 */ 236 237 238 __checkReturn efx_rc_t 239 efx_tx_init( 240 __in efx_nic_t *enp) 241 { 242 const efx_tx_ops_t *etxop; 243 efx_rc_t rc; 244 245 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 246 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 247 248 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 249 rc = EINVAL; 250 goto fail1; 251 } 252 253 if (enp->en_mod_flags & EFX_MOD_TX) { 254 rc = EINVAL; 255 goto fail2; 256 } 257 258 switch (enp->en_family) { 259 #if EFSYS_OPT_SIENA 260 case EFX_FAMILY_SIENA: 261 etxop = &__efx_tx_siena_ops; 262 break; 263 #endif /* EFSYS_OPT_SIENA */ 264 265 #if EFSYS_OPT_HUNTINGTON 266 case EFX_FAMILY_HUNTINGTON: 267 etxop = &__efx_tx_hunt_ops; 268 break; 269 #endif /* EFSYS_OPT_HUNTINGTON */ 270 271 #if EFSYS_OPT_MEDFORD 272 case EFX_FAMILY_MEDFORD: 273 etxop = &__efx_tx_medford_ops; 274 break; 275 #endif /* EFSYS_OPT_MEDFORD */ 276 277 #if EFSYS_OPT_MEDFORD2 278 case EFX_FAMILY_MEDFORD2: 279 etxop = &__efx_tx_medford2_ops; 280 break; 281 #endif /* EFSYS_OPT_MEDFORD2 */ 282 283 default: 284 EFSYS_ASSERT(0); 285 rc = ENOTSUP; 286 goto fail3; 287 } 288 289 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 290 291 if ((rc = etxop->etxo_init(enp)) != 0) 292 goto fail4; 293 294 enp->en_etxop = etxop; 295 enp->en_mod_flags |= EFX_MOD_TX; 296 return (0); 297 298 fail4: 299 EFSYS_PROBE(fail4); 300 fail3: 301 EFSYS_PROBE(fail3); 302 fail2: 303 EFSYS_PROBE(fail2); 304 fail1: 305 EFSYS_PROBE1(fail1, efx_rc_t, rc); 306 307 enp->en_etxop = NULL; 308 enp->en_mod_flags &= ~EFX_MOD_TX; 309 return (rc); 310 } 311 312 void 313 efx_tx_fini( 314 __in efx_nic_t *enp) 315 { 316 const efx_tx_ops_t *etxop = enp->en_etxop; 317 318 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 319 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 320 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 321 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 322 323 etxop->etxo_fini(enp); 324 325 enp->en_etxop = NULL; 326 enp->en_mod_flags &= ~EFX_MOD_TX; 327 } 328 329 __checkReturn efx_rc_t 330 efx_tx_qcreate( 331 __in efx_nic_t *enp, 332 __in unsigned int index, 333 __in unsigned int label, 334 __in efsys_mem_t *esmp, 335 __in size_t ndescs, 336 __in uint32_t id, 337 __in uint16_t flags, 338 __in efx_evq_t *eep, 339 __deref_out efx_txq_t **etpp, 340 __out unsigned int *addedp) 341 { 342 const efx_tx_ops_t *etxop = enp->en_etxop; 343 efx_txq_t *etp; 344 efx_rc_t rc; 345 346 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 347 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 348 349 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, 350 enp->en_nic_cfg.enc_txq_limit); 351 352 /* Allocate an TXQ object */ 353 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 354 355 if (etp == NULL) { 356 rc = ENOMEM; 357 goto fail1; 358 } 359 360 etp->et_magic = EFX_TXQ_MAGIC; 361 etp->et_enp = enp; 362 etp->et_index = index; 363 etp->et_mask = ndescs - 1; 364 etp->et_esmp = esmp; 365 366 /* Initial descriptor index may be modified by etxo_qcreate */ 367 *addedp = 0; 368 369 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 370 ndescs, id, flags, eep, etp, addedp)) != 0) 371 goto fail2; 372 373 enp->en_tx_qcount++; 374 *etpp = etp; 375 376 return (0); 377 378 fail2: 379 EFSYS_PROBE(fail2); 380 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 381 fail1: 382 EFSYS_PROBE1(fail1, efx_rc_t, rc); 383 return (rc); 384 } 385 386 void 387 efx_tx_qdestroy( 388 __in efx_txq_t *etp) 389 { 390 efx_nic_t *enp = etp->et_enp; 391 const efx_tx_ops_t *etxop = enp->en_etxop; 392 393 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 394 395 EFSYS_ASSERT(enp->en_tx_qcount != 0); 396 --enp->en_tx_qcount; 397 398 etxop->etxo_qdestroy(etp); 399 400 /* Free the TXQ object */ 401 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 402 } 403 404 __checkReturn efx_rc_t 405 efx_tx_qpost( 406 __in efx_txq_t *etp, 407 __in_ecount(ndescs) efx_buffer_t *eb, 408 __in unsigned int ndescs, 409 __in unsigned int completed, 410 __inout unsigned int *addedp) 411 { 412 efx_nic_t *enp = etp->et_enp; 413 const efx_tx_ops_t *etxop = enp->en_etxop; 414 efx_rc_t rc; 415 416 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 417 418 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0) 419 goto fail1; 420 421 return (0); 422 423 fail1: 424 EFSYS_PROBE1(fail1, efx_rc_t, rc); 425 return (rc); 426 } 427 428 void 429 efx_tx_qpush( 430 __in efx_txq_t *etp, 431 __in unsigned int added, 432 __in unsigned int pushed) 433 { 434 efx_nic_t *enp = etp->et_enp; 435 const efx_tx_ops_t *etxop = enp->en_etxop; 436 437 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 438 439 etxop->etxo_qpush(etp, added, pushed); 440 } 441 442 __checkReturn efx_rc_t 443 efx_tx_qpace( 444 __in efx_txq_t *etp, 445 __in unsigned int ns) 446 { 447 efx_nic_t *enp = etp->et_enp; 448 const efx_tx_ops_t *etxop = enp->en_etxop; 449 efx_rc_t rc; 450 451 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 452 453 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 454 goto fail1; 455 456 return (0); 457 458 fail1: 459 EFSYS_PROBE1(fail1, efx_rc_t, rc); 460 return (rc); 461 } 462 463 __checkReturn efx_rc_t 464 efx_tx_qflush( 465 __in efx_txq_t *etp) 466 { 467 efx_nic_t *enp = etp->et_enp; 468 const efx_tx_ops_t *etxop = enp->en_etxop; 469 efx_rc_t rc; 470 471 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 472 473 if ((rc = etxop->etxo_qflush(etp)) != 0) 474 goto fail1; 475 476 return (0); 477 478 fail1: 479 EFSYS_PROBE1(fail1, efx_rc_t, rc); 480 return (rc); 481 } 482 483 void 484 efx_tx_qenable( 485 __in efx_txq_t *etp) 486 { 487 efx_nic_t *enp = etp->et_enp; 488 const efx_tx_ops_t *etxop = enp->en_etxop; 489 490 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 491 492 etxop->etxo_qenable(etp); 493 } 494 495 __checkReturn efx_rc_t 496 efx_tx_qpio_enable( 497 __in efx_txq_t *etp) 498 { 499 efx_nic_t *enp = etp->et_enp; 500 const efx_tx_ops_t *etxop = enp->en_etxop; 501 efx_rc_t rc; 502 503 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 504 505 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 506 rc = ENOTSUP; 507 goto fail1; 508 } 509 if (etxop->etxo_qpio_enable == NULL) { 510 rc = ENOTSUP; 511 goto fail2; 512 } 513 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 514 goto fail3; 515 516 return (0); 517 518 fail3: 519 EFSYS_PROBE(fail3); 520 fail2: 521 EFSYS_PROBE(fail2); 522 fail1: 523 EFSYS_PROBE1(fail1, efx_rc_t, rc); 524 return (rc); 525 } 526 527 void 528 efx_tx_qpio_disable( 529 __in efx_txq_t *etp) 530 { 531 efx_nic_t *enp = etp->et_enp; 532 const efx_tx_ops_t *etxop = enp->en_etxop; 533 534 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 535 536 if (etxop->etxo_qpio_disable != NULL) 537 etxop->etxo_qpio_disable(etp); 538 } 539 540 __checkReturn efx_rc_t 541 efx_tx_qpio_write( 542 __in efx_txq_t *etp, 543 __in_ecount(buf_length) uint8_t *buffer, 544 __in size_t buf_length, 545 __in size_t pio_buf_offset) 546 { 547 efx_nic_t *enp = etp->et_enp; 548 const efx_tx_ops_t *etxop = enp->en_etxop; 549 efx_rc_t rc; 550 551 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 552 553 if (etxop->etxo_qpio_write != NULL) { 554 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 555 pio_buf_offset)) != 0) 556 goto fail1; 557 return (0); 558 } 559 560 return (ENOTSUP); 561 562 fail1: 563 EFSYS_PROBE1(fail1, efx_rc_t, rc); 564 return (rc); 565 } 566 567 __checkReturn efx_rc_t 568 efx_tx_qpio_post( 569 __in efx_txq_t *etp, 570 __in size_t pkt_length, 571 __in unsigned int completed, 572 __inout unsigned int *addedp) 573 { 574 efx_nic_t *enp = etp->et_enp; 575 const efx_tx_ops_t *etxop = enp->en_etxop; 576 efx_rc_t rc; 577 578 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 579 580 if (etxop->etxo_qpio_post != NULL) { 581 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 582 addedp)) != 0) 583 goto fail1; 584 return (0); 585 } 586 587 return (ENOTSUP); 588 589 fail1: 590 EFSYS_PROBE1(fail1, efx_rc_t, rc); 591 return (rc); 592 } 593 594 __checkReturn efx_rc_t 595 efx_tx_qdesc_post( 596 __in efx_txq_t *etp, 597 __in_ecount(ndescs) efx_desc_t *ed, 598 __in unsigned int ndescs, 599 __in unsigned int completed, 600 __inout unsigned int *addedp) 601 { 602 efx_nic_t *enp = etp->et_enp; 603 const efx_tx_ops_t *etxop = enp->en_etxop; 604 605 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 606 607 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp)); 608 } 609 610 void 611 efx_tx_qdesc_dma_create( 612 __in efx_txq_t *etp, 613 __in efsys_dma_addr_t addr, 614 __in size_t size, 615 __in boolean_t eop, 616 __out efx_desc_t *edp) 617 { 618 efx_nic_t *enp = etp->et_enp; 619 const efx_tx_ops_t *etxop = enp->en_etxop; 620 621 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 622 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 623 624 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 625 } 626 627 void 628 efx_tx_qdesc_tso_create( 629 __in efx_txq_t *etp, 630 __in uint16_t ipv4_id, 631 __in uint32_t tcp_seq, 632 __in uint8_t tcp_flags, 633 __out efx_desc_t *edp) 634 { 635 efx_nic_t *enp = etp->et_enp; 636 const efx_tx_ops_t *etxop = enp->en_etxop; 637 638 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 639 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 640 641 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 642 } 643 644 void 645 efx_tx_qdesc_tso2_create( 646 __in efx_txq_t *etp, 647 __in uint16_t ipv4_id, 648 __in uint16_t outer_ipv4_id, 649 __in uint32_t tcp_seq, 650 __in uint16_t mss, 651 __out_ecount(count) efx_desc_t *edp, 652 __in int count) 653 { 654 efx_nic_t *enp = etp->et_enp; 655 const efx_tx_ops_t *etxop = enp->en_etxop; 656 657 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 658 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 659 660 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id, 661 tcp_seq, mss, edp, count); 662 } 663 664 void 665 efx_tx_qdesc_vlantci_create( 666 __in efx_txq_t *etp, 667 __in uint16_t tci, 668 __out efx_desc_t *edp) 669 { 670 efx_nic_t *enp = etp->et_enp; 671 const efx_tx_ops_t *etxop = enp->en_etxop; 672 673 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 674 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 675 676 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 677 } 678 679 void 680 efx_tx_qdesc_checksum_create( 681 __in efx_txq_t *etp, 682 __in uint16_t flags, 683 __out efx_desc_t *edp) 684 { 685 efx_nic_t *enp = etp->et_enp; 686 const efx_tx_ops_t *etxop = enp->en_etxop; 687 688 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 689 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL); 690 691 etxop->etxo_qdesc_checksum_create(etp, flags, edp); 692 } 693 694 695 #if EFSYS_OPT_QSTATS 696 void 697 efx_tx_qstats_update( 698 __in efx_txq_t *etp, 699 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 700 { 701 efx_nic_t *enp = etp->et_enp; 702 const efx_tx_ops_t *etxop = enp->en_etxop; 703 704 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 705 706 etxop->etxo_qstats_update(etp, stat); 707 } 708 #endif 709 710 711 #if EFSYS_OPT_SIENA 712 713 static __checkReturn efx_rc_t 714 siena_tx_init( 715 __in efx_nic_t *enp) 716 { 717 efx_oword_t oword; 718 719 /* 720 * Disable the timer-based TX DMA backoff and allow TX DMA to be 721 * controlled by the RX FIFO fill level (although always allow a 722 * minimal trickle). 723 */ 724 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 725 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 726 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 727 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 728 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 729 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 730 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 731 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 732 733 /* 734 * Filter all packets less than 14 bytes to avoid parsing 735 * errors. 736 */ 737 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 738 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 739 740 /* 741 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 742 * descriptors (which is bad). 743 */ 744 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 745 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 746 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 747 748 return (0); 749 } 750 751 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 752 do { \ 753 unsigned int id; \ 754 size_t offset; \ 755 efx_qword_t qword; \ 756 \ 757 id = (_added)++ & (_etp)->et_mask; \ 758 offset = id * sizeof (efx_qword_t); \ 759 \ 760 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 761 unsigned int, id, efsys_dma_addr_t, (_addr), \ 762 size_t, (_size), boolean_t, (_eop)); \ 763 \ 764 EFX_POPULATE_QWORD_4(qword, \ 765 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 766 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 767 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 768 (uint32_t)((_addr) & 0xffffffff), \ 769 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 770 (uint32_t)((_addr) >> 32)); \ 771 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 772 \ 773 _NOTE(CONSTANTCONDITION) \ 774 } while (B_FALSE) 775 776 static __checkReturn efx_rc_t 777 siena_tx_qpost( 778 __in efx_txq_t *etp, 779 __in_ecount(ndescs) efx_buffer_t *eb, 780 __in unsigned int ndescs, 781 __in unsigned int completed, 782 __inout unsigned int *addedp) 783 { 784 unsigned int added = *addedp; 785 unsigned int i; 786 787 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) 788 return (ENOSPC); 789 790 for (i = 0; i < ndescs; i++) { 791 efx_buffer_t *ebp = &eb[i]; 792 efsys_dma_addr_t start = ebp->eb_addr; 793 size_t size = ebp->eb_size; 794 efsys_dma_addr_t end = start + size; 795 796 /* 797 * Fragments must not span 4k boundaries. 798 * Here it is a stricter requirement than the maximum length. 799 */ 800 EFSYS_ASSERT(P2ROUNDUP(start + 1, 801 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); 802 803 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 804 } 805 806 EFX_TX_QSTAT_INCR(etp, TX_POST); 807 808 *addedp = added; 809 return (0); 810 } 811 812 static void 813 siena_tx_qpush( 814 __in efx_txq_t *etp, 815 __in unsigned int added, 816 __in unsigned int pushed) 817 { 818 efx_nic_t *enp = etp->et_enp; 819 uint32_t wptr; 820 efx_dword_t dword; 821 efx_oword_t oword; 822 823 /* Push the populated descriptors out */ 824 wptr = added & etp->et_mask; 825 826 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 827 828 /* Only write the third DWORD */ 829 EFX_POPULATE_DWORD_1(dword, 830 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 831 832 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 833 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 834 wptr, pushed & etp->et_mask); 835 EFSYS_PIO_WRITE_BARRIER(); 836 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 837 etp->et_index, &dword, B_FALSE); 838 } 839 840 #define EFX_MAX_PACE_VALUE 20 841 #define EFX_TX_PACE_CLOCK_BASE 104 842 843 static __checkReturn efx_rc_t 844 siena_tx_qpace( 845 __in efx_txq_t *etp, 846 __in unsigned int ns) 847 { 848 efx_nic_t *enp = etp->et_enp; 849 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 850 efx_oword_t oword; 851 unsigned int pace_val; 852 unsigned int timer_period; 853 efx_rc_t rc; 854 855 if (ns == 0) { 856 pace_val = 0; 857 } else { 858 /* 859 * The pace_val to write into the table is s.t 860 * ns <= timer_period * (2 ^ pace_val) 861 */ 862 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult; 863 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 864 if ((timer_period << pace_val) >= ns) 865 break; 866 } 867 } 868 if (pace_val > EFX_MAX_PACE_VALUE) { 869 rc = EINVAL; 870 goto fail1; 871 } 872 873 /* Update the pacing table */ 874 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 875 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 876 &oword, B_TRUE); 877 878 return (0); 879 880 fail1: 881 EFSYS_PROBE1(fail1, efx_rc_t, rc); 882 883 return (rc); 884 } 885 886 static __checkReturn efx_rc_t 887 siena_tx_qflush( 888 __in efx_txq_t *etp) 889 { 890 efx_nic_t *enp = etp->et_enp; 891 efx_oword_t oword; 892 uint32_t label; 893 894 efx_tx_qpace(etp, 0); 895 896 label = etp->et_index; 897 898 /* Flush the queue */ 899 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 900 FRF_AZ_TX_FLUSH_DESCQ, label); 901 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 902 903 return (0); 904 } 905 906 static void 907 siena_tx_qenable( 908 __in efx_txq_t *etp) 909 { 910 efx_nic_t *enp = etp->et_enp; 911 efx_oword_t oword; 912 913 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 914 etp->et_index, &oword, B_TRUE); 915 916 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 917 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 918 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 919 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 920 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 921 922 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 923 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 924 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 925 926 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 927 etp->et_index, &oword, B_TRUE); 928 } 929 930 static __checkReturn efx_rc_t 931 siena_tx_qcreate( 932 __in efx_nic_t *enp, 933 __in unsigned int index, 934 __in unsigned int label, 935 __in efsys_mem_t *esmp, 936 __in size_t ndescs, 937 __in uint32_t id, 938 __in uint16_t flags, 939 __in efx_evq_t *eep, 940 __in efx_txq_t *etp, 941 __out unsigned int *addedp) 942 { 943 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 944 efx_oword_t oword; 945 uint32_t size; 946 uint16_t inner_csum; 947 efx_rc_t rc; 948 949 _NOTE(ARGUNUSED(esmp)) 950 951 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 952 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 953 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 954 955 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs)); 956 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS)); 957 958 if (!ISP2(ndescs) || 959 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) { 960 rc = EINVAL; 961 goto fail1; 962 } 963 if (index >= encp->enc_txq_limit) { 964 rc = EINVAL; 965 goto fail2; 966 } 967 for (size = 0; 968 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS); 969 size++) 970 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS)) 971 break; 972 if (id + (1 << size) >= encp->enc_buftbl_limit) { 973 rc = EINVAL; 974 goto fail3; 975 } 976 977 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; 978 if ((flags & inner_csum) != 0) { 979 rc = EINVAL; 980 goto fail4; 981 } 982 983 /* Set up the new descriptor queue */ 984 *addedp = 0; 985 986 EFX_POPULATE_OWORD_6(oword, 987 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 988 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 989 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 990 FRF_AZ_TX_DESCQ_LABEL, label, 991 FRF_AZ_TX_DESCQ_SIZE, size, 992 FRF_AZ_TX_DESCQ_TYPE, 0); 993 994 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 995 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 996 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 997 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 998 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 999 1000 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1001 etp->et_index, &oword, B_TRUE); 1002 1003 return (0); 1004 1005 fail4: 1006 EFSYS_PROBE(fail4); 1007 fail3: 1008 EFSYS_PROBE(fail3); 1009 fail2: 1010 EFSYS_PROBE(fail2); 1011 fail1: 1012 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1013 1014 return (rc); 1015 } 1016 1017 __checkReturn efx_rc_t 1018 siena_tx_qdesc_post( 1019 __in efx_txq_t *etp, 1020 __in_ecount(ndescs) efx_desc_t *ed, 1021 __in unsigned int ndescs, 1022 __in unsigned int completed, 1023 __inout unsigned int *addedp) 1024 { 1025 unsigned int added = *addedp; 1026 unsigned int i; 1027 efx_rc_t rc; 1028 1029 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 1030 rc = ENOSPC; 1031 goto fail1; 1032 } 1033 1034 for (i = 0; i < ndescs; i++) { 1035 efx_desc_t *edp = &ed[i]; 1036 unsigned int id; 1037 size_t offset; 1038 1039 id = added++ & etp->et_mask; 1040 offset = id * sizeof (efx_desc_t); 1041 1042 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 1043 } 1044 1045 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 1046 unsigned int, added, unsigned int, ndescs); 1047 1048 EFX_TX_QSTAT_INCR(etp, TX_POST); 1049 1050 *addedp = added; 1051 return (0); 1052 1053 fail1: 1054 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1055 return (rc); 1056 } 1057 1058 void 1059 siena_tx_qdesc_dma_create( 1060 __in efx_txq_t *etp, 1061 __in efsys_dma_addr_t addr, 1062 __in size_t size, 1063 __in boolean_t eop, 1064 __out efx_desc_t *edp) 1065 { 1066 /* 1067 * Fragments must not span 4k boundaries. 1068 * Here it is a stricter requirement than the maximum length. 1069 */ 1070 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 1071 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); 1072 1073 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1074 efsys_dma_addr_t, addr, 1075 size_t, size, boolean_t, eop); 1076 1077 EFX_POPULATE_QWORD_4(edp->ed_eq, 1078 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1079 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1080 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1081 (uint32_t)(addr & 0xffffffff), 1082 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1083 (uint32_t)(addr >> 32)); 1084 } 1085 1086 #endif /* EFSYS_OPT_SIENA */ 1087 1088 #if EFSYS_OPT_QSTATS 1089 #if EFSYS_OPT_NAMES 1090 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ 1091 static const char * const __efx_tx_qstat_name[] = { 1092 "post", 1093 "post_pio", 1094 }; 1095 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1096 1097 const char * 1098 efx_tx_qstat_name( 1099 __in efx_nic_t *enp, 1100 __in unsigned int id) 1101 { 1102 _NOTE(ARGUNUSED(enp)) 1103 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1104 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1105 1106 return (__efx_tx_qstat_name[id]); 1107 } 1108 #endif /* EFSYS_OPT_NAMES */ 1109 #endif /* EFSYS_OPT_QSTATS */ 1110 1111 #if EFSYS_OPT_SIENA 1112 1113 #if EFSYS_OPT_QSTATS 1114 static void 1115 siena_tx_qstats_update( 1116 __in efx_txq_t *etp, 1117 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1118 { 1119 unsigned int id; 1120 1121 for (id = 0; id < TX_NQSTATS; id++) { 1122 efsys_stat_t *essp = &stat[id]; 1123 1124 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1125 etp->et_stat[id] = 0; 1126 } 1127 } 1128 #endif /* EFSYS_OPT_QSTATS */ 1129 1130 static void 1131 siena_tx_qdestroy( 1132 __in efx_txq_t *etp) 1133 { 1134 efx_nic_t *enp = etp->et_enp; 1135 efx_oword_t oword; 1136 1137 /* Purge descriptor queue */ 1138 EFX_ZERO_OWORD(oword); 1139 1140 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1141 etp->et_index, &oword, B_TRUE); 1142 } 1143 1144 static void 1145 siena_tx_fini( 1146 __in efx_nic_t *enp) 1147 { 1148 _NOTE(ARGUNUSED(enp)) 1149 } 1150 1151 #endif /* EFSYS_OPT_SIENA */ 1152