1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2020 Intel Corporation. 4 * 5 */ 6 7 /* 8 * This file contains HFI1 support for IPOIB SDMA functionality 9 */ 10 11 #include <linux/log2.h> 12 #include <linux/circ_buf.h> 13 14 #include "sdma.h" 15 #include "verbs.h" 16 #include "trace_ibhdrs.h" 17 #include "ipoib.h" 18 #include "trace_tx.h" 19 20 /* Add a convenience helper */ 21 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) 22 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size) 23 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size) 24 25 struct ipoib_txparms { 26 struct hfi1_devdata *dd; 27 struct rdma_ah_attr *ah_attr; 28 struct hfi1_ibport *ibp; 29 struct hfi1_ipoib_txq *txq; 30 union hfi1_ipoib_flow flow; 31 u32 dqpn; 32 u8 hdr_dwords; 33 u8 entropy; 34 }; 35 36 static struct ipoib_txreq * 37 hfi1_txreq_from_idx(struct hfi1_ipoib_circ_buf *r, u32 idx) 38 { 39 return (struct ipoib_txreq *)(r->items + (idx << r->shift)); 40 } 41 42 static u32 hfi1_ipoib_txreqs(const u64 sent, const u64 completed) 43 { 44 return sent - completed; 45 } 46 47 static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq) 48 { 49 return hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs, 50 txq->tx_ring.complete_txreqs); 51 } 52 53 static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq) 54 { 55 trace_hfi1_txq_stop(txq); 56 if (atomic_inc_return(&txq->tx_ring.stops) == 1) 57 netif_stop_subqueue(txq->priv->netdev, txq->q_idx); 58 } 59 60 static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq) 61 { 62 trace_hfi1_txq_wake(txq); 63 if (atomic_dec_and_test(&txq->tx_ring.stops)) 64 netif_wake_subqueue(txq->priv->netdev, txq->q_idx); 65 } 66 67 static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq) 68 { 69 return min_t(uint, txq->priv->netdev->tx_queue_len, 70 txq->tx_ring.max_items - 1); 71 } 72 73 static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq) 74 { 75 return min_t(uint, txq->priv->netdev->tx_queue_len, 76 txq->tx_ring.max_items) >> 1; 77 } 78 79 static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq) 80 { 81 ++txq->tx_ring.sent_txreqs; 82 if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) && 83 !atomic_xchg(&txq->tx_ring.ring_full, 1)) { 84 trace_hfi1_txq_full(txq); 85 hfi1_ipoib_stop_txq(txq); 86 } 87 } 88 89 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq) 90 { 91 struct net_device *dev = txq->priv->netdev; 92 93 /* If shutting down just return as queue state is irrelevant */ 94 if (unlikely(dev->reg_state != NETREG_REGISTERED)) 95 return; 96 97 /* 98 * When the queue has been drained to less than half full it will be 99 * restarted. 100 * The size of the txreq ring is fixed at initialization. 101 * The tx queue len can be adjusted upward while the interface is 102 * running. 103 * The tx queue len can be large enough to overflow the txreq_ring. 104 * Use the minimum of the current tx_queue_len or the rings max txreqs 105 * to protect against ring overflow. 106 */ 107 if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) && 108 atomic_xchg(&txq->tx_ring.ring_full, 0)) { 109 trace_hfi1_txq_xmit_unstopped(txq); 110 hfi1_ipoib_wake_txq(txq); 111 } 112 } 113 114 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget) 115 { 116 struct hfi1_ipoib_dev_priv *priv = tx->txq->priv; 117 118 if (likely(!tx->sdma_status)) { 119 dev_sw_netstats_tx_add(priv->netdev, 1, tx->skb->len); 120 } else { 121 ++priv->netdev->stats.tx_errors; 122 dd_dev_warn(priv->dd, 123 "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n", 124 __func__, tx->sdma_status, 125 le64_to_cpu(tx->sdma_hdr->pbc), tx->txq->q_idx, 126 tx->txq->sde->this_idx); 127 } 128 129 napi_consume_skb(tx->skb, budget); 130 tx->skb = NULL; 131 sdma_txclean(priv->dd, &tx->txreq); 132 } 133 134 static void hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq) 135 { 136 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; 137 int i; 138 struct ipoib_txreq *tx; 139 140 for (i = 0; i < tx_ring->max_items; i++) { 141 tx = hfi1_txreq_from_idx(tx_ring, i); 142 tx->complete = 0; 143 dev_kfree_skb_any(tx->skb); 144 tx->skb = NULL; 145 sdma_txclean(txq->priv->dd, &tx->txreq); 146 } 147 tx_ring->head = 0; 148 tx_ring->tail = 0; 149 tx_ring->complete_txreqs = 0; 150 tx_ring->sent_txreqs = 0; 151 tx_ring->avail = hfi1_ipoib_ring_hwat(txq); 152 } 153 154 static int hfi1_ipoib_poll_tx_ring(struct napi_struct *napi, int budget) 155 { 156 struct hfi1_ipoib_txq *txq = 157 container_of(napi, struct hfi1_ipoib_txq, napi); 158 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; 159 u32 head = tx_ring->head; 160 u32 max_tx = tx_ring->max_items; 161 int work_done; 162 struct ipoib_txreq *tx = hfi1_txreq_from_idx(tx_ring, head); 163 164 trace_hfi1_txq_poll(txq); 165 for (work_done = 0; work_done < budget; work_done++) { 166 /* See hfi1_ipoib_sdma_complete() */ 167 if (!smp_load_acquire(&tx->complete)) 168 break; 169 tx->complete = 0; 170 trace_hfi1_tx_produce(tx, head); 171 hfi1_ipoib_free_tx(tx, budget); 172 head = CIRC_NEXT(head, max_tx); 173 tx = hfi1_txreq_from_idx(tx_ring, head); 174 } 175 tx_ring->complete_txreqs += work_done; 176 177 /* Finished freeing tx items so store the head value. */ 178 smp_store_release(&tx_ring->head, head); 179 180 hfi1_ipoib_check_queue_stopped(txq); 181 182 if (work_done < budget) 183 napi_complete_done(napi, work_done); 184 185 return work_done; 186 } 187 188 static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status) 189 { 190 struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq); 191 192 trace_hfi1_txq_complete(tx->txq); 193 tx->sdma_status = status; 194 /* see hfi1_ipoib_poll_tx_ring */ 195 smp_store_release(&tx->complete, 1); 196 napi_schedule_irqoff(&tx->txq->napi); 197 } 198 199 static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx, 200 struct ipoib_txparms *txp) 201 { 202 struct hfi1_devdata *dd = txp->dd; 203 struct sdma_txreq *txreq = &tx->txreq; 204 struct sk_buff *skb = tx->skb; 205 int ret = 0; 206 int i; 207 208 if (skb_headlen(skb)) { 209 ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb)); 210 if (unlikely(ret)) 211 return ret; 212 } 213 214 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 215 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 216 217 ret = sdma_txadd_page(dd, 218 txreq, 219 skb_frag_page(frag), 220 frag->bv_offset, 221 skb_frag_size(frag)); 222 if (unlikely(ret)) 223 break; 224 } 225 226 return ret; 227 } 228 229 static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx, 230 struct ipoib_txparms *txp) 231 { 232 struct hfi1_devdata *dd = txp->dd; 233 struct sdma_txreq *txreq = &tx->txreq; 234 struct hfi1_sdma_header *sdma_hdr = tx->sdma_hdr; 235 u16 pkt_bytes = 236 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len; 237 int ret; 238 239 ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete); 240 if (unlikely(ret)) 241 return ret; 242 243 /* add pbc + headers */ 244 ret = sdma_txadd_kvaddr(dd, 245 txreq, 246 sdma_hdr, 247 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2)); 248 if (unlikely(ret)) 249 return ret; 250 251 /* add the ulp payload */ 252 return hfi1_ipoib_build_ulp_payload(tx, txp); 253 } 254 255 static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx, 256 struct ipoib_txparms *txp) 257 { 258 struct hfi1_ipoib_dev_priv *priv = tx->txq->priv; 259 struct hfi1_sdma_header *sdma_hdr = tx->sdma_hdr; 260 struct sk_buff *skb = tx->skb; 261 struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp); 262 struct rdma_ah_attr *ah_attr = txp->ah_attr; 263 struct ib_other_headers *ohdr; 264 struct ib_grh *grh; 265 u16 dwords; 266 u16 slid; 267 u16 dlid; 268 u16 lrh0; 269 u32 bth0; 270 u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 | 271 priv->netdev->dev_addr[2] << 8 | 272 priv->netdev->dev_addr[3]); 273 u16 payload_dwords; 274 u8 pad_cnt; 275 276 pad_cnt = -skb->len & 3; 277 278 /* Includes ICRC */ 279 payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC; 280 281 /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */ 282 txp->hdr_dwords = 7; 283 284 if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) { 285 grh = &sdma_hdr->hdr.ibh.u.l.grh; 286 txp->hdr_dwords += 287 hfi1_make_grh(txp->ibp, 288 grh, 289 rdma_ah_read_grh(ah_attr), 290 txp->hdr_dwords - LRH_9B_DWORDS, 291 payload_dwords); 292 lrh0 = HFI1_LRH_GRH; 293 ohdr = &sdma_hdr->hdr.ibh.u.l.oth; 294 } else { 295 lrh0 = HFI1_LRH_BTH; 296 ohdr = &sdma_hdr->hdr.ibh.u.oth; 297 } 298 299 lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4; 300 lrh0 |= (txp->flow.sc5 & 0xf) << 12; 301 302 dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B); 303 if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) { 304 slid = be16_to_cpu(IB_LID_PERMISSIVE); 305 } else { 306 u16 lid = (u16)ppd->lid; 307 308 if (lid) { 309 lid |= rdma_ah_get_path_bits(ah_attr) & 310 ((1 << ppd->lmc) - 1); 311 slid = lid; 312 } else { 313 slid = be16_to_cpu(IB_LID_PERMISSIVE); 314 } 315 } 316 317 /* Includes ICRC */ 318 dwords = txp->hdr_dwords + payload_dwords; 319 320 /* Build the lrh */ 321 sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B; 322 hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid); 323 324 /* Build the bth */ 325 bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey; 326 327 ohdr->bth[0] = cpu_to_be32(bth0); 328 ohdr->bth[1] = cpu_to_be32(txp->dqpn); 329 ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->tx_ring.sent_txreqs)); 330 331 /* Build the deth */ 332 ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey); 333 ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy << 334 HFI1_IPOIB_ENTROPY_SHIFT) | sqpn); 335 336 /* Construct the pbc. */ 337 sdma_hdr->pbc = 338 cpu_to_le64(create_pbc(ppd, 339 ib_is_sc5(txp->flow.sc5) << 340 PBC_DC_INFO_SHIFT, 341 0, 342 sc_to_vlt(priv->dd, txp->flow.sc5), 343 dwords - SIZE_OF_CRC + 344 (sizeof(sdma_hdr->pbc) >> 2))); 345 } 346 347 static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev, 348 struct sk_buff *skb, 349 struct ipoib_txparms *txp) 350 { 351 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 352 struct hfi1_ipoib_txq *txq = txp->txq; 353 struct ipoib_txreq *tx; 354 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; 355 u32 tail = tx_ring->tail; 356 int ret; 357 358 if (unlikely(!tx_ring->avail)) { 359 u32 head; 360 361 if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq)) 362 /* This shouldn't happen with a stopped queue */ 363 return ERR_PTR(-ENOMEM); 364 /* See hfi1_ipoib_poll_tx_ring() */ 365 head = smp_load_acquire(&tx_ring->head); 366 tx_ring->avail = 367 min_t(u32, hfi1_ipoib_ring_hwat(txq), 368 CIRC_CNT(head, tail, tx_ring->max_items)); 369 } else { 370 tx_ring->avail--; 371 } 372 tx = hfi1_txreq_from_idx(tx_ring, tail); 373 trace_hfi1_txq_alloc_tx(txq); 374 375 /* so that we can test if the sdma descriptors are there */ 376 tx->txreq.num_desc = 0; 377 tx->txq = txq; 378 tx->skb = skb; 379 INIT_LIST_HEAD(&tx->txreq.list); 380 381 hfi1_ipoib_build_ib_tx_headers(tx, txp); 382 383 ret = hfi1_ipoib_build_tx_desc(tx, txp); 384 if (likely(!ret)) { 385 if (txq->flow.as_int != txp->flow.as_int) { 386 txq->flow.tx_queue = txp->flow.tx_queue; 387 txq->flow.sc5 = txp->flow.sc5; 388 txq->sde = 389 sdma_select_engine_sc(priv->dd, 390 txp->flow.tx_queue, 391 txp->flow.sc5); 392 trace_hfi1_flow_switch(txq); 393 } 394 395 return tx; 396 } 397 398 sdma_txclean(priv->dd, &tx->txreq); 399 400 return ERR_PTR(ret); 401 } 402 403 static int hfi1_ipoib_submit_tx_list(struct net_device *dev, 404 struct hfi1_ipoib_txq *txq) 405 { 406 int ret; 407 u16 count_out; 408 409 ret = sdma_send_txlist(txq->sde, 410 iowait_get_ib_work(&txq->wait), 411 &txq->tx_list, 412 &count_out); 413 if (likely(!ret) || ret == -EBUSY || ret == -ECOMM) 414 return ret; 415 416 dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret); 417 418 return ret; 419 } 420 421 static int hfi1_ipoib_flush_tx_list(struct net_device *dev, 422 struct hfi1_ipoib_txq *txq) 423 { 424 int ret = 0; 425 426 if (!list_empty(&txq->tx_list)) { 427 /* Flush the current list */ 428 ret = hfi1_ipoib_submit_tx_list(dev, txq); 429 430 if (unlikely(ret)) 431 if (ret != -EBUSY) 432 ++dev->stats.tx_carrier_errors; 433 } 434 435 return ret; 436 } 437 438 static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq, 439 struct ipoib_txreq *tx) 440 { 441 int ret; 442 443 ret = sdma_send_txreq(txq->sde, 444 iowait_get_ib_work(&txq->wait), 445 &tx->txreq, 446 txq->pkts_sent); 447 if (likely(!ret)) { 448 txq->pkts_sent = true; 449 iowait_starve_clear(txq->pkts_sent, &txq->wait); 450 } 451 452 return ret; 453 } 454 455 static int hfi1_ipoib_send_dma_single(struct net_device *dev, 456 struct sk_buff *skb, 457 struct ipoib_txparms *txp) 458 { 459 struct hfi1_ipoib_txq *txq = txp->txq; 460 struct hfi1_ipoib_circ_buf *tx_ring; 461 struct ipoib_txreq *tx; 462 int ret; 463 464 tx = hfi1_ipoib_send_dma_common(dev, skb, txp); 465 if (IS_ERR(tx)) { 466 int ret = PTR_ERR(tx); 467 468 dev_kfree_skb_any(skb); 469 470 if (ret == -ENOMEM) 471 ++dev->stats.tx_errors; 472 else 473 ++dev->stats.tx_carrier_errors; 474 475 return NETDEV_TX_OK; 476 } 477 478 tx_ring = &txq->tx_ring; 479 trace_hfi1_tx_consume(tx, tx_ring->tail); 480 /* consume tx */ 481 smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items)); 482 ret = hfi1_ipoib_submit_tx(txq, tx); 483 if (likely(!ret)) { 484 tx_ok: 485 trace_sdma_output_ibhdr(txq->priv->dd, 486 &tx->sdma_hdr->hdr, 487 ib_is_sc5(txp->flow.sc5)); 488 hfi1_ipoib_check_queue_depth(txq); 489 return NETDEV_TX_OK; 490 } 491 492 txq->pkts_sent = false; 493 494 if (ret == -EBUSY || ret == -ECOMM) 495 goto tx_ok; 496 497 /* mark complete and kick napi tx */ 498 smp_store_release(&tx->complete, 1); 499 napi_schedule(&tx->txq->napi); 500 501 ++dev->stats.tx_carrier_errors; 502 503 return NETDEV_TX_OK; 504 } 505 506 static int hfi1_ipoib_send_dma_list(struct net_device *dev, 507 struct sk_buff *skb, 508 struct ipoib_txparms *txp) 509 { 510 struct hfi1_ipoib_txq *txq = txp->txq; 511 struct hfi1_ipoib_circ_buf *tx_ring; 512 struct ipoib_txreq *tx; 513 514 /* Has the flow change ? */ 515 if (txq->flow.as_int != txp->flow.as_int) { 516 int ret; 517 518 trace_hfi1_flow_flush(txq); 519 ret = hfi1_ipoib_flush_tx_list(dev, txq); 520 if (unlikely(ret)) { 521 if (ret == -EBUSY) 522 ++dev->stats.tx_dropped; 523 dev_kfree_skb_any(skb); 524 return NETDEV_TX_OK; 525 } 526 } 527 tx = hfi1_ipoib_send_dma_common(dev, skb, txp); 528 if (IS_ERR(tx)) { 529 int ret = PTR_ERR(tx); 530 531 dev_kfree_skb_any(skb); 532 533 if (ret == -ENOMEM) 534 ++dev->stats.tx_errors; 535 else 536 ++dev->stats.tx_carrier_errors; 537 538 return NETDEV_TX_OK; 539 } 540 541 tx_ring = &txq->tx_ring; 542 trace_hfi1_tx_consume(tx, tx_ring->tail); 543 /* consume tx */ 544 smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items)); 545 list_add_tail(&tx->txreq.list, &txq->tx_list); 546 547 hfi1_ipoib_check_queue_depth(txq); 548 549 trace_sdma_output_ibhdr(txq->priv->dd, 550 &tx->sdma_hdr->hdr, 551 ib_is_sc5(txp->flow.sc5)); 552 553 if (!netdev_xmit_more()) 554 (void)hfi1_ipoib_flush_tx_list(dev, txq); 555 556 return NETDEV_TX_OK; 557 } 558 559 static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb) 560 { 561 if (skb_transport_header_was_set(skb)) { 562 u8 *hdr = (u8 *)skb_transport_header(skb); 563 564 return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]); 565 } 566 567 return (u8)skb_get_queue_mapping(skb); 568 } 569 570 int hfi1_ipoib_send(struct net_device *dev, 571 struct sk_buff *skb, 572 struct ib_ah *address, 573 u32 dqpn) 574 { 575 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 576 struct ipoib_txparms txp; 577 struct rdma_netdev *rn = netdev_priv(dev); 578 579 if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) { 580 dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n", 581 skb->len, 582 rn->mtu + HFI1_IPOIB_ENCAP_LEN); 583 ++dev->stats.tx_dropped; 584 ++dev->stats.tx_errors; 585 dev_kfree_skb_any(skb); 586 return NETDEV_TX_OK; 587 } 588 589 txp.dd = priv->dd; 590 txp.ah_attr = &ibah_to_rvtah(address)->attr; 591 txp.ibp = to_iport(priv->device, priv->port_num); 592 txp.txq = &priv->txqs[skb_get_queue_mapping(skb)]; 593 txp.dqpn = dqpn; 594 txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)]; 595 txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb); 596 txp.entropy = hfi1_ipoib_calc_entropy(skb); 597 598 if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list)) 599 return hfi1_ipoib_send_dma_list(dev, skb, &txp); 600 601 return hfi1_ipoib_send_dma_single(dev, skb, &txp); 602 } 603 604 /* 605 * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function 606 * 607 * This function gets called from sdma_send_txreq() when there are not enough 608 * sdma descriptors available to send the packet. It adds Tx queue's wait 609 * structure to sdma engine's dmawait list to be woken up when descriptors 610 * become available. 611 */ 612 static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde, 613 struct iowait_work *wait, 614 struct sdma_txreq *txreq, 615 uint seq, 616 bool pkts_sent) 617 { 618 struct hfi1_ipoib_txq *txq = 619 container_of(wait->iow, struct hfi1_ipoib_txq, wait); 620 621 write_seqlock(&sde->waitlock); 622 623 if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) { 624 if (sdma_progress(sde, seq, txreq)) { 625 write_sequnlock(&sde->waitlock); 626 return -EAGAIN; 627 } 628 629 if (list_empty(&txreq->list)) 630 /* came from non-list submit */ 631 list_add_tail(&txreq->list, &txq->tx_list); 632 if (list_empty(&txq->wait.list)) { 633 struct hfi1_ibport *ibp = &sde->ppd->ibport_data; 634 635 if (!atomic_xchg(&txq->tx_ring.no_desc, 1)) { 636 trace_hfi1_txq_queued(txq); 637 hfi1_ipoib_stop_txq(txq); 638 } 639 ibp->rvp.n_dmawait++; 640 iowait_queue(pkts_sent, wait->iow, &sde->dmawait); 641 } 642 643 write_sequnlock(&sde->waitlock); 644 return -EBUSY; 645 } 646 647 write_sequnlock(&sde->waitlock); 648 return -EINVAL; 649 } 650 651 /* 652 * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function 653 * 654 * This function gets called when SDMA descriptors becomes available and Tx 655 * queue's wait structure was previously added to sdma engine's dmawait list. 656 */ 657 static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason) 658 { 659 struct hfi1_ipoib_txq *txq = 660 container_of(wait, struct hfi1_ipoib_txq, wait); 661 662 trace_hfi1_txq_wakeup(txq); 663 if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) 664 iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND); 665 } 666 667 static void hfi1_ipoib_flush_txq(struct work_struct *work) 668 { 669 struct iowait_work *ioww = 670 container_of(work, struct iowait_work, iowork); 671 struct iowait *wait = iowait_ioww_to_iow(ioww); 672 struct hfi1_ipoib_txq *txq = 673 container_of(wait, struct hfi1_ipoib_txq, wait); 674 struct net_device *dev = txq->priv->netdev; 675 676 if (likely(dev->reg_state == NETREG_REGISTERED) && 677 likely(!hfi1_ipoib_flush_tx_list(dev, txq))) 678 if (atomic_xchg(&txq->tx_ring.no_desc, 0)) 679 hfi1_ipoib_wake_txq(txq); 680 } 681 682 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) 683 { 684 struct net_device *dev = priv->netdev; 685 u32 tx_ring_size, tx_item_size; 686 struct hfi1_ipoib_circ_buf *tx_ring; 687 int i, j; 688 689 /* 690 * Ring holds 1 less than tx_ring_size 691 * Round up to next power of 2 in order to hold at least tx_queue_len 692 */ 693 tx_ring_size = roundup_pow_of_two(dev->tx_queue_len + 1); 694 tx_item_size = roundup_pow_of_two(sizeof(struct ipoib_txreq)); 695 696 priv->txqs = kcalloc_node(dev->num_tx_queues, 697 sizeof(struct hfi1_ipoib_txq), 698 GFP_KERNEL, 699 priv->dd->node); 700 if (!priv->txqs) 701 return -ENOMEM; 702 703 for (i = 0; i < dev->num_tx_queues; i++) { 704 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 705 struct ipoib_txreq *tx; 706 707 tx_ring = &txq->tx_ring; 708 iowait_init(&txq->wait, 709 0, 710 hfi1_ipoib_flush_txq, 711 NULL, 712 hfi1_ipoib_sdma_sleep, 713 hfi1_ipoib_sdma_wakeup, 714 NULL, 715 NULL); 716 txq->priv = priv; 717 txq->sde = NULL; 718 INIT_LIST_HEAD(&txq->tx_list); 719 atomic_set(&txq->tx_ring.stops, 0); 720 atomic_set(&txq->tx_ring.ring_full, 0); 721 atomic_set(&txq->tx_ring.no_desc, 0); 722 txq->q_idx = i; 723 txq->flow.tx_queue = 0xff; 724 txq->flow.sc5 = 0xff; 725 txq->pkts_sent = false; 726 727 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), 728 priv->dd->node); 729 730 txq->tx_ring.items = 731 kvzalloc_node(array_size(tx_ring_size, tx_item_size), 732 GFP_KERNEL, priv->dd->node); 733 if (!txq->tx_ring.items) 734 goto free_txqs; 735 736 txq->tx_ring.max_items = tx_ring_size; 737 txq->tx_ring.shift = ilog2(tx_item_size); 738 txq->tx_ring.avail = hfi1_ipoib_ring_hwat(txq); 739 tx_ring = &txq->tx_ring; 740 for (j = 0; j < tx_ring_size; j++) 741 hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr = 742 kzalloc_node(sizeof(*tx->sdma_hdr), 743 GFP_KERNEL, priv->dd->node); 744 745 netif_napi_add_tx(dev, &txq->napi, hfi1_ipoib_poll_tx_ring); 746 } 747 748 return 0; 749 750 free_txqs: 751 for (i--; i >= 0; i--) { 752 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 753 754 netif_napi_del(&txq->napi); 755 tx_ring = &txq->tx_ring; 756 for (j = 0; j < tx_ring_size; j++) 757 kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr); 758 kvfree(tx_ring->items); 759 } 760 761 kfree(priv->txqs); 762 priv->txqs = NULL; 763 return -ENOMEM; 764 } 765 766 static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq) 767 { 768 struct sdma_txreq *txreq; 769 struct sdma_txreq *txreq_tmp; 770 771 list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) { 772 struct ipoib_txreq *tx = 773 container_of(txreq, struct ipoib_txreq, txreq); 774 775 list_del(&txreq->list); 776 sdma_txclean(txq->priv->dd, &tx->txreq); 777 dev_kfree_skb_any(tx->skb); 778 tx->skb = NULL; 779 txq->tx_ring.complete_txreqs++; 780 } 781 782 if (hfi1_ipoib_used(txq)) 783 dd_dev_warn(txq->priv->dd, 784 "txq %d not empty found %u requests\n", 785 txq->q_idx, 786 hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs, 787 txq->tx_ring.complete_txreqs)); 788 } 789 790 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv) 791 { 792 int i, j; 793 794 for (i = 0; i < priv->netdev->num_tx_queues; i++) { 795 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 796 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring; 797 798 iowait_cancel_work(&txq->wait); 799 iowait_sdma_drain(&txq->wait); 800 hfi1_ipoib_drain_tx_list(txq); 801 netif_napi_del(&txq->napi); 802 hfi1_ipoib_drain_tx_ring(txq); 803 for (j = 0; j < tx_ring->max_items; j++) 804 kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr); 805 kvfree(tx_ring->items); 806 } 807 808 kfree(priv->txqs); 809 priv->txqs = NULL; 810 } 811 812 void hfi1_ipoib_napi_tx_enable(struct net_device *dev) 813 { 814 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 815 int i; 816 817 for (i = 0; i < dev->num_tx_queues; i++) { 818 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 819 820 napi_enable(&txq->napi); 821 } 822 } 823 824 void hfi1_ipoib_napi_tx_disable(struct net_device *dev) 825 { 826 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 827 int i; 828 829 for (i = 0; i < dev->num_tx_queues; i++) { 830 struct hfi1_ipoib_txq *txq = &priv->txqs[i]; 831 832 napi_disable(&txq->napi); 833 hfi1_ipoib_drain_tx_ring(txq); 834 } 835 } 836 837 void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q) 838 { 839 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev); 840 struct hfi1_ipoib_txq *txq = &priv->txqs[q]; 841 842 dd_dev_info(priv->dd, "timeout txq %p q %u stopped %u stops %d no_desc %d ring_full %d\n", 843 txq, q, 844 __netif_subqueue_stopped(dev, txq->q_idx), 845 atomic_read(&txq->tx_ring.stops), 846 atomic_read(&txq->tx_ring.no_desc), 847 atomic_read(&txq->tx_ring.ring_full)); 848 dd_dev_info(priv->dd, "sde %p engine %u\n", 849 txq->sde, 850 txq->sde ? txq->sde->this_idx : 0); 851 dd_dev_info(priv->dd, "flow %x\n", txq->flow.as_int); 852 dd_dev_info(priv->dd, "sent %llu completed %llu used %llu\n", 853 txq->tx_ring.sent_txreqs, txq->tx_ring.complete_txreqs, 854 hfi1_ipoib_used(txq)); 855 dd_dev_info(priv->dd, "tx_queue_len %u max_items %u\n", 856 dev->tx_queue_len, txq->tx_ring.max_items); 857 dd_dev_info(priv->dd, "head %u tail %u\n", 858 txq->tx_ring.head, txq->tx_ring.tail); 859 dd_dev_info(priv->dd, "wait queued %u\n", 860 !list_empty(&txq->wait.list)); 861 dd_dev_info(priv->dd, "tx_list empty %u\n", 862 list_empty(&txq->tx_list)); 863 } 864 865