Lines Matching refs:priv
84 static void ipoib_cm_dma_unmap_rx(struct ipoib_dev_priv *priv, struct ipoib_cm_rx_buf *rx_req) in ipoib_cm_dma_unmap_rx() argument
87 ipoib_dma_unmap_rx(priv, (struct ipoib_rx_buf *)rx_req); in ipoib_cm_dma_unmap_rx()
91 static int ipoib_cm_post_receive_srq(struct ipoib_dev_priv *priv, int id) in ipoib_cm_post_receive_srq() argument
99 rx_req = (struct ipoib_rx_buf *)&priv->cm.srq_ring[id]; in ipoib_cm_post_receive_srq()
101 priv->cm.rx_sge[i].addr = rx_req->mapping[i]; in ipoib_cm_post_receive_srq()
102 priv->cm.rx_sge[i].length = m->m_len; in ipoib_cm_post_receive_srq()
105 priv->cm.rx_wr.num_sge = i; in ipoib_cm_post_receive_srq()
106 priv->cm.rx_wr.wr_id = id | IPOIB_OP_CM | IPOIB_OP_RECV; in ipoib_cm_post_receive_srq()
108 ret = ib_post_srq_recv(priv->cm.srq, &priv->cm.rx_wr, &bad_wr); in ipoib_cm_post_receive_srq()
110 ipoib_warn(priv, "post srq failed for buf %d (%d)\n", id, ret); in ipoib_cm_post_receive_srq()
111 ipoib_dma_unmap_rx(priv, rx_req); in ipoib_cm_post_receive_srq()
112 m_freem(priv->cm.srq_ring[id].mb); in ipoib_cm_post_receive_srq()
113 priv->cm.srq_ring[id].mb = NULL; in ipoib_cm_post_receive_srq()
119 static int ipoib_cm_post_receive_nonsrq(struct ipoib_dev_priv *priv, in ipoib_cm_post_receive_nonsrq() argument
141 ipoib_warn(priv, "post recv failed for buf %d (%d)\n", id, ret); in ipoib_cm_post_receive_nonsrq()
142 ipoib_dma_unmap_rx(priv, rx_req); in ipoib_cm_post_receive_nonsrq()
151 ipoib_cm_alloc_rx_mb(struct ipoib_dev_priv *priv, struct ipoib_cm_rx_buf *rx_req) in ipoib_cm_alloc_rx_mb() argument
153 return ipoib_alloc_map_mb(priv, (struct ipoib_rx_buf *)rx_req, in ipoib_cm_alloc_rx_mb()
154 sizeof(struct ipoib_pseudoheader), priv->cm.max_cm_mtu, IPOIB_CM_RX_SG); in ipoib_cm_alloc_rx_mb()
157 static void ipoib_cm_free_rx_ring(struct ipoib_dev_priv *priv, in ipoib_cm_free_rx_ring() argument
164 ipoib_cm_dma_unmap_rx(priv, &rx_ring[i]); in ipoib_cm_free_rx_ring()
171 static void ipoib_cm_start_rx_drain(struct ipoib_dev_priv *priv) in ipoib_cm_start_rx_drain() argument
178 if (list_empty(&priv->cm.rx_flush_list) || in ipoib_cm_start_rx_drain()
179 !list_empty(&priv->cm.rx_drain_list)) in ipoib_cm_start_rx_drain()
186 p = list_entry(priv->cm.rx_flush_list.next, typeof(*p), list); in ipoib_cm_start_rx_drain()
188 ipoib_warn(priv, "failed to post drain wr\n"); in ipoib_cm_start_rx_drain()
190 list_splice_init(&priv->cm.rx_flush_list, &priv->cm.rx_drain_list); in ipoib_cm_start_rx_drain()
196 struct ipoib_dev_priv *priv = p->priv; in ipoib_cm_rx_event_handler() local
202 spin_lock_irqsave(&priv->lock, flags); in ipoib_cm_rx_event_handler()
203 list_move(&p->list, &priv->cm.rx_flush_list); in ipoib_cm_rx_event_handler()
205 ipoib_cm_start_rx_drain(priv); in ipoib_cm_rx_event_handler()
206 spin_unlock_irqrestore(&priv->lock, flags); in ipoib_cm_rx_event_handler()
209 static struct ib_qp *ipoib_cm_create_rx_qp(struct ipoib_dev_priv *priv, in ipoib_cm_create_rx_qp() argument
214 .send_cq = priv->recv_cq, /* For drain WR */ in ipoib_cm_create_rx_qp()
215 .recv_cq = priv->recv_cq, in ipoib_cm_create_rx_qp()
216 .srq = priv->cm.srq, in ipoib_cm_create_rx_qp()
224 if (!ipoib_cm_has_srq(priv)) { in ipoib_cm_create_rx_qp()
226 attr.cap.max_recv_sge = priv->cm.num_frags; in ipoib_cm_create_rx_qp()
229 return ib_create_qp(priv->pd, &attr); in ipoib_cm_create_rx_qp()
232 static int ipoib_cm_modify_rx_qp(struct ipoib_dev_priv *priv, in ipoib_cm_modify_rx_qp() argument
242 ipoib_warn(priv, "failed to init QP attr for INIT: %d\n", ret); in ipoib_cm_modify_rx_qp()
247 ipoib_warn(priv, "failed to modify QP to INIT: %d\n", ret); in ipoib_cm_modify_rx_qp()
253 ipoib_warn(priv, "failed to init QP attr for RTR: %d\n", ret); in ipoib_cm_modify_rx_qp()
259 ipoib_warn(priv, "failed to modify QP to RTR: %d\n", ret); in ipoib_cm_modify_rx_qp()
274 ipoib_warn(priv, "failed to init QP attr for RTS: %d\n", ret); in ipoib_cm_modify_rx_qp()
279 ipoib_warn(priv, "failed to modify QP to RTS: %d\n", ret); in ipoib_cm_modify_rx_qp()
286 static void ipoib_cm_init_rx_wr(struct ipoib_dev_priv *priv, in ipoib_cm_init_rx_wr() argument
293 sge[i].lkey = priv->pd->local_dma_lkey; in ipoib_cm_init_rx_wr()
300 static int ipoib_cm_nonsrq_init_rx(struct ipoib_dev_priv *priv, in ipoib_cm_nonsrq_init_rx() argument
313 priv->ca->name, ipoib_recvq_size); in ipoib_cm_nonsrq_init_rx()
325 ipoib_cm_init_rx_wr(priv, &t->wr, t->sge); in ipoib_cm_nonsrq_init_rx()
327 spin_lock_irq(&priv->lock); in ipoib_cm_nonsrq_init_rx()
329 if (priv->cm.nonsrq_conn_qp >= ipoib_max_conn_qp) { in ipoib_cm_nonsrq_init_rx()
330 spin_unlock_irq(&priv->lock); in ipoib_cm_nonsrq_init_rx()
335 ++priv->cm.nonsrq_conn_qp; in ipoib_cm_nonsrq_init_rx()
337 spin_unlock_irq(&priv->lock); in ipoib_cm_nonsrq_init_rx()
340 if (!ipoib_cm_alloc_rx_mb(priv, &rx->rx_ring[i])) { in ipoib_cm_nonsrq_init_rx()
341 ipoib_warn(priv, "failed to allocate receive buffer %d\n", i); in ipoib_cm_nonsrq_init_rx()
345 ret = ipoib_cm_post_receive_nonsrq(priv, rx, &t->wr, t->sge, i); in ipoib_cm_nonsrq_init_rx()
347 ipoib_warn(priv, "ipoib_cm_post_receive_nonsrq " in ipoib_cm_nonsrq_init_rx()
361 spin_lock_irq(&priv->lock); in ipoib_cm_nonsrq_init_rx()
362 --priv->cm.nonsrq_conn_qp; in ipoib_cm_nonsrq_init_rx()
363 spin_unlock_irq(&priv->lock); in ipoib_cm_nonsrq_init_rx()
367 ipoib_cm_free_rx_ring(priv, rx->rx_ring); in ipoib_cm_nonsrq_init_rx()
372 static int ipoib_cm_send_rep(struct ipoib_dev_priv *priv, struct ib_cm_id *cm_id, in ipoib_cm_send_rep() argument
379 data.qpn = cpu_to_be32(priv->qp->qp_num); in ipoib_cm_send_rep()
380 data.mtu = cpu_to_be32(priv->cm.max_cm_mtu); in ipoib_cm_send_rep()
386 rep.srq = ipoib_cm_has_srq(priv); in ipoib_cm_send_rep()
394 struct ipoib_dev_priv *priv = cm_id->context; in ipoib_cm_req_handler() local
399 ipoib_dbg(priv, "REQ arrived\n"); in ipoib_cm_req_handler()
403 p->priv = priv; in ipoib_cm_req_handler()
410 p->qp = ipoib_cm_create_rx_qp(priv, p); in ipoib_cm_req_handler()
417 ret = ipoib_cm_modify_rx_qp(priv, cm_id, p->qp, psn); in ipoib_cm_req_handler()
421 if (!ipoib_cm_has_srq(priv)) { in ipoib_cm_req_handler()
422 ret = ipoib_cm_nonsrq_init_rx(priv, cm_id, p); in ipoib_cm_req_handler()
427 spin_lock_irq(&priv->lock); in ipoib_cm_req_handler()
429 &priv->cm.stale_task, IPOIB_CM_RX_DELAY); in ipoib_cm_req_handler()
434 list_move(&p->list, &priv->cm.passive_ids); in ipoib_cm_req_handler()
435 spin_unlock_irq(&priv->lock); in ipoib_cm_req_handler()
437 ret = ipoib_cm_send_rep(priv, cm_id, p->qp, &event->param.req_rcvd, psn); in ipoib_cm_req_handler()
439 ipoib_warn(priv, "failed to send REP: %d\n", ret); in ipoib_cm_req_handler()
441 ipoib_warn(priv, "unable to move qp to error state\n"); in ipoib_cm_req_handler()
456 struct ipoib_dev_priv *priv; in ipoib_cm_rx_handler() local
467 priv = p->priv; in ipoib_cm_rx_handler()
469 ipoib_warn(priv, "unable to move qp to error state\n"); in ipoib_cm_rx_handler()
476 void ipoib_cm_handle_rx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc) in ipoib_cm_handle_rx_wc() argument
481 if_t dev = priv->dev; in ipoib_cm_handle_rx_wc()
486 ipoib_dbg_data(priv, "cm recv completion: id %d, status: %d\n", in ipoib_cm_handle_rx_wc()
491 spin_lock(&priv->lock); in ipoib_cm_handle_rx_wc()
492 list_splice_init(&priv->cm.rx_drain_list, &priv->cm.rx_reap_list); in ipoib_cm_handle_rx_wc()
493 ipoib_cm_start_rx_drain(priv); in ipoib_cm_handle_rx_wc()
494 if (priv->cm.id != NULL) in ipoib_cm_handle_rx_wc()
496 &priv->cm.rx_reap_task); in ipoib_cm_handle_rx_wc()
497 spin_unlock(&priv->lock); in ipoib_cm_handle_rx_wc()
499 ipoib_warn(priv, "cm recv completion event with wrid %d (> %d)\n", in ipoib_cm_handle_rx_wc()
506 has_srq = ipoib_cm_has_srq(priv); in ipoib_cm_handle_rx_wc()
507 rx_ring = has_srq ? priv->cm.srq_ring : p->rx_ring; in ipoib_cm_handle_rx_wc()
512 ipoib_dbg(priv, "cm recv error " in ipoib_cm_handle_rx_wc()
520 spin_lock(&priv->lock); in ipoib_cm_handle_rx_wc()
521 list_move(&p->list, &priv->cm.rx_reap_list); in ipoib_cm_handle_rx_wc()
522 queue_work(ipoib_workqueue, &priv->cm.rx_reap_task); in ipoib_cm_handle_rx_wc()
523 spin_unlock(&priv->lock); in ipoib_cm_handle_rx_wc()
535 list_move(&p->list, &priv->cm.passive_ids); in ipoib_cm_handle_rx_wc()
540 newmb = ipoib_cm_alloc_rx_mb(priv, &rx_ring[wr_id]); in ipoib_cm_handle_rx_wc()
546 ipoib_dbg(priv, "failed to allocate receive buffer %d\n", wr_id); in ipoib_cm_handle_rx_wc()
552 ipoib_cm_dma_unmap_rx(priv, &saverx); in ipoib_cm_handle_rx_wc()
554 ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n", in ipoib_cm_handle_rx_wc()
557 ipoib_dma_mb(priv, mb, wc->byte_len); in ipoib_cm_handle_rx_wc()
571 memcpy(ibh->hwaddr + 4, priv->local_gid.raw, sizeof(union ib_gid)); in ipoib_cm_handle_rx_wc()
579 if (unlikely(ipoib_cm_post_receive_srq(priv, wr_id))) in ipoib_cm_handle_rx_wc()
580 ipoib_warn(priv, "ipoib_cm_post_receive_srq failed " in ipoib_cm_handle_rx_wc()
583 if (unlikely(ipoib_cm_post_receive_nonsrq(priv, p, in ipoib_cm_handle_rx_wc()
584 &priv->cm.rx_wr, in ipoib_cm_handle_rx_wc()
585 priv->cm.rx_sge, in ipoib_cm_handle_rx_wc()
588 ipoib_warn(priv, "ipoib_cm_post_receive_nonsrq failed " in ipoib_cm_handle_rx_wc()
596 static inline int post_send(struct ipoib_dev_priv *priv, in post_send() argument
608 priv->tx_sge[i].addr = mapping[i]; in post_send()
609 priv->tx_sge[i].length = m->m_len; in post_send()
611 priv->tx_wr.wr.num_sge = i; in post_send()
612 priv->tx_wr.wr.wr_id = wr_id | IPOIB_OP_CM; in post_send()
613 priv->tx_wr.wr.opcode = IB_WR_SEND; in post_send()
615 return ib_post_send(tx->qp, &priv->tx_wr.wr, &bad_wr); in post_send()
618 void ipoib_cm_send(struct ipoib_dev_priv *priv, struct mbuf *mb, struct ipoib_cm_tx *tx) in ipoib_cm_send() argument
621 if_t dev = priv->dev; in ipoib_cm_send()
623 if (unlikely(priv->tx_outstanding > MAX_SEND_CQE)) { in ipoib_cm_send()
624 while (ipoib_poll_tx(priv, false)) in ipoib_cm_send()
630 ipoib_warn(priv, "packet len %d (> %d) too long to send, dropping\n", in ipoib_cm_send()
633 ipoib_cm_mb_too_long(priv, mb, IPOIB_CM_MTU(tx->mtu)); in ipoib_cm_send()
637 ipoib_dbg_data(priv, "sending packet: head 0x%x length %d connection 0x%x\n", in ipoib_cm_send()
650 if (unlikely(ipoib_dma_map_tx(priv->ca, (struct ipoib_tx_buf *)tx_req, in ipoib_cm_send()
651 priv->cm.num_frags))) { in ipoib_cm_send()
658 if (unlikely(post_send(priv, tx, tx_req, tx->tx_head & (ipoib_sendq_size - 1)))) { in ipoib_cm_send()
659 ipoib_warn(priv, "post_send failed\n"); in ipoib_cm_send()
661 ipoib_dma_unmap_tx(priv->ca, (struct ipoib_tx_buf *)tx_req); in ipoib_cm_send()
666 if (++priv->tx_outstanding == ipoib_sendq_size) { in ipoib_cm_send()
667 ipoib_dbg(priv, "TX ring 0x%x full, stopping kernel net queue\n", in ipoib_cm_send()
669 if (ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP)) in ipoib_cm_send()
670 ipoib_warn(priv, "request notify on send CQ failed\n"); in ipoib_cm_send()
677 void ipoib_cm_handle_tx_wc(struct ipoib_dev_priv *priv, struct ib_wc *wc) in ipoib_cm_handle_tx_wc() argument
681 if_t dev = priv->dev; in ipoib_cm_handle_tx_wc()
684 ipoib_dbg_data(priv, "cm send completion: id %d, status: %d\n", in ipoib_cm_handle_tx_wc()
688 ipoib_warn(priv, "cm send completion event with wrid %d (> %d)\n", in ipoib_cm_handle_tx_wc()
695 ipoib_dma_unmap_tx(priv->ca, (struct ipoib_tx_buf *)tx_req); in ipoib_cm_handle_tx_wc()
703 if (unlikely(--priv->tx_outstanding == ipoib_sendq_size >> 1) && in ipoib_cm_handle_tx_wc()
705 test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) in ipoib_cm_handle_tx_wc()
712 ipoib_dbg(priv, "failed cm send event " in ipoib_cm_handle_tx_wc()
720 rb_erase(&path->rb_node, &priv->path_tree); in ipoib_cm_handle_tx_wc()
725 list_move(&tx->list, &priv->cm.reap_list); in ipoib_cm_handle_tx_wc()
726 queue_work(ipoib_workqueue, &priv->cm.reap_task); in ipoib_cm_handle_tx_wc()
734 int ipoib_cm_dev_open(struct ipoib_dev_priv *priv) in ipoib_cm_dev_open() argument
738 if (!IPOIB_CM_SUPPORTED(if_getlladdr(priv->dev))) in ipoib_cm_dev_open()
741 priv->cm.id = ib_create_cm_id(priv->ca, ipoib_cm_rx_handler, priv); in ipoib_cm_dev_open()
742 if (IS_ERR(priv->cm.id)) { in ipoib_cm_dev_open()
743 printk(KERN_WARNING "%s: failed to create CM ID\n", priv->ca->name); in ipoib_cm_dev_open()
744 ret = PTR_ERR(priv->cm.id); in ipoib_cm_dev_open()
748 ret = ib_cm_listen(priv->cm.id, cpu_to_be64(IPOIB_CM_IETF_ID | priv->qp->qp_num), 0); in ipoib_cm_dev_open()
750 printk(KERN_WARNING "%s: failed to listen on ID 0x%llx\n", priv->ca->name, in ipoib_cm_dev_open()
751 IPOIB_CM_IETF_ID | priv->qp->qp_num); in ipoib_cm_dev_open()
758 ib_destroy_cm_id(priv->cm.id); in ipoib_cm_dev_open()
760 priv->cm.id = NULL; in ipoib_cm_dev_open()
764 static void ipoib_cm_free_rx_reap_list(struct ipoib_dev_priv *priv) in ipoib_cm_free_rx_reap_list() argument
769 spin_lock_irq(&priv->lock); in ipoib_cm_free_rx_reap_list()
770 list_splice_init(&priv->cm.rx_reap_list, &list); in ipoib_cm_free_rx_reap_list()
771 spin_unlock_irq(&priv->lock); in ipoib_cm_free_rx_reap_list()
776 if (!ipoib_cm_has_srq(priv)) { in ipoib_cm_free_rx_reap_list()
777 ipoib_cm_free_rx_ring(priv, rx->rx_ring); in ipoib_cm_free_rx_reap_list()
778 spin_lock_irq(&priv->lock); in ipoib_cm_free_rx_reap_list()
779 --priv->cm.nonsrq_conn_qp; in ipoib_cm_free_rx_reap_list()
780 spin_unlock_irq(&priv->lock); in ipoib_cm_free_rx_reap_list()
786 void ipoib_cm_dev_stop(struct ipoib_dev_priv *priv) in ipoib_cm_dev_stop() argument
792 if (!IPOIB_CM_SUPPORTED(if_getlladdr(priv->dev)) || !priv->cm.id) in ipoib_cm_dev_stop()
795 ib_destroy_cm_id(priv->cm.id); in ipoib_cm_dev_stop()
796 priv->cm.id = NULL; in ipoib_cm_dev_stop()
798 cancel_work_sync(&priv->cm.rx_reap_task); in ipoib_cm_dev_stop()
800 spin_lock_irq(&priv->lock); in ipoib_cm_dev_stop()
801 while (!list_empty(&priv->cm.passive_ids)) { in ipoib_cm_dev_stop()
802 p = list_entry(priv->cm.passive_ids.next, typeof(*p), list); in ipoib_cm_dev_stop()
803 list_move(&p->list, &priv->cm.rx_error_list); in ipoib_cm_dev_stop()
805 spin_unlock_irq(&priv->lock); in ipoib_cm_dev_stop()
808 ipoib_warn(priv, "unable to move qp to error state: %d\n", ret); in ipoib_cm_dev_stop()
809 spin_lock_irq(&priv->lock); in ipoib_cm_dev_stop()
815 while (!list_empty(&priv->cm.rx_error_list) || in ipoib_cm_dev_stop()
816 !list_empty(&priv->cm.rx_flush_list) || in ipoib_cm_dev_stop()
817 !list_empty(&priv->cm.rx_drain_list)) { in ipoib_cm_dev_stop()
819 ipoib_warn(priv, "RX drain timing out\n"); in ipoib_cm_dev_stop()
824 list_splice_init(&priv->cm.rx_flush_list, in ipoib_cm_dev_stop()
825 &priv->cm.rx_reap_list); in ipoib_cm_dev_stop()
826 list_splice_init(&priv->cm.rx_error_list, in ipoib_cm_dev_stop()
827 &priv->cm.rx_reap_list); in ipoib_cm_dev_stop()
828 list_splice_init(&priv->cm.rx_drain_list, in ipoib_cm_dev_stop()
829 &priv->cm.rx_reap_list); in ipoib_cm_dev_stop()
832 spin_unlock_irq(&priv->lock); in ipoib_cm_dev_stop()
834 ipoib_drain_cq(priv); in ipoib_cm_dev_stop()
835 spin_lock_irq(&priv->lock); in ipoib_cm_dev_stop()
838 spin_unlock_irq(&priv->lock); in ipoib_cm_dev_stop()
840 ipoib_cm_free_rx_reap_list(priv); in ipoib_cm_dev_stop()
842 cancel_delayed_work_sync(&priv->cm.stale_task); in ipoib_cm_dev_stop()
848 struct ipoib_dev_priv *priv = p->priv; in ipoib_cm_rep_handler() local
856 ipoib_dbg(priv, "cm rep handler\n"); in ipoib_cm_rep_handler()
860 ipoib_warn(priv, "Rejecting connection: mtu %d <= %d\n", in ipoib_cm_rep_handler()
868 ipoib_warn(priv, "failed to init QP attr for RTR: %d\n", ret); in ipoib_cm_rep_handler()
875 ipoib_warn(priv, "failed to modify QP to RTR: %d\n", ret); in ipoib_cm_rep_handler()
882 ipoib_warn(priv, "failed to init QP attr for RTS: %d\n", ret); in ipoib_cm_rep_handler()
887 ipoib_warn(priv, "failed to modify QP to RTS: %d\n", ret); in ipoib_cm_rep_handler()
893 spin_lock_irq(&priv->lock); in ipoib_cm_rep_handler()
902 spin_unlock_irq(&priv->lock); in ipoib_cm_rep_handler()
906 if_t dev = p->priv->dev; in ipoib_cm_rep_handler()
912 ipoib_warn(priv, "dev_queue_xmit failed " in ipoib_cm_rep_handler()
919 ipoib_warn(priv, "failed to send RTU: %d\n", ret); in ipoib_cm_rep_handler()
925 static struct ib_qp *ipoib_cm_create_tx_qp(struct ipoib_dev_priv *priv, in ipoib_cm_create_tx_qp() argument
929 .send_cq = priv->send_cq, in ipoib_cm_create_tx_qp()
930 .recv_cq = priv->recv_cq, in ipoib_cm_create_tx_qp()
931 .srq = priv->cm.srq, in ipoib_cm_create_tx_qp()
933 .cap.max_send_sge = priv->cm.num_frags, in ipoib_cm_create_tx_qp()
939 return ib_create_qp(priv->pd, &attr); in ipoib_cm_create_tx_qp()
942 static int ipoib_cm_send_req(struct ipoib_dev_priv *priv, in ipoib_cm_send_req() argument
950 ipoib_dbg(priv, "cm send req\n"); in ipoib_cm_send_req()
952 data.qpn = cpu_to_be32(priv->qp->qp_num); in ipoib_cm_send_req()
953 data.mtu = cpu_to_be32(priv->cm.max_cm_mtu); in ipoib_cm_send_req()
976 req.srq = ipoib_cm_has_srq(priv); in ipoib_cm_send_req()
980 static int ipoib_cm_modify_tx_init(struct ipoib_dev_priv *priv, in ipoib_cm_modify_tx_init() argument
985 ret = ib_find_pkey(priv->ca, priv->port, priv->pkey, &qp_attr.pkey_index); in ipoib_cm_modify_tx_init()
987 ipoib_warn(priv, "pkey 0x%x not found: %d\n", priv->pkey, ret); in ipoib_cm_modify_tx_init()
993 qp_attr.port_num = priv->port; in ipoib_cm_modify_tx_init()
998 ipoib_warn(priv, "failed to modify tx QP to INIT: %d\n", ret); in ipoib_cm_modify_tx_init()
1007 struct ipoib_dev_priv *priv = p->priv; in ipoib_cm_tx_init() local
1012 ipoib_warn(priv, "failed to allocate tx ring\n"); in ipoib_cm_tx_init()
1018 p->qp = ipoib_cm_create_tx_qp(p->priv, p); in ipoib_cm_tx_init()
1021 ipoib_warn(priv, "failed to allocate tx qp: %d\n", ret); in ipoib_cm_tx_init()
1025 p->id = ib_create_cm_id(priv->ca, ipoib_cm_tx_handler, p); in ipoib_cm_tx_init()
1028 ipoib_warn(priv, "failed to create tx cm id: %d\n", ret); in ipoib_cm_tx_init()
1032 ret = ipoib_cm_modify_tx_init(p->priv, p->id, p->qp); in ipoib_cm_tx_init()
1034 ipoib_warn(priv, "failed to modify tx qp to rtr: %d\n", ret); in ipoib_cm_tx_init()
1038 ret = ipoib_cm_send_req(p->priv, p->id, p->qp, qpn, pathrec); in ipoib_cm_tx_init()
1040 ipoib_warn(priv, "failed to send cm req: %d\n", ret); in ipoib_cm_tx_init()
1044 ipoib_dbg(priv, "Request connection 0x%x for gid %pI6 qpn 0x%x\n", in ipoib_cm_tx_init()
1064 struct ipoib_dev_priv *priv = p->priv; in ipoib_cm_tx_destroy() local
1065 if_t dev = priv->dev; in ipoib_cm_tx_destroy()
1069 ipoib_dbg(priv, "Destroy active connection 0x%x head 0x%x tail 0x%x\n", in ipoib_cm_tx_destroy()
1073 ipoib_path_free(priv, p->path); in ipoib_cm_tx_destroy()
1083 ipoib_warn(priv, "timing out; %d sends not completed\n", in ipoib_cm_tx_destroy()
1096 ipoib_dma_unmap_tx(priv->ca, (struct ipoib_tx_buf *)tx_req); in ipoib_cm_tx_destroy()
1099 if (unlikely(--priv->tx_outstanding == ipoib_sendq_size >> 1) && in ipoib_cm_tx_destroy()
1101 test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) in ipoib_cm_tx_destroy()
1116 struct ipoib_dev_priv *priv = tx->priv; in ipoib_cm_tx_handler() local
1123 ipoib_dbg(priv, "DREQ received.\n"); in ipoib_cm_tx_handler()
1127 ipoib_dbg(priv, "REP received.\n"); in ipoib_cm_tx_handler()
1136 ipoib_dbg(priv, "CM error %d.\n", event->event); in ipoib_cm_tx_handler()
1137 spin_lock_irqsave(&priv->lock, flags); in ipoib_cm_tx_handler()
1143 rb_erase(&path->rb_node, &priv->path_tree); in ipoib_cm_tx_handler()
1148 list_move(&tx->list, &priv->cm.reap_list); in ipoib_cm_tx_handler()
1149 queue_work(ipoib_workqueue, &priv->cm.reap_task); in ipoib_cm_tx_handler()
1152 spin_unlock_irqrestore(&priv->lock, flags); in ipoib_cm_tx_handler()
1154 ipoib_path_free(tx->priv, path); in ipoib_cm_tx_handler()
1163 struct ipoib_cm_tx *ipoib_cm_create_tx(struct ipoib_dev_priv *priv, in ipoib_cm_create_tx() argument
1172 ipoib_dbg(priv, "Creating cm tx\n"); in ipoib_cm_create_tx()
1175 tx->priv = priv; in ipoib_cm_create_tx()
1176 list_add(&tx->list, &priv->cm.start_list); in ipoib_cm_create_tx()
1178 queue_work(ipoib_workqueue, &priv->cm.start_task); in ipoib_cm_create_tx()
1184 struct ipoib_dev_priv *priv = tx->priv; in ipoib_cm_destroy_tx() local
1186 spin_lock(&priv->lock); in ipoib_cm_destroy_tx()
1187 list_move(&tx->list, &priv->cm.reap_list); in ipoib_cm_destroy_tx()
1188 spin_unlock(&priv->lock); in ipoib_cm_destroy_tx()
1189 queue_work(ipoib_workqueue, &priv->cm.reap_task); in ipoib_cm_destroy_tx()
1190 ipoib_dbg(priv, "Reap connection for gid %pI6\n", in ipoib_cm_destroy_tx()
1198 struct ipoib_dev_priv *priv = container_of(work, struct ipoib_dev_priv, in ipoib_cm_tx_start() local
1208 ipoib_dbg(priv, "cm start task\n"); in ipoib_cm_tx_start()
1209 spin_lock_irqsave(&priv->lock, flags); in ipoib_cm_tx_start()
1211 while (!list_empty(&priv->cm.start_list)) { in ipoib_cm_tx_start()
1212 p = list_entry(priv->cm.start_list.next, typeof(*p), list); in ipoib_cm_tx_start()
1218 spin_unlock_irqrestore(&priv->lock, flags); in ipoib_cm_tx_start()
1222 spin_lock_irqsave(&priv->lock, flags); in ipoib_cm_tx_start()
1228 rb_erase(&path->rb_node, &priv->path_tree); in ipoib_cm_tx_start()
1230 ipoib_path_free(priv, path); in ipoib_cm_tx_start()
1237 spin_unlock_irqrestore(&priv->lock, flags); in ipoib_cm_tx_start()
1242 struct ipoib_dev_priv *priv = container_of(work, struct ipoib_dev_priv, in ipoib_cm_tx_reap() local
1247 spin_lock_irqsave(&priv->lock, flags); in ipoib_cm_tx_reap()
1249 while (!list_empty(&priv->cm.reap_list)) { in ipoib_cm_tx_reap()
1250 p = list_entry(priv->cm.reap_list.next, typeof(*p), list); in ipoib_cm_tx_reap()
1252 spin_unlock_irqrestore(&priv->lock, flags); in ipoib_cm_tx_reap()
1254 spin_lock_irqsave(&priv->lock, flags); in ipoib_cm_tx_reap()
1257 spin_unlock_irqrestore(&priv->lock, flags); in ipoib_cm_tx_reap()
1262 struct ipoib_dev_priv *priv = container_of(work, struct ipoib_dev_priv, in ipoib_cm_mb_reap() local
1267 unsigned mtu = priv->mcast_mtu; in ipoib_cm_mb_reap()
1271 spin_lock_irqsave(&priv->lock, flags); in ipoib_cm_mb_reap()
1273 CURVNET_SET_QUIET(if_getvnet(priv->dev)); in ipoib_cm_mb_reap()
1276 IF_DEQUEUE(&priv->cm.mb_queue, mb); in ipoib_cm_mb_reap()
1279 spin_unlock_irqrestore(&priv->lock, flags); in ipoib_cm_mb_reap()
1298 spin_lock_irqsave(&priv->lock, flags); in ipoib_cm_mb_reap()
1303 spin_unlock_irqrestore(&priv->lock, flags); in ipoib_cm_mb_reap()
1307 ipoib_cm_mb_too_long(struct ipoib_dev_priv *priv, struct mbuf *mb, unsigned int mtu) in ipoib_cm_mb_too_long() argument
1309 int e = priv->cm.mb_queue.ifq_len; in ipoib_cm_mb_too_long()
1311 IF_ENQUEUE(&priv->cm.mb_queue, mb); in ipoib_cm_mb_too_long()
1313 queue_work(ipoib_workqueue, &priv->cm.mb_task); in ipoib_cm_mb_too_long()
1324 struct ipoib_dev_priv *priv = container_of(work, struct ipoib_dev_priv, in ipoib_cm_stale_task() local
1329 spin_lock_irq(&priv->lock); in ipoib_cm_stale_task()
1330 while (!list_empty(&priv->cm.passive_ids)) { in ipoib_cm_stale_task()
1333 p = list_entry(priv->cm.passive_ids.prev, typeof(*p), list); in ipoib_cm_stale_task()
1336 list_move(&p->list, &priv->cm.rx_error_list); in ipoib_cm_stale_task()
1338 spin_unlock_irq(&priv->lock); in ipoib_cm_stale_task()
1341 ipoib_warn(priv, "unable to move qp to error state: %d\n", ret); in ipoib_cm_stale_task()
1342 spin_lock_irq(&priv->lock); in ipoib_cm_stale_task()
1345 if (!list_empty(&priv->cm.passive_ids)) in ipoib_cm_stale_task()
1347 &priv->cm.stale_task, IPOIB_CM_RX_DELAY); in ipoib_cm_stale_task()
1348 spin_unlock_irq(&priv->lock); in ipoib_cm_stale_task()
1352 static void ipoib_cm_create_srq(struct ipoib_dev_priv *priv, int max_sge) in ipoib_cm_create_srq() argument
1361 priv->cm.srq = ib_create_srq(priv->pd, &srq_init_attr); in ipoib_cm_create_srq()
1362 if (IS_ERR(priv->cm.srq)) { in ipoib_cm_create_srq()
1363 if (PTR_ERR(priv->cm.srq) != -ENOSYS) in ipoib_cm_create_srq()
1365 priv->ca->name, PTR_ERR(priv->cm.srq)); in ipoib_cm_create_srq()
1366 priv->cm.srq = NULL; in ipoib_cm_create_srq()
1370 priv->cm.srq_ring = kzalloc(ipoib_recvq_size * sizeof *priv->cm.srq_ring, GFP_KERNEL); in ipoib_cm_create_srq()
1371 if (!priv->cm.srq_ring) { in ipoib_cm_create_srq()
1373 priv->ca->name, ipoib_recvq_size); in ipoib_cm_create_srq()
1374 ib_destroy_srq(priv->cm.srq); in ipoib_cm_create_srq()
1375 priv->cm.srq = NULL; in ipoib_cm_create_srq()
1379 memset(priv->cm.srq_ring, 0, ipoib_recvq_size * sizeof *priv->cm.srq_ring); in ipoib_cm_create_srq()
1382 int ipoib_cm_dev_init(struct ipoib_dev_priv *priv) in ipoib_cm_dev_init() argument
1384 if_t dev = priv->dev; in ipoib_cm_dev_init()
1388 INIT_LIST_HEAD(&priv->cm.passive_ids); in ipoib_cm_dev_init()
1389 INIT_LIST_HEAD(&priv->cm.reap_list); in ipoib_cm_dev_init()
1390 INIT_LIST_HEAD(&priv->cm.start_list); in ipoib_cm_dev_init()
1391 INIT_LIST_HEAD(&priv->cm.rx_error_list); in ipoib_cm_dev_init()
1392 INIT_LIST_HEAD(&priv->cm.rx_flush_list); in ipoib_cm_dev_init()
1393 INIT_LIST_HEAD(&priv->cm.rx_drain_list); in ipoib_cm_dev_init()
1394 INIT_LIST_HEAD(&priv->cm.rx_reap_list); in ipoib_cm_dev_init()
1395 INIT_WORK(&priv->cm.start_task, ipoib_cm_tx_start); in ipoib_cm_dev_init()
1396 INIT_WORK(&priv->cm.reap_task, ipoib_cm_tx_reap); in ipoib_cm_dev_init()
1397 INIT_WORK(&priv->cm.mb_task, ipoib_cm_mb_reap); in ipoib_cm_dev_init()
1398 INIT_WORK(&priv->cm.rx_reap_task, ipoib_cm_rx_reap); in ipoib_cm_dev_init()
1399 INIT_DELAYED_WORK(&priv->cm.stale_task, ipoib_cm_stale_task); in ipoib_cm_dev_init()
1401 bzero(&priv->cm.mb_queue, sizeof(priv->cm.mb_queue)); in ipoib_cm_dev_init()
1402 mtx_init(&priv->cm.mb_queue.ifq_mtx, in ipoib_cm_dev_init()
1405 max_srq_sge = priv->ca->attrs.max_srq_sge; in ipoib_cm_dev_init()
1407 ipoib_dbg(priv, "max_srq_sge=%d\n", max_srq_sge); in ipoib_cm_dev_init()
1410 ipoib_cm_create_srq(priv, max_srq_sge); in ipoib_cm_dev_init()
1411 if (ipoib_cm_has_srq(priv)) { in ipoib_cm_dev_init()
1412 priv->cm.max_cm_mtu = max_srq_sge * MJUMPAGESIZE; in ipoib_cm_dev_init()
1413 priv->cm.num_frags = max_srq_sge; in ipoib_cm_dev_init()
1414 ipoib_dbg(priv, "max_cm_mtu = 0x%x, num_frags=%d\n", in ipoib_cm_dev_init()
1415 priv->cm.max_cm_mtu, priv->cm.num_frags); in ipoib_cm_dev_init()
1417 priv->cm.max_cm_mtu = IPOIB_CM_MAX_MTU; in ipoib_cm_dev_init()
1418 priv->cm.num_frags = IPOIB_CM_RX_SG; in ipoib_cm_dev_init()
1421 ipoib_cm_init_rx_wr(priv, &priv->cm.rx_wr, priv->cm.rx_sge); in ipoib_cm_dev_init()
1423 if (ipoib_cm_has_srq(priv)) { in ipoib_cm_dev_init()
1425 if (!ipoib_cm_alloc_rx_mb(priv, &priv->cm.srq_ring[i])) { in ipoib_cm_dev_init()
1426 ipoib_warn(priv, "failed to allocate " in ipoib_cm_dev_init()
1428 ipoib_cm_dev_cleanup(priv); in ipoib_cm_dev_init()
1432 if (ipoib_cm_post_receive_srq(priv, i)) { in ipoib_cm_dev_init()
1433 ipoib_warn(priv, "ipoib_cm_post_receive_srq " in ipoib_cm_dev_init()
1435 ipoib_cm_dev_cleanup(priv); in ipoib_cm_dev_init()
1441 if_getlladdr(priv->dev)[0] = IPOIB_FLAGS_RC; in ipoib_cm_dev_init() local
1445 void ipoib_cm_dev_cleanup(struct ipoib_dev_priv *priv) in ipoib_cm_dev_cleanup() argument
1449 if (!priv->cm.srq) in ipoib_cm_dev_cleanup()
1452 ipoib_dbg(priv, "Cleanup ipoib connected mode.\n"); in ipoib_cm_dev_cleanup()
1454 ret = ib_destroy_srq(priv->cm.srq); in ipoib_cm_dev_cleanup()
1456 ipoib_warn(priv, "ib_destroy_srq failed: %d\n", ret); in ipoib_cm_dev_cleanup()
1458 priv->cm.srq = NULL; in ipoib_cm_dev_cleanup()
1459 if (!priv->cm.srq_ring) in ipoib_cm_dev_cleanup()
1462 ipoib_cm_free_rx_ring(priv, priv->cm.srq_ring); in ipoib_cm_dev_cleanup()
1463 priv->cm.srq_ring = NULL; in ipoib_cm_dev_cleanup()
1465 mtx_destroy(&priv->cm.mb_queue.ifq_mtx); in ipoib_cm_dev_cleanup()