1aa0a1e58SJeff Roberson /* 2aa0a1e58SJeff Roberson * Copyright (c) 2009 Mellanox Technologies Ltd. All rights reserved. 3aa0a1e58SJeff Roberson * 4aa0a1e58SJeff Roberson * This software is available to you under a choice of one of two 5aa0a1e58SJeff Roberson * licenses. You may choose to be licensed under the terms of the GNU 6aa0a1e58SJeff Roberson * General Public License (GPL) Version 2, available from the file 7aa0a1e58SJeff Roberson * COPYING in the main directory of this source tree, or the 8aa0a1e58SJeff Roberson * OpenIB.org BSD license below: 9aa0a1e58SJeff Roberson * 10aa0a1e58SJeff Roberson * Redistribution and use in source and binary forms, with or 11aa0a1e58SJeff Roberson * without modification, are permitted provided that the following 12aa0a1e58SJeff Roberson * conditions are met: 13aa0a1e58SJeff Roberson * 14aa0a1e58SJeff Roberson * - Redistributions of source code must retain the above 15aa0a1e58SJeff Roberson * copyright notice, this list of conditions and the following 16aa0a1e58SJeff Roberson * disclaimer. 17aa0a1e58SJeff Roberson * 18aa0a1e58SJeff Roberson * - Redistributions in binary form must reproduce the above 19aa0a1e58SJeff Roberson * copyright notice, this list of conditions and the following 20aa0a1e58SJeff Roberson * disclaimer in the documentation and/or other materials 21aa0a1e58SJeff Roberson * provided with the distribution. 22aa0a1e58SJeff Roberson * 23aa0a1e58SJeff Roberson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24aa0a1e58SJeff Roberson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25aa0a1e58SJeff Roberson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26aa0a1e58SJeff Roberson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27aa0a1e58SJeff Roberson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28aa0a1e58SJeff Roberson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29aa0a1e58SJeff Roberson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30aa0a1e58SJeff Roberson * SOFTWARE. 31aa0a1e58SJeff Roberson */ 32aa0a1e58SJeff Roberson #include "sdp.h" 33aa0a1e58SJeff Roberson 34aa0a1e58SJeff Roberson SDP_MODPARAM_INT(rcvbuf_initial_size, 32 * 1024, 35aa0a1e58SJeff Roberson "Receive buffer initial size in bytes."); 36aa0a1e58SJeff Roberson SDP_MODPARAM_SINT(rcvbuf_scale, 0x8, 37aa0a1e58SJeff Roberson "Receive buffer size scale factor."); 38aa0a1e58SJeff Roberson 39aa0a1e58SJeff Roberson /* Like tcp_fin - called when SDP_MID_DISCONNECT is received */ 40aa0a1e58SJeff Roberson static void 41aa0a1e58SJeff Roberson sdp_handle_disconn(struct sdp_sock *ssk) 42aa0a1e58SJeff Roberson { 43aa0a1e58SJeff Roberson 44aa0a1e58SJeff Roberson sdp_dbg(ssk->socket, "%s\n", __func__); 45aa0a1e58SJeff Roberson 46aa0a1e58SJeff Roberson SDP_WLOCK_ASSERT(ssk); 47aa0a1e58SJeff Roberson if (TCPS_HAVERCVDFIN(ssk->state) == 0) 48aa0a1e58SJeff Roberson socantrcvmore(ssk->socket); 49aa0a1e58SJeff Roberson 50aa0a1e58SJeff Roberson switch (ssk->state) { 51aa0a1e58SJeff Roberson case TCPS_SYN_RECEIVED: 52aa0a1e58SJeff Roberson case TCPS_ESTABLISHED: 53aa0a1e58SJeff Roberson ssk->state = TCPS_CLOSE_WAIT; 54aa0a1e58SJeff Roberson break; 55aa0a1e58SJeff Roberson 56aa0a1e58SJeff Roberson case TCPS_FIN_WAIT_1: 57aa0a1e58SJeff Roberson /* Received a reply FIN - start Infiniband tear down */ 58aa0a1e58SJeff Roberson sdp_dbg(ssk->socket, 59aa0a1e58SJeff Roberson "%s: Starting Infiniband tear down sending DREQ\n", 60aa0a1e58SJeff Roberson __func__); 61aa0a1e58SJeff Roberson 62aa0a1e58SJeff Roberson sdp_cancel_dreq_wait_timeout(ssk); 63aa0a1e58SJeff Roberson ssk->qp_active = 0; 64aa0a1e58SJeff Roberson if (ssk->id) { 65aa0a1e58SJeff Roberson struct rdma_cm_id *id; 66aa0a1e58SJeff Roberson 67aa0a1e58SJeff Roberson id = ssk->id; 68aa0a1e58SJeff Roberson SDP_WUNLOCK(ssk); 69aa0a1e58SJeff Roberson rdma_disconnect(id); 70aa0a1e58SJeff Roberson SDP_WLOCK(ssk); 71aa0a1e58SJeff Roberson } else { 72aa0a1e58SJeff Roberson sdp_warn(ssk->socket, 73aa0a1e58SJeff Roberson "%s: ssk->id is NULL\n", __func__); 74aa0a1e58SJeff Roberson return; 75aa0a1e58SJeff Roberson } 76aa0a1e58SJeff Roberson break; 77aa0a1e58SJeff Roberson case TCPS_TIME_WAIT: 78aa0a1e58SJeff Roberson /* This is a mutual close situation and we've got the DREQ from 79aa0a1e58SJeff Roberson the peer before the SDP_MID_DISCONNECT */ 80aa0a1e58SJeff Roberson break; 81aa0a1e58SJeff Roberson case TCPS_CLOSED: 82aa0a1e58SJeff Roberson /* FIN arrived after IB teardown started - do nothing */ 83aa0a1e58SJeff Roberson sdp_dbg(ssk->socket, "%s: fin in state %s\n", 84aa0a1e58SJeff Roberson __func__, sdp_state_str(ssk->state)); 85aa0a1e58SJeff Roberson return; 86aa0a1e58SJeff Roberson default: 87aa0a1e58SJeff Roberson sdp_warn(ssk->socket, 88aa0a1e58SJeff Roberson "%s: FIN in unexpected state. state=%d\n", 89aa0a1e58SJeff Roberson __func__, ssk->state); 90aa0a1e58SJeff Roberson break; 91aa0a1e58SJeff Roberson } 92aa0a1e58SJeff Roberson } 93aa0a1e58SJeff Roberson 94aa0a1e58SJeff Roberson static int 95aa0a1e58SJeff Roberson sdp_post_recv(struct sdp_sock *ssk) 96aa0a1e58SJeff Roberson { 97aa0a1e58SJeff Roberson struct sdp_buf *rx_req; 98aa0a1e58SJeff Roberson int i, rc; 99aa0a1e58SJeff Roberson u64 addr; 100aa0a1e58SJeff Roberson struct ib_device *dev; 101aa0a1e58SJeff Roberson struct ib_recv_wr rx_wr = { NULL }; 102aa0a1e58SJeff Roberson struct ib_sge ibsge[SDP_MAX_RECV_SGES]; 103aa0a1e58SJeff Roberson struct ib_sge *sge = ibsge; 104aa0a1e58SJeff Roberson struct ib_recv_wr *bad_wr; 105aa0a1e58SJeff Roberson struct mbuf *mb, *m; 106aa0a1e58SJeff Roberson struct sdp_bsdh *h; 107aa0a1e58SJeff Roberson int id = ring_head(ssk->rx_ring); 108aa0a1e58SJeff Roberson 109aa0a1e58SJeff Roberson /* Now, allocate and repost recv */ 110aa0a1e58SJeff Roberson sdp_prf(ssk->socket, mb, "Posting mb"); 111aa0a1e58SJeff Roberson mb = m_getm2(NULL, ssk->recv_bytes, M_NOWAIT, MT_DATA, M_PKTHDR); 112aa0a1e58SJeff Roberson if (mb == NULL) { 113aa0a1e58SJeff Roberson /* Retry so we can't stall out with no memory. */ 114aa0a1e58SJeff Roberson if (!rx_ring_posted(ssk)) 115aa0a1e58SJeff Roberson queue_work(rx_comp_wq, &ssk->rx_comp_work); 116aa0a1e58SJeff Roberson return -1; 117aa0a1e58SJeff Roberson } 118aa0a1e58SJeff Roberson for (m = mb; m != NULL; m = m->m_next) { 119aa0a1e58SJeff Roberson m->m_len = (m->m_flags & M_EXT) ? m->m_ext.ext_size : 120aa0a1e58SJeff Roberson ((m->m_flags & M_PKTHDR) ? MHLEN : MLEN); 121aa0a1e58SJeff Roberson mb->m_pkthdr.len += m->m_len; 122aa0a1e58SJeff Roberson } 123aa0a1e58SJeff Roberson h = mtod(mb, struct sdp_bsdh *); 124aa0a1e58SJeff Roberson rx_req = ssk->rx_ring.buffer + (id & (SDP_RX_SIZE - 1)); 125aa0a1e58SJeff Roberson rx_req->mb = mb; 126aa0a1e58SJeff Roberson dev = ssk->ib_device; 127aa0a1e58SJeff Roberson for (i = 0; mb != NULL; i++, mb = mb->m_next, sge++) { 128aa0a1e58SJeff Roberson addr = ib_dma_map_single(dev, mb->m_data, mb->m_len, 129aa0a1e58SJeff Roberson DMA_TO_DEVICE); 130aa0a1e58SJeff Roberson /* TODO: proper error handling */ 131aa0a1e58SJeff Roberson BUG_ON(ib_dma_mapping_error(dev, addr)); 132aa0a1e58SJeff Roberson BUG_ON(i >= SDP_MAX_RECV_SGES); 133aa0a1e58SJeff Roberson rx_req->mapping[i] = addr; 134aa0a1e58SJeff Roberson sge->addr = addr; 135aa0a1e58SJeff Roberson sge->length = mb->m_len; 136aa0a1e58SJeff Roberson sge->lkey = ssk->sdp_dev->mr->lkey; 137aa0a1e58SJeff Roberson } 138aa0a1e58SJeff Roberson 139aa0a1e58SJeff Roberson rx_wr.next = NULL; 140aa0a1e58SJeff Roberson rx_wr.wr_id = id | SDP_OP_RECV; 141aa0a1e58SJeff Roberson rx_wr.sg_list = ibsge; 142aa0a1e58SJeff Roberson rx_wr.num_sge = i; 143aa0a1e58SJeff Roberson rc = ib_post_recv(ssk->qp, &rx_wr, &bad_wr); 144aa0a1e58SJeff Roberson if (unlikely(rc)) { 145aa0a1e58SJeff Roberson sdp_warn(ssk->socket, "ib_post_recv failed. status %d\n", rc); 146aa0a1e58SJeff Roberson 147aa0a1e58SJeff Roberson sdp_cleanup_sdp_buf(ssk, rx_req, DMA_FROM_DEVICE); 148aa0a1e58SJeff Roberson m_freem(mb); 149aa0a1e58SJeff Roberson 150aa0a1e58SJeff Roberson sdp_notify(ssk, ECONNRESET); 151aa0a1e58SJeff Roberson 152aa0a1e58SJeff Roberson return -1; 153aa0a1e58SJeff Roberson } 154aa0a1e58SJeff Roberson 155aa0a1e58SJeff Roberson atomic_inc(&ssk->rx_ring.head); 156aa0a1e58SJeff Roberson SDPSTATS_COUNTER_INC(post_recv); 157aa0a1e58SJeff Roberson 158aa0a1e58SJeff Roberson return 0; 159aa0a1e58SJeff Roberson } 160aa0a1e58SJeff Roberson 161aa0a1e58SJeff Roberson static inline int 162aa0a1e58SJeff Roberson sdp_post_recvs_needed(struct sdp_sock *ssk) 163aa0a1e58SJeff Roberson { 164aa0a1e58SJeff Roberson unsigned long bytes_in_process; 165aa0a1e58SJeff Roberson unsigned long max_bytes; 166aa0a1e58SJeff Roberson int buffer_size; 167aa0a1e58SJeff Roberson int posted; 168aa0a1e58SJeff Roberson 169aa0a1e58SJeff Roberson if (!ssk->qp_active || !ssk->socket) 170aa0a1e58SJeff Roberson return 0; 171aa0a1e58SJeff Roberson 172aa0a1e58SJeff Roberson posted = rx_ring_posted(ssk); 173aa0a1e58SJeff Roberson if (posted >= SDP_RX_SIZE) 174aa0a1e58SJeff Roberson return 0; 175aa0a1e58SJeff Roberson if (posted < SDP_MIN_TX_CREDITS) 176aa0a1e58SJeff Roberson return 1; 177aa0a1e58SJeff Roberson 178aa0a1e58SJeff Roberson buffer_size = ssk->recv_bytes; 179aa0a1e58SJeff Roberson max_bytes = max(ssk->socket->so_snd.sb_hiwat, 180aa0a1e58SJeff Roberson (1 + SDP_MIN_TX_CREDITS) * buffer_size); 181aa0a1e58SJeff Roberson max_bytes *= rcvbuf_scale; 182aa0a1e58SJeff Roberson /* 183aa0a1e58SJeff Roberson * Compute bytes in the receive queue and socket buffer. 184aa0a1e58SJeff Roberson */ 185aa0a1e58SJeff Roberson bytes_in_process = (posted - SDP_MIN_TX_CREDITS) * buffer_size; 186aa0a1e58SJeff Roberson bytes_in_process += ssk->socket->so_rcv.sb_cc; 187aa0a1e58SJeff Roberson 188aa0a1e58SJeff Roberson return bytes_in_process < max_bytes; 189aa0a1e58SJeff Roberson } 190aa0a1e58SJeff Roberson 191aa0a1e58SJeff Roberson static inline void 192aa0a1e58SJeff Roberson sdp_post_recvs(struct sdp_sock *ssk) 193aa0a1e58SJeff Roberson { 194aa0a1e58SJeff Roberson 195aa0a1e58SJeff Roberson while (sdp_post_recvs_needed(ssk)) 196aa0a1e58SJeff Roberson if (sdp_post_recv(ssk)) 197aa0a1e58SJeff Roberson return; 198aa0a1e58SJeff Roberson } 199aa0a1e58SJeff Roberson 200aa0a1e58SJeff Roberson static inline struct mbuf * 201aa0a1e58SJeff Roberson sdp_sock_queue_rcv_mb(struct socket *sk, struct mbuf *mb) 202aa0a1e58SJeff Roberson { 203aa0a1e58SJeff Roberson struct sdp_sock *ssk = sdp_sk(sk); 204aa0a1e58SJeff Roberson struct sdp_bsdh *h; 205aa0a1e58SJeff Roberson 206aa0a1e58SJeff Roberson h = mtod(mb, struct sdp_bsdh *); 207aa0a1e58SJeff Roberson 208aa0a1e58SJeff Roberson #ifdef SDP_ZCOPY 209aa0a1e58SJeff Roberson SDP_SKB_CB(mb)->seq = rcv_nxt(ssk); 210aa0a1e58SJeff Roberson if (h->mid == SDP_MID_SRCAVAIL) { 211aa0a1e58SJeff Roberson struct sdp_srcah *srcah = (struct sdp_srcah *)(h+1); 212aa0a1e58SJeff Roberson struct rx_srcavail_state *rx_sa; 213aa0a1e58SJeff Roberson 214aa0a1e58SJeff Roberson ssk->srcavail_cancel_mseq = 0; 215aa0a1e58SJeff Roberson 216aa0a1e58SJeff Roberson ssk->rx_sa = rx_sa = RX_SRCAVAIL_STATE(mb) = kzalloc( 217aa0a1e58SJeff Roberson sizeof(struct rx_srcavail_state), M_NOWAIT); 218aa0a1e58SJeff Roberson 219aa0a1e58SJeff Roberson rx_sa->mseq = ntohl(h->mseq); 220aa0a1e58SJeff Roberson rx_sa->used = 0; 221aa0a1e58SJeff Roberson rx_sa->len = mb_len = ntohl(srcah->len); 222aa0a1e58SJeff Roberson rx_sa->rkey = ntohl(srcah->rkey); 223aa0a1e58SJeff Roberson rx_sa->vaddr = be64_to_cpu(srcah->vaddr); 224aa0a1e58SJeff Roberson rx_sa->flags = 0; 225aa0a1e58SJeff Roberson 226aa0a1e58SJeff Roberson if (ssk->tx_sa) { 227aa0a1e58SJeff Roberson sdp_dbg_data(ssk->socket, "got RX SrcAvail while waiting " 228aa0a1e58SJeff Roberson "for TX SrcAvail. waking up TX SrcAvail" 229aa0a1e58SJeff Roberson "to be aborted\n"); 230aa0a1e58SJeff Roberson wake_up(sk->sk_sleep); 231aa0a1e58SJeff Roberson } 232aa0a1e58SJeff Roberson 233aa0a1e58SJeff Roberson atomic_add(mb->len, &ssk->rcv_nxt); 234aa0a1e58SJeff Roberson sdp_dbg_data(sk, "queueing SrcAvail. mb_len = %d vaddr = %lld\n", 235aa0a1e58SJeff Roberson mb_len, rx_sa->vaddr); 236aa0a1e58SJeff Roberson } else 237aa0a1e58SJeff Roberson #endif 238aa0a1e58SJeff Roberson { 239aa0a1e58SJeff Roberson atomic_add(mb->m_pkthdr.len, &ssk->rcv_nxt); 240aa0a1e58SJeff Roberson } 241aa0a1e58SJeff Roberson 242aa0a1e58SJeff Roberson m_adj(mb, SDP_HEAD_SIZE); 243aa0a1e58SJeff Roberson SOCKBUF_LOCK(&sk->so_rcv); 244aa0a1e58SJeff Roberson if (unlikely(h->flags & SDP_OOB_PRES)) 245aa0a1e58SJeff Roberson sdp_urg(ssk, mb); 246aa0a1e58SJeff Roberson sbappend_locked(&sk->so_rcv, mb); 247aa0a1e58SJeff Roberson sorwakeup_locked(sk); 248aa0a1e58SJeff Roberson return mb; 249aa0a1e58SJeff Roberson } 250aa0a1e58SJeff Roberson 251aa0a1e58SJeff Roberson static int 252aa0a1e58SJeff Roberson sdp_get_recv_bytes(struct sdp_sock *ssk, u32 new_size) 253aa0a1e58SJeff Roberson { 254aa0a1e58SJeff Roberson 255aa0a1e58SJeff Roberson return MIN(new_size, SDP_MAX_PACKET); 256aa0a1e58SJeff Roberson } 257aa0a1e58SJeff Roberson 258aa0a1e58SJeff Roberson int 259aa0a1e58SJeff Roberson sdp_init_buffers(struct sdp_sock *ssk, u32 new_size) 260aa0a1e58SJeff Roberson { 261aa0a1e58SJeff Roberson 262aa0a1e58SJeff Roberson ssk->recv_bytes = sdp_get_recv_bytes(ssk, new_size); 263aa0a1e58SJeff Roberson sdp_post_recvs(ssk); 264aa0a1e58SJeff Roberson 265aa0a1e58SJeff Roberson return 0; 266aa0a1e58SJeff Roberson } 267aa0a1e58SJeff Roberson 268aa0a1e58SJeff Roberson int 269aa0a1e58SJeff Roberson sdp_resize_buffers(struct sdp_sock *ssk, u32 new_size) 270aa0a1e58SJeff Roberson { 271aa0a1e58SJeff Roberson u32 curr_size = ssk->recv_bytes; 272aa0a1e58SJeff Roberson u32 max_size = SDP_MAX_PACKET; 273aa0a1e58SJeff Roberson 274aa0a1e58SJeff Roberson if (new_size > curr_size && new_size <= max_size) { 275aa0a1e58SJeff Roberson ssk->recv_bytes = sdp_get_recv_bytes(ssk, new_size); 276aa0a1e58SJeff Roberson return 0; 277aa0a1e58SJeff Roberson } 278aa0a1e58SJeff Roberson return -1; 279aa0a1e58SJeff Roberson } 280aa0a1e58SJeff Roberson 281aa0a1e58SJeff Roberson static void 282aa0a1e58SJeff Roberson sdp_handle_resize_request(struct sdp_sock *ssk, struct sdp_chrecvbuf *buf) 283aa0a1e58SJeff Roberson { 284aa0a1e58SJeff Roberson if (sdp_resize_buffers(ssk, ntohl(buf->size)) == 0) 285aa0a1e58SJeff Roberson ssk->recv_request_head = ring_head(ssk->rx_ring) + 1; 286aa0a1e58SJeff Roberson else 287aa0a1e58SJeff Roberson ssk->recv_request_head = ring_tail(ssk->rx_ring); 288aa0a1e58SJeff Roberson ssk->recv_request = 1; 289aa0a1e58SJeff Roberson } 290aa0a1e58SJeff Roberson 291aa0a1e58SJeff Roberson static void 292aa0a1e58SJeff Roberson sdp_handle_resize_ack(struct sdp_sock *ssk, struct sdp_chrecvbuf *buf) 293aa0a1e58SJeff Roberson { 294aa0a1e58SJeff Roberson u32 new_size = ntohl(buf->size); 295aa0a1e58SJeff Roberson 296aa0a1e58SJeff Roberson if (new_size > ssk->xmit_size_goal) 297aa0a1e58SJeff Roberson ssk->xmit_size_goal = new_size; 298aa0a1e58SJeff Roberson } 299aa0a1e58SJeff Roberson 300aa0a1e58SJeff Roberson static struct mbuf * 301aa0a1e58SJeff Roberson sdp_recv_completion(struct sdp_sock *ssk, int id) 302aa0a1e58SJeff Roberson { 303aa0a1e58SJeff Roberson struct sdp_buf *rx_req; 304aa0a1e58SJeff Roberson struct ib_device *dev; 305aa0a1e58SJeff Roberson struct mbuf *mb; 306aa0a1e58SJeff Roberson 307aa0a1e58SJeff Roberson if (unlikely(id != ring_tail(ssk->rx_ring))) { 308aa0a1e58SJeff Roberson printk(KERN_WARNING "Bogus recv completion id %d tail %d\n", 309aa0a1e58SJeff Roberson id, ring_tail(ssk->rx_ring)); 310aa0a1e58SJeff Roberson return NULL; 311aa0a1e58SJeff Roberson } 312aa0a1e58SJeff Roberson 313aa0a1e58SJeff Roberson dev = ssk->ib_device; 314aa0a1e58SJeff Roberson rx_req = &ssk->rx_ring.buffer[id & (SDP_RX_SIZE - 1)]; 315aa0a1e58SJeff Roberson mb = rx_req->mb; 316aa0a1e58SJeff Roberson sdp_cleanup_sdp_buf(ssk, rx_req, DMA_FROM_DEVICE); 317aa0a1e58SJeff Roberson 318aa0a1e58SJeff Roberson atomic_inc(&ssk->rx_ring.tail); 319aa0a1e58SJeff Roberson atomic_dec(&ssk->remote_credits); 320aa0a1e58SJeff Roberson return mb; 321aa0a1e58SJeff Roberson } 322aa0a1e58SJeff Roberson 323aa0a1e58SJeff Roberson /* socket lock should be taken before calling this */ 324aa0a1e58SJeff Roberson static int 325aa0a1e58SJeff Roberson sdp_process_rx_ctl_mb(struct sdp_sock *ssk, struct mbuf *mb) 326aa0a1e58SJeff Roberson { 327aa0a1e58SJeff Roberson struct sdp_bsdh *h; 328aa0a1e58SJeff Roberson struct socket *sk; 329aa0a1e58SJeff Roberson 330aa0a1e58SJeff Roberson SDP_WLOCK_ASSERT(ssk); 331aa0a1e58SJeff Roberson sk = ssk->socket; 332aa0a1e58SJeff Roberson h = mtod(mb, struct sdp_bsdh *); 333aa0a1e58SJeff Roberson switch (h->mid) { 334aa0a1e58SJeff Roberson case SDP_MID_DATA: 335aa0a1e58SJeff Roberson case SDP_MID_SRCAVAIL: 336aa0a1e58SJeff Roberson sdp_dbg(sk, "DATA after socket rcv was shutdown\n"); 337aa0a1e58SJeff Roberson 338aa0a1e58SJeff Roberson /* got data in RCV_SHUTDOWN */ 339aa0a1e58SJeff Roberson if (ssk->state == TCPS_FIN_WAIT_1) { 340aa0a1e58SJeff Roberson sdp_dbg(sk, "RX data when state = FIN_WAIT1\n"); 341aa0a1e58SJeff Roberson sdp_notify(ssk, ECONNRESET); 342aa0a1e58SJeff Roberson } 343aa0a1e58SJeff Roberson m_freem(mb); 344aa0a1e58SJeff Roberson 345aa0a1e58SJeff Roberson break; 346aa0a1e58SJeff Roberson #ifdef SDP_ZCOPY 347aa0a1e58SJeff Roberson case SDP_MID_RDMARDCOMPL: 348aa0a1e58SJeff Roberson m_freem(mb); 349aa0a1e58SJeff Roberson break; 350aa0a1e58SJeff Roberson case SDP_MID_SENDSM: 351aa0a1e58SJeff Roberson sdp_handle_sendsm(ssk, ntohl(h->mseq_ack)); 352aa0a1e58SJeff Roberson m_freem(mb); 353aa0a1e58SJeff Roberson break; 354aa0a1e58SJeff Roberson case SDP_MID_SRCAVAIL_CANCEL: 355aa0a1e58SJeff Roberson sdp_dbg_data(sk, "Handling SrcAvailCancel\n"); 356aa0a1e58SJeff Roberson sdp_prf(sk, NULL, "Handling SrcAvailCancel"); 357aa0a1e58SJeff Roberson if (ssk->rx_sa) { 358aa0a1e58SJeff Roberson ssk->srcavail_cancel_mseq = ntohl(h->mseq); 359aa0a1e58SJeff Roberson ssk->rx_sa->flags |= RX_SA_ABORTED; 360aa0a1e58SJeff Roberson ssk->rx_sa = NULL; /* TODO: change it into SDP_MID_DATA and get 361aa0a1e58SJeff Roberson the dirty logic from recvmsg */ 362aa0a1e58SJeff Roberson } else { 363aa0a1e58SJeff Roberson sdp_dbg(sk, "Got SrcAvailCancel - " 364aa0a1e58SJeff Roberson "but no SrcAvail in process\n"); 365aa0a1e58SJeff Roberson } 366aa0a1e58SJeff Roberson m_freem(mb); 367aa0a1e58SJeff Roberson break; 368aa0a1e58SJeff Roberson case SDP_MID_SINKAVAIL: 369aa0a1e58SJeff Roberson sdp_dbg_data(sk, "Got SinkAvail - not supported: ignored\n"); 370aa0a1e58SJeff Roberson sdp_prf(sk, NULL, "Got SinkAvail - not supported: ignored"); 371aa0a1e58SJeff Roberson /* FALLTHROUGH */ 372aa0a1e58SJeff Roberson #endif 373aa0a1e58SJeff Roberson case SDP_MID_ABORT: 374aa0a1e58SJeff Roberson sdp_dbg_data(sk, "Handling ABORT\n"); 375aa0a1e58SJeff Roberson sdp_prf(sk, NULL, "Handling ABORT"); 376aa0a1e58SJeff Roberson sdp_notify(ssk, ECONNRESET); 377aa0a1e58SJeff Roberson m_freem(mb); 378aa0a1e58SJeff Roberson break; 379aa0a1e58SJeff Roberson case SDP_MID_DISCONN: 380aa0a1e58SJeff Roberson sdp_dbg_data(sk, "Handling DISCONN\n"); 381aa0a1e58SJeff Roberson sdp_prf(sk, NULL, "Handling DISCONN"); 382aa0a1e58SJeff Roberson sdp_handle_disconn(ssk); 383aa0a1e58SJeff Roberson break; 384aa0a1e58SJeff Roberson case SDP_MID_CHRCVBUF: 385aa0a1e58SJeff Roberson sdp_dbg_data(sk, "Handling RX CHRCVBUF\n"); 386aa0a1e58SJeff Roberson sdp_handle_resize_request(ssk, (struct sdp_chrecvbuf *)(h+1)); 387aa0a1e58SJeff Roberson m_freem(mb); 388aa0a1e58SJeff Roberson break; 389aa0a1e58SJeff Roberson case SDP_MID_CHRCVBUF_ACK: 390aa0a1e58SJeff Roberson sdp_dbg_data(sk, "Handling RX CHRCVBUF_ACK\n"); 391aa0a1e58SJeff Roberson sdp_handle_resize_ack(ssk, (struct sdp_chrecvbuf *)(h+1)); 392aa0a1e58SJeff Roberson m_freem(mb); 393aa0a1e58SJeff Roberson break; 394aa0a1e58SJeff Roberson default: 395aa0a1e58SJeff Roberson /* TODO: Handle other messages */ 396aa0a1e58SJeff Roberson sdp_warn(sk, "SDP: FIXME MID %d\n", h->mid); 397aa0a1e58SJeff Roberson m_freem(mb); 398aa0a1e58SJeff Roberson } 399aa0a1e58SJeff Roberson 400aa0a1e58SJeff Roberson return 0; 401aa0a1e58SJeff Roberson } 402aa0a1e58SJeff Roberson 403aa0a1e58SJeff Roberson static int 404aa0a1e58SJeff Roberson sdp_process_rx_mb(struct sdp_sock *ssk, struct mbuf *mb) 405aa0a1e58SJeff Roberson { 406aa0a1e58SJeff Roberson struct socket *sk; 407aa0a1e58SJeff Roberson struct sdp_bsdh *h; 408aa0a1e58SJeff Roberson unsigned long mseq_ack; 409aa0a1e58SJeff Roberson int credits_before; 410aa0a1e58SJeff Roberson 411aa0a1e58SJeff Roberson h = mtod(mb, struct sdp_bsdh *); 412aa0a1e58SJeff Roberson sk = ssk->socket; 413aa0a1e58SJeff Roberson /* 414aa0a1e58SJeff Roberson * If another thread is in so_pcbfree this may be partially torn 415aa0a1e58SJeff Roberson * down but no further synchronization is required as the destroying 416aa0a1e58SJeff Roberson * thread will wait for receive to shutdown before discarding the 417aa0a1e58SJeff Roberson * socket. 418aa0a1e58SJeff Roberson */ 419aa0a1e58SJeff Roberson if (sk == NULL) { 420aa0a1e58SJeff Roberson m_freem(mb); 421aa0a1e58SJeff Roberson return 0; 422aa0a1e58SJeff Roberson } 423aa0a1e58SJeff Roberson 424aa0a1e58SJeff Roberson SDPSTATS_HIST_LINEAR(credits_before_update, tx_credits(ssk)); 425aa0a1e58SJeff Roberson 426aa0a1e58SJeff Roberson mseq_ack = ntohl(h->mseq_ack); 427aa0a1e58SJeff Roberson credits_before = tx_credits(ssk); 428aa0a1e58SJeff Roberson atomic_set(&ssk->tx_ring.credits, mseq_ack - ring_head(ssk->tx_ring) + 429aa0a1e58SJeff Roberson 1 + ntohs(h->bufs)); 430aa0a1e58SJeff Roberson if (mseq_ack >= ssk->nagle_last_unacked) 431aa0a1e58SJeff Roberson ssk->nagle_last_unacked = 0; 432aa0a1e58SJeff Roberson 433aa0a1e58SJeff Roberson sdp_prf1(ssk->socket, mb, "RX %s +%d c:%d->%d mseq:%d ack:%d\n", 434aa0a1e58SJeff Roberson mid2str(h->mid), ntohs(h->bufs), credits_before, 435aa0a1e58SJeff Roberson tx_credits(ssk), ntohl(h->mseq), ntohl(h->mseq_ack)); 436aa0a1e58SJeff Roberson 437aa0a1e58SJeff Roberson if (unlikely(h->mid == SDP_MID_DATA && 438aa0a1e58SJeff Roberson mb->m_pkthdr.len == SDP_HEAD_SIZE)) { 439aa0a1e58SJeff Roberson /* Credit update is valid even after RCV_SHUTDOWN */ 440aa0a1e58SJeff Roberson m_freem(mb); 441aa0a1e58SJeff Roberson return 0; 442aa0a1e58SJeff Roberson } 443aa0a1e58SJeff Roberson 444aa0a1e58SJeff Roberson if ((h->mid != SDP_MID_DATA && h->mid != SDP_MID_SRCAVAIL) || 445aa0a1e58SJeff Roberson TCPS_HAVERCVDFIN(ssk->state)) { 446aa0a1e58SJeff Roberson sdp_prf(sk, NULL, "Control mb - queing to control queue"); 447aa0a1e58SJeff Roberson #ifdef SDP_ZCOPY 448aa0a1e58SJeff Roberson if (h->mid == SDP_MID_SRCAVAIL_CANCEL) { 449aa0a1e58SJeff Roberson sdp_dbg_data(sk, "Got SrcAvailCancel. " 450aa0a1e58SJeff Roberson "seq: 0x%d seq_ack: 0x%d\n", 451aa0a1e58SJeff Roberson ntohl(h->mseq), ntohl(h->mseq_ack)); 452aa0a1e58SJeff Roberson ssk->srcavail_cancel_mseq = ntohl(h->mseq); 453aa0a1e58SJeff Roberson } 454aa0a1e58SJeff Roberson 455aa0a1e58SJeff Roberson 456aa0a1e58SJeff Roberson if (h->mid == SDP_MID_RDMARDCOMPL) { 457aa0a1e58SJeff Roberson struct sdp_rrch *rrch = (struct sdp_rrch *)(h+1); 458aa0a1e58SJeff Roberson sdp_dbg_data(sk, "RdmaRdCompl message arrived\n"); 459aa0a1e58SJeff Roberson sdp_handle_rdma_read_compl(ssk, ntohl(h->mseq_ack), 460aa0a1e58SJeff Roberson ntohl(rrch->len)); 461aa0a1e58SJeff Roberson } 462aa0a1e58SJeff Roberson #endif 463aa0a1e58SJeff Roberson mb->m_nextpkt = NULL; 464aa0a1e58SJeff Roberson if (ssk->rx_ctl_tail) 465aa0a1e58SJeff Roberson ssk->rx_ctl_tail->m_nextpkt = mb; 466aa0a1e58SJeff Roberson else 467aa0a1e58SJeff Roberson ssk->rx_ctl_q = mb; 468aa0a1e58SJeff Roberson ssk->rx_ctl_tail = mb; 469aa0a1e58SJeff Roberson 470aa0a1e58SJeff Roberson return 0; 471aa0a1e58SJeff Roberson } 472aa0a1e58SJeff Roberson 473aa0a1e58SJeff Roberson sdp_prf1(sk, NULL, "queueing %s mb\n", mid2str(h->mid)); 474aa0a1e58SJeff Roberson mb = sdp_sock_queue_rcv_mb(sk, mb); 475aa0a1e58SJeff Roberson 476aa0a1e58SJeff Roberson 477aa0a1e58SJeff Roberson return 0; 478aa0a1e58SJeff Roberson } 479aa0a1e58SJeff Roberson 480aa0a1e58SJeff Roberson /* called only from irq */ 481aa0a1e58SJeff Roberson static struct mbuf * 482aa0a1e58SJeff Roberson sdp_process_rx_wc(struct sdp_sock *ssk, struct ib_wc *wc) 483aa0a1e58SJeff Roberson { 484aa0a1e58SJeff Roberson struct mbuf *mb; 485aa0a1e58SJeff Roberson struct sdp_bsdh *h; 486aa0a1e58SJeff Roberson struct socket *sk = ssk->socket; 487aa0a1e58SJeff Roberson int mseq; 488aa0a1e58SJeff Roberson 489aa0a1e58SJeff Roberson mb = sdp_recv_completion(ssk, wc->wr_id); 490aa0a1e58SJeff Roberson if (unlikely(!mb)) 491aa0a1e58SJeff Roberson return NULL; 492aa0a1e58SJeff Roberson 493aa0a1e58SJeff Roberson if (unlikely(wc->status)) { 494aa0a1e58SJeff Roberson if (ssk->qp_active && sk) { 495aa0a1e58SJeff Roberson sdp_dbg(sk, "Recv completion with error. " 496aa0a1e58SJeff Roberson "Status %d, vendor: %d\n", 497aa0a1e58SJeff Roberson wc->status, wc->vendor_err); 498aa0a1e58SJeff Roberson sdp_abort(sk); 499aa0a1e58SJeff Roberson ssk->qp_active = 0; 500aa0a1e58SJeff Roberson } 501aa0a1e58SJeff Roberson m_freem(mb); 502aa0a1e58SJeff Roberson return NULL; 503aa0a1e58SJeff Roberson } 504aa0a1e58SJeff Roberson 505aa0a1e58SJeff Roberson sdp_dbg_data(sk, "Recv completion. ID %d Length %d\n", 506aa0a1e58SJeff Roberson (int)wc->wr_id, wc->byte_len); 507aa0a1e58SJeff Roberson if (unlikely(wc->byte_len < sizeof(struct sdp_bsdh))) { 508aa0a1e58SJeff Roberson sdp_warn(sk, "SDP BUG! byte_len %d < %zd\n", 509aa0a1e58SJeff Roberson wc->byte_len, sizeof(struct sdp_bsdh)); 510aa0a1e58SJeff Roberson m_freem(mb); 511aa0a1e58SJeff Roberson return NULL; 512aa0a1e58SJeff Roberson } 513aa0a1e58SJeff Roberson /* Use m_adj to trim the tail of data we didn't use. */ 514aa0a1e58SJeff Roberson m_adj(mb, -(mb->m_pkthdr.len - wc->byte_len)); 515aa0a1e58SJeff Roberson h = mtod(mb, struct sdp_bsdh *); 516aa0a1e58SJeff Roberson 517aa0a1e58SJeff Roberson SDP_DUMP_PACKET(ssk->socket, "RX", mb, h); 518aa0a1e58SJeff Roberson 519aa0a1e58SJeff Roberson ssk->rx_packets++; 520aa0a1e58SJeff Roberson ssk->rx_bytes += mb->m_pkthdr.len; 521aa0a1e58SJeff Roberson 522aa0a1e58SJeff Roberson mseq = ntohl(h->mseq); 523aa0a1e58SJeff Roberson atomic_set(&ssk->mseq_ack, mseq); 524aa0a1e58SJeff Roberson if (mseq != (int)wc->wr_id) 525aa0a1e58SJeff Roberson sdp_warn(sk, "SDP BUG! mseq %d != wrid %d\n", 526aa0a1e58SJeff Roberson mseq, (int)wc->wr_id); 527aa0a1e58SJeff Roberson 528aa0a1e58SJeff Roberson return mb; 529aa0a1e58SJeff Roberson } 530aa0a1e58SJeff Roberson 531aa0a1e58SJeff Roberson /* Wakeup writers if we now have credits. */ 532aa0a1e58SJeff Roberson static void 533aa0a1e58SJeff Roberson sdp_bzcopy_write_space(struct sdp_sock *ssk) 534aa0a1e58SJeff Roberson { 535aa0a1e58SJeff Roberson struct socket *sk = ssk->socket; 536aa0a1e58SJeff Roberson 537aa0a1e58SJeff Roberson if (tx_credits(ssk) >= ssk->min_bufs && sk) 538aa0a1e58SJeff Roberson sowwakeup(sk); 539aa0a1e58SJeff Roberson } 540aa0a1e58SJeff Roberson 541aa0a1e58SJeff Roberson /* only from interrupt. */ 542aa0a1e58SJeff Roberson static int 543aa0a1e58SJeff Roberson sdp_poll_rx_cq(struct sdp_sock *ssk) 544aa0a1e58SJeff Roberson { 545aa0a1e58SJeff Roberson struct ib_cq *cq = ssk->rx_ring.cq; 546aa0a1e58SJeff Roberson struct ib_wc ibwc[SDP_NUM_WC]; 547aa0a1e58SJeff Roberson int n, i; 548aa0a1e58SJeff Roberson int wc_processed = 0; 549aa0a1e58SJeff Roberson struct mbuf *mb; 550aa0a1e58SJeff Roberson 551aa0a1e58SJeff Roberson do { 552aa0a1e58SJeff Roberson n = ib_poll_cq(cq, SDP_NUM_WC, ibwc); 553aa0a1e58SJeff Roberson for (i = 0; i < n; ++i) { 554aa0a1e58SJeff Roberson struct ib_wc *wc = &ibwc[i]; 555aa0a1e58SJeff Roberson 556aa0a1e58SJeff Roberson BUG_ON(!(wc->wr_id & SDP_OP_RECV)); 557aa0a1e58SJeff Roberson mb = sdp_process_rx_wc(ssk, wc); 558aa0a1e58SJeff Roberson if (!mb) 559aa0a1e58SJeff Roberson continue; 560aa0a1e58SJeff Roberson 561aa0a1e58SJeff Roberson sdp_process_rx_mb(ssk, mb); 562aa0a1e58SJeff Roberson wc_processed++; 563aa0a1e58SJeff Roberson } 564aa0a1e58SJeff Roberson } while (n == SDP_NUM_WC); 565aa0a1e58SJeff Roberson 566aa0a1e58SJeff Roberson if (wc_processed) 567aa0a1e58SJeff Roberson sdp_bzcopy_write_space(ssk); 568aa0a1e58SJeff Roberson 569aa0a1e58SJeff Roberson return wc_processed; 570aa0a1e58SJeff Roberson } 571aa0a1e58SJeff Roberson 572aa0a1e58SJeff Roberson static void 573aa0a1e58SJeff Roberson sdp_rx_comp_work(struct work_struct *work) 574aa0a1e58SJeff Roberson { 575aa0a1e58SJeff Roberson struct sdp_sock *ssk = container_of(work, struct sdp_sock, 576aa0a1e58SJeff Roberson rx_comp_work); 577aa0a1e58SJeff Roberson 578aa0a1e58SJeff Roberson sdp_prf(ssk->socket, NULL, "%s", __func__); 579aa0a1e58SJeff Roberson 580aa0a1e58SJeff Roberson SDP_WLOCK(ssk); 581aa0a1e58SJeff Roberson if (unlikely(!ssk->qp)) { 582aa0a1e58SJeff Roberson sdp_prf(ssk->socket, NULL, "qp was destroyed"); 583aa0a1e58SJeff Roberson goto out; 584aa0a1e58SJeff Roberson } 585aa0a1e58SJeff Roberson if (unlikely(!ssk->rx_ring.cq)) { 586aa0a1e58SJeff Roberson sdp_prf(ssk->socket, NULL, "rx_ring.cq is NULL"); 587aa0a1e58SJeff Roberson goto out; 588aa0a1e58SJeff Roberson } 589aa0a1e58SJeff Roberson 590aa0a1e58SJeff Roberson if (unlikely(!ssk->poll_cq)) { 591aa0a1e58SJeff Roberson struct rdma_cm_id *id = ssk->id; 592aa0a1e58SJeff Roberson if (id && id->qp) 5933910bc63SDimitry Andric rdma_notify(id, IB_EVENT_COMM_EST); 594aa0a1e58SJeff Roberson goto out; 595aa0a1e58SJeff Roberson } 596aa0a1e58SJeff Roberson 597aa0a1e58SJeff Roberson sdp_do_posts(ssk); 598aa0a1e58SJeff Roberson out: 599aa0a1e58SJeff Roberson SDP_WUNLOCK(ssk); 600aa0a1e58SJeff Roberson } 601aa0a1e58SJeff Roberson 602aa0a1e58SJeff Roberson void 603aa0a1e58SJeff Roberson sdp_do_posts(struct sdp_sock *ssk) 604aa0a1e58SJeff Roberson { 605aa0a1e58SJeff Roberson struct socket *sk = ssk->socket; 606aa0a1e58SJeff Roberson int xmit_poll_force; 607aa0a1e58SJeff Roberson struct mbuf *mb; 608aa0a1e58SJeff Roberson 609aa0a1e58SJeff Roberson SDP_WLOCK_ASSERT(ssk); 610aa0a1e58SJeff Roberson if (!ssk->qp_active) { 611aa0a1e58SJeff Roberson sdp_dbg(sk, "QP is deactivated\n"); 612aa0a1e58SJeff Roberson return; 613aa0a1e58SJeff Roberson } 614aa0a1e58SJeff Roberson 615aa0a1e58SJeff Roberson while ((mb = ssk->rx_ctl_q)) { 616aa0a1e58SJeff Roberson ssk->rx_ctl_q = mb->m_nextpkt; 617aa0a1e58SJeff Roberson mb->m_nextpkt = NULL; 618aa0a1e58SJeff Roberson sdp_process_rx_ctl_mb(ssk, mb); 619aa0a1e58SJeff Roberson } 620aa0a1e58SJeff Roberson 621aa0a1e58SJeff Roberson if (ssk->state == TCPS_TIME_WAIT) 622aa0a1e58SJeff Roberson return; 623aa0a1e58SJeff Roberson 624aa0a1e58SJeff Roberson if (!ssk->rx_ring.cq || !ssk->tx_ring.cq) 625aa0a1e58SJeff Roberson return; 626aa0a1e58SJeff Roberson 627aa0a1e58SJeff Roberson sdp_post_recvs(ssk); 628aa0a1e58SJeff Roberson 629aa0a1e58SJeff Roberson if (tx_ring_posted(ssk)) 630aa0a1e58SJeff Roberson sdp_xmit_poll(ssk, 1); 631aa0a1e58SJeff Roberson 632aa0a1e58SJeff Roberson sdp_post_sends(ssk, M_NOWAIT); 633aa0a1e58SJeff Roberson 634aa0a1e58SJeff Roberson xmit_poll_force = tx_credits(ssk) < SDP_MIN_TX_CREDITS; 635aa0a1e58SJeff Roberson 636aa0a1e58SJeff Roberson if (credit_update_needed(ssk) || xmit_poll_force) { 637aa0a1e58SJeff Roberson /* if has pending tx because run out of tx_credits - xmit it */ 638aa0a1e58SJeff Roberson sdp_prf(sk, NULL, "Processing to free pending sends"); 639aa0a1e58SJeff Roberson sdp_xmit_poll(ssk, xmit_poll_force); 640aa0a1e58SJeff Roberson sdp_prf(sk, NULL, "Sending credit update"); 641aa0a1e58SJeff Roberson sdp_post_sends(ssk, M_NOWAIT); 642aa0a1e58SJeff Roberson } 643aa0a1e58SJeff Roberson 644aa0a1e58SJeff Roberson } 645aa0a1e58SJeff Roberson 646aa0a1e58SJeff Roberson int 647aa0a1e58SJeff Roberson sdp_process_rx(struct sdp_sock *ssk) 648aa0a1e58SJeff Roberson { 649aa0a1e58SJeff Roberson int wc_processed = 0; 650aa0a1e58SJeff Roberson int credits_before; 651aa0a1e58SJeff Roberson 652aa0a1e58SJeff Roberson if (!rx_ring_trylock(&ssk->rx_ring)) { 653aa0a1e58SJeff Roberson sdp_dbg(ssk->socket, "ring destroyed. not polling it\n"); 654aa0a1e58SJeff Roberson return 0; 655aa0a1e58SJeff Roberson } 656aa0a1e58SJeff Roberson 657aa0a1e58SJeff Roberson credits_before = tx_credits(ssk); 658aa0a1e58SJeff Roberson 659aa0a1e58SJeff Roberson wc_processed = sdp_poll_rx_cq(ssk); 660aa0a1e58SJeff Roberson sdp_prf(ssk->socket, NULL, "processed %d", wc_processed); 661aa0a1e58SJeff Roberson 662aa0a1e58SJeff Roberson if (wc_processed) { 663aa0a1e58SJeff Roberson sdp_prf(ssk->socket, NULL, "credits: %d -> %d", 664aa0a1e58SJeff Roberson credits_before, tx_credits(ssk)); 665aa0a1e58SJeff Roberson queue_work(rx_comp_wq, &ssk->rx_comp_work); 666aa0a1e58SJeff Roberson } 667aa0a1e58SJeff Roberson sdp_arm_rx_cq(ssk); 668aa0a1e58SJeff Roberson 669aa0a1e58SJeff Roberson rx_ring_unlock(&ssk->rx_ring); 670aa0a1e58SJeff Roberson 671aa0a1e58SJeff Roberson return (wc_processed); 672aa0a1e58SJeff Roberson } 673aa0a1e58SJeff Roberson 674aa0a1e58SJeff Roberson static void 675aa0a1e58SJeff Roberson sdp_rx_irq(struct ib_cq *cq, void *cq_context) 676aa0a1e58SJeff Roberson { 677aa0a1e58SJeff Roberson struct socket *sk = cq_context; 678aa0a1e58SJeff Roberson struct sdp_sock *ssk = sdp_sk(sk); 679aa0a1e58SJeff Roberson 680aa0a1e58SJeff Roberson if (cq != ssk->rx_ring.cq) { 681aa0a1e58SJeff Roberson sdp_dbg(sk, "cq = %p, ssk->cq = %p\n", cq, ssk->rx_ring.cq); 682aa0a1e58SJeff Roberson return; 683aa0a1e58SJeff Roberson } 684aa0a1e58SJeff Roberson 685aa0a1e58SJeff Roberson SDPSTATS_COUNTER_INC(rx_int_count); 686aa0a1e58SJeff Roberson 687aa0a1e58SJeff Roberson sdp_prf(sk, NULL, "rx irq"); 688aa0a1e58SJeff Roberson 689aa0a1e58SJeff Roberson sdp_process_rx(ssk); 690aa0a1e58SJeff Roberson } 691aa0a1e58SJeff Roberson 692aa0a1e58SJeff Roberson static 693aa0a1e58SJeff Roberson void sdp_rx_ring_purge(struct sdp_sock *ssk) 694aa0a1e58SJeff Roberson { 695aa0a1e58SJeff Roberson while (rx_ring_posted(ssk) > 0) { 696aa0a1e58SJeff Roberson struct mbuf *mb; 697aa0a1e58SJeff Roberson mb = sdp_recv_completion(ssk, ring_tail(ssk->rx_ring)); 698aa0a1e58SJeff Roberson if (!mb) 699aa0a1e58SJeff Roberson break; 700aa0a1e58SJeff Roberson m_freem(mb); 701aa0a1e58SJeff Roberson } 702aa0a1e58SJeff Roberson } 703aa0a1e58SJeff Roberson 704aa0a1e58SJeff Roberson void 705aa0a1e58SJeff Roberson sdp_rx_ring_init(struct sdp_sock *ssk) 706aa0a1e58SJeff Roberson { 707aa0a1e58SJeff Roberson ssk->rx_ring.buffer = NULL; 708aa0a1e58SJeff Roberson ssk->rx_ring.destroyed = 0; 709aa0a1e58SJeff Roberson rw_init(&ssk->rx_ring.destroyed_lock, "sdp rx lock"); 710aa0a1e58SJeff Roberson } 711aa0a1e58SJeff Roberson 712aa0a1e58SJeff Roberson static void 713aa0a1e58SJeff Roberson sdp_rx_cq_event_handler(struct ib_event *event, void *data) 714aa0a1e58SJeff Roberson { 715aa0a1e58SJeff Roberson } 716aa0a1e58SJeff Roberson 717aa0a1e58SJeff Roberson int 718aa0a1e58SJeff Roberson sdp_rx_ring_create(struct sdp_sock *ssk, struct ib_device *device) 719aa0a1e58SJeff Roberson { 720aa0a1e58SJeff Roberson struct ib_cq *rx_cq; 721aa0a1e58SJeff Roberson int rc = 0; 722aa0a1e58SJeff Roberson 723aa0a1e58SJeff Roberson 724aa0a1e58SJeff Roberson sdp_dbg(ssk->socket, "rx ring created"); 725aa0a1e58SJeff Roberson INIT_WORK(&ssk->rx_comp_work, sdp_rx_comp_work); 726aa0a1e58SJeff Roberson atomic_set(&ssk->rx_ring.head, 1); 727aa0a1e58SJeff Roberson atomic_set(&ssk->rx_ring.tail, 1); 728aa0a1e58SJeff Roberson 729aa0a1e58SJeff Roberson ssk->rx_ring.buffer = kmalloc( 730aa0a1e58SJeff Roberson sizeof *ssk->rx_ring.buffer * SDP_RX_SIZE, GFP_KERNEL); 731aa0a1e58SJeff Roberson if (!ssk->rx_ring.buffer) { 732aa0a1e58SJeff Roberson sdp_warn(ssk->socket, 733aa0a1e58SJeff Roberson "Unable to allocate RX Ring size %zd.\n", 734aa0a1e58SJeff Roberson sizeof(*ssk->rx_ring.buffer) * SDP_RX_SIZE); 735aa0a1e58SJeff Roberson 736aa0a1e58SJeff Roberson return -ENOMEM; 737aa0a1e58SJeff Roberson } 738aa0a1e58SJeff Roberson 739aa0a1e58SJeff Roberson rx_cq = ib_create_cq(device, sdp_rx_irq, sdp_rx_cq_event_handler, 740aa0a1e58SJeff Roberson ssk->socket, SDP_RX_SIZE, IB_CQ_VECTOR_LEAST_ATTACHED); 741aa0a1e58SJeff Roberson 742aa0a1e58SJeff Roberson if (IS_ERR(rx_cq)) { 743aa0a1e58SJeff Roberson rc = PTR_ERR(rx_cq); 744aa0a1e58SJeff Roberson sdp_warn(ssk->socket, "Unable to allocate RX CQ: %d.\n", rc); 745aa0a1e58SJeff Roberson goto err_cq; 746aa0a1e58SJeff Roberson } 747aa0a1e58SJeff Roberson 748aa0a1e58SJeff Roberson sdp_sk(ssk->socket)->rx_ring.cq = rx_cq; 749aa0a1e58SJeff Roberson sdp_arm_rx_cq(ssk); 750aa0a1e58SJeff Roberson 751aa0a1e58SJeff Roberson return 0; 752aa0a1e58SJeff Roberson 753aa0a1e58SJeff Roberson err_cq: 754aa0a1e58SJeff Roberson kfree(ssk->rx_ring.buffer); 755aa0a1e58SJeff Roberson ssk->rx_ring.buffer = NULL; 756aa0a1e58SJeff Roberson return rc; 757aa0a1e58SJeff Roberson } 758aa0a1e58SJeff Roberson 759aa0a1e58SJeff Roberson void 760aa0a1e58SJeff Roberson sdp_rx_ring_destroy(struct sdp_sock *ssk) 761aa0a1e58SJeff Roberson { 762aa0a1e58SJeff Roberson 763aa0a1e58SJeff Roberson cancel_work_sync(&ssk->rx_comp_work); 764aa0a1e58SJeff Roberson rx_ring_destroy_lock(&ssk->rx_ring); 765aa0a1e58SJeff Roberson 766aa0a1e58SJeff Roberson if (ssk->rx_ring.buffer) { 767aa0a1e58SJeff Roberson sdp_rx_ring_purge(ssk); 768aa0a1e58SJeff Roberson 769aa0a1e58SJeff Roberson kfree(ssk->rx_ring.buffer); 770aa0a1e58SJeff Roberson ssk->rx_ring.buffer = NULL; 771aa0a1e58SJeff Roberson } 772aa0a1e58SJeff Roberson 773aa0a1e58SJeff Roberson if (ssk->rx_ring.cq) { 774aa0a1e58SJeff Roberson if (ib_destroy_cq(ssk->rx_ring.cq)) { 775aa0a1e58SJeff Roberson sdp_warn(ssk->socket, "destroy cq(%p) failed\n", 776aa0a1e58SJeff Roberson ssk->rx_ring.cq); 777aa0a1e58SJeff Roberson } else { 778aa0a1e58SJeff Roberson ssk->rx_ring.cq = NULL; 779aa0a1e58SJeff Roberson } 780aa0a1e58SJeff Roberson } 781aa0a1e58SJeff Roberson 782aa0a1e58SJeff Roberson WARN_ON(ring_head(ssk->rx_ring) != ring_tail(ssk->rx_ring)); 783aa0a1e58SJeff Roberson } 784