1 /* 2 * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. 3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/io.h> 35 36 #include "qib.h" 37 38 /* cut down ridiculously long IB macro names */ 39 #define OP(x) IB_OPCODE_RC_##x 40 41 42 static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, 43 u32 psn, u32 pmtu) 44 { 45 u32 len; 46 47 len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu; 48 return rvt_restart_sge(ss, wqe, len); 49 } 50 51 /** 52 * qib_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) 53 * @dev: the device for this QP 54 * @qp: a pointer to the QP 55 * @ohdr: a pointer to the IB header being constructed 56 * @pmtu: the path MTU 57 * 58 * Return 1 if constructed; otherwise, return 0. 59 * Note that we are in the responder's side of the QP context. 60 * Note the QP s_lock must be held. 61 */ 62 static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp, 63 struct ib_other_headers *ohdr, u32 pmtu) 64 { 65 struct rvt_ack_entry *e; 66 u32 hwords; 67 u32 len; 68 u32 bth0; 69 u32 bth2; 70 71 /* Don't send an ACK if we aren't supposed to. */ 72 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) 73 goto bail; 74 75 /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 76 hwords = 5; 77 78 switch (qp->s_ack_state) { 79 case OP(RDMA_READ_RESPONSE_LAST): 80 case OP(RDMA_READ_RESPONSE_ONLY): 81 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 82 if (e->rdma_sge.mr) { 83 rvt_put_mr(e->rdma_sge.mr); 84 e->rdma_sge.mr = NULL; 85 } 86 /* FALLTHROUGH */ 87 case OP(ATOMIC_ACKNOWLEDGE): 88 /* 89 * We can increment the tail pointer now that the last 90 * response has been sent instead of only being 91 * constructed. 92 */ 93 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) 94 qp->s_tail_ack_queue = 0; 95 /* FALLTHROUGH */ 96 case OP(SEND_ONLY): 97 case OP(ACKNOWLEDGE): 98 /* Check for no next entry in the queue. */ 99 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { 100 if (qp->s_flags & RVT_S_ACK_PENDING) 101 goto normal; 102 goto bail; 103 } 104 105 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 106 if (e->opcode == OP(RDMA_READ_REQUEST)) { 107 /* 108 * If a RDMA read response is being resent and 109 * we haven't seen the duplicate request yet, 110 * then stop sending the remaining responses the 111 * responder has seen until the requester resends it. 112 */ 113 len = e->rdma_sge.sge_length; 114 if (len && !e->rdma_sge.mr) { 115 qp->s_tail_ack_queue = qp->r_head_ack_queue; 116 goto bail; 117 } 118 /* Copy SGE state in case we need to resend */ 119 qp->s_rdma_mr = e->rdma_sge.mr; 120 if (qp->s_rdma_mr) 121 rvt_get_mr(qp->s_rdma_mr); 122 qp->s_ack_rdma_sge.sge = e->rdma_sge; 123 qp->s_ack_rdma_sge.num_sge = 1; 124 qp->s_cur_sge = &qp->s_ack_rdma_sge; 125 if (len > pmtu) { 126 len = pmtu; 127 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); 128 } else { 129 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); 130 e->sent = 1; 131 } 132 ohdr->u.aeth = rvt_compute_aeth(qp); 133 hwords++; 134 qp->s_ack_rdma_psn = e->psn; 135 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; 136 } else { 137 /* COMPARE_SWAP or FETCH_ADD */ 138 qp->s_cur_sge = NULL; 139 len = 0; 140 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 141 ohdr->u.at.aeth = rvt_compute_aeth(qp); 142 ib_u64_put(e->atomic_data, &ohdr->u.at.atomic_ack_eth); 143 hwords += sizeof(ohdr->u.at) / sizeof(u32); 144 bth2 = e->psn & QIB_PSN_MASK; 145 e->sent = 1; 146 } 147 bth0 = qp->s_ack_state << 24; 148 break; 149 150 case OP(RDMA_READ_RESPONSE_FIRST): 151 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); 152 /* FALLTHROUGH */ 153 case OP(RDMA_READ_RESPONSE_MIDDLE): 154 qp->s_cur_sge = &qp->s_ack_rdma_sge; 155 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; 156 if (qp->s_rdma_mr) 157 rvt_get_mr(qp->s_rdma_mr); 158 len = qp->s_ack_rdma_sge.sge.sge_length; 159 if (len > pmtu) 160 len = pmtu; 161 else { 162 ohdr->u.aeth = rvt_compute_aeth(qp); 163 hwords++; 164 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 165 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 166 e->sent = 1; 167 } 168 bth0 = qp->s_ack_state << 24; 169 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; 170 break; 171 172 default: 173 normal: 174 /* 175 * Send a regular ACK. 176 * Set the s_ack_state so we wait until after sending 177 * the ACK before setting s_ack_state to ACKNOWLEDGE 178 * (see above). 179 */ 180 qp->s_ack_state = OP(SEND_ONLY); 181 qp->s_flags &= ~RVT_S_ACK_PENDING; 182 qp->s_cur_sge = NULL; 183 if (qp->s_nak_state) 184 ohdr->u.aeth = 185 cpu_to_be32((qp->r_msn & IB_MSN_MASK) | 186 (qp->s_nak_state << 187 IB_AETH_CREDIT_SHIFT)); 188 else 189 ohdr->u.aeth = rvt_compute_aeth(qp); 190 hwords++; 191 len = 0; 192 bth0 = OP(ACKNOWLEDGE) << 24; 193 bth2 = qp->s_ack_psn & QIB_PSN_MASK; 194 } 195 qp->s_rdma_ack_cnt++; 196 qp->s_hdrwords = hwords; 197 qp->s_cur_size = len; 198 qib_make_ruc_header(qp, ohdr, bth0, bth2); 199 return 1; 200 201 bail: 202 qp->s_ack_state = OP(ACKNOWLEDGE); 203 qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING); 204 return 0; 205 } 206 207 /** 208 * qib_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) 209 * @qp: a pointer to the QP 210 * 211 * Assumes the s_lock is held. 212 * 213 * Return 1 if constructed; otherwise, return 0. 214 */ 215 int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags) 216 { 217 struct qib_qp_priv *priv = qp->priv; 218 struct qib_ibdev *dev = to_idev(qp->ibqp.device); 219 struct ib_other_headers *ohdr; 220 struct rvt_sge_state *ss; 221 struct rvt_swqe *wqe; 222 u32 hwords; 223 u32 len; 224 u32 bth0; 225 u32 bth2; 226 u32 pmtu = qp->pmtu; 227 char newreq; 228 int ret = 0; 229 int delta; 230 231 ohdr = &priv->s_hdr->u.oth; 232 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) 233 ohdr = &priv->s_hdr->u.l.oth; 234 235 /* Sending responses has higher priority over sending requests. */ 236 if ((qp->s_flags & RVT_S_RESP_PENDING) && 237 qib_make_rc_ack(dev, qp, ohdr, pmtu)) 238 goto done; 239 240 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { 241 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) 242 goto bail; 243 /* We are in the error state, flush the work request. */ 244 if (qp->s_last == READ_ONCE(qp->s_head)) 245 goto bail; 246 /* If DMAs are in progress, we can't flush immediately. */ 247 if (atomic_read(&priv->s_dma_busy)) { 248 qp->s_flags |= RVT_S_WAIT_DMA; 249 goto bail; 250 } 251 wqe = rvt_get_swqe_ptr(qp, qp->s_last); 252 rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ? 253 IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR); 254 /* will get called again */ 255 goto done; 256 } 257 258 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK)) 259 goto bail; 260 261 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { 262 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { 263 qp->s_flags |= RVT_S_WAIT_PSN; 264 goto bail; 265 } 266 qp->s_sending_psn = qp->s_psn; 267 qp->s_sending_hpsn = qp->s_psn - 1; 268 } 269 270 /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 271 hwords = 5; 272 bth0 = 0; 273 274 /* Send a request. */ 275 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); 276 switch (qp->s_state) { 277 default: 278 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) 279 goto bail; 280 /* 281 * Resend an old request or start a new one. 282 * 283 * We keep track of the current SWQE so that 284 * we don't reset the "furthest progress" state 285 * if we need to back up. 286 */ 287 newreq = 0; 288 if (qp->s_cur == qp->s_tail) { 289 /* Check if send work queue is empty. */ 290 if (qp->s_tail == READ_ONCE(qp->s_head)) 291 goto bail; 292 /* 293 * If a fence is requested, wait for previous 294 * RDMA read and atomic operations to finish. 295 */ 296 if ((wqe->wr.send_flags & IB_SEND_FENCE) && 297 qp->s_num_rd_atomic) { 298 qp->s_flags |= RVT_S_WAIT_FENCE; 299 goto bail; 300 } 301 newreq = 1; 302 qp->s_psn = wqe->psn; 303 } 304 /* 305 * Note that we have to be careful not to modify the 306 * original work request since we may need to resend 307 * it. 308 */ 309 len = wqe->length; 310 ss = &qp->s_sge; 311 bth2 = qp->s_psn & QIB_PSN_MASK; 312 switch (wqe->wr.opcode) { 313 case IB_WR_SEND: 314 case IB_WR_SEND_WITH_IMM: 315 /* If no credit, return. */ 316 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) && 317 rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) { 318 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT; 319 goto bail; 320 } 321 if (len > pmtu) { 322 qp->s_state = OP(SEND_FIRST); 323 len = pmtu; 324 break; 325 } 326 if (wqe->wr.opcode == IB_WR_SEND) 327 qp->s_state = OP(SEND_ONLY); 328 else { 329 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); 330 /* Immediate data comes after the BTH */ 331 ohdr->u.imm_data = wqe->wr.ex.imm_data; 332 hwords += 1; 333 } 334 if (wqe->wr.send_flags & IB_SEND_SOLICITED) 335 bth0 |= IB_BTH_SOLICITED; 336 bth2 |= IB_BTH_REQ_ACK; 337 if (++qp->s_cur == qp->s_size) 338 qp->s_cur = 0; 339 break; 340 341 case IB_WR_RDMA_WRITE: 342 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) 343 qp->s_lsn++; 344 goto no_flow_control; 345 case IB_WR_RDMA_WRITE_WITH_IMM: 346 /* If no credit, return. */ 347 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) && 348 rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) { 349 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT; 350 goto bail; 351 } 352 no_flow_control: 353 ohdr->u.rc.reth.vaddr = 354 cpu_to_be64(wqe->rdma_wr.remote_addr); 355 ohdr->u.rc.reth.rkey = 356 cpu_to_be32(wqe->rdma_wr.rkey); 357 ohdr->u.rc.reth.length = cpu_to_be32(len); 358 hwords += sizeof(struct ib_reth) / sizeof(u32); 359 if (len > pmtu) { 360 qp->s_state = OP(RDMA_WRITE_FIRST); 361 len = pmtu; 362 break; 363 } 364 if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE) 365 qp->s_state = OP(RDMA_WRITE_ONLY); 366 else { 367 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 368 /* Immediate data comes after RETH */ 369 ohdr->u.rc.imm_data = 370 wqe->rdma_wr.wr.ex.imm_data; 371 hwords += 1; 372 if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED) 373 bth0 |= IB_BTH_SOLICITED; 374 } 375 bth2 |= IB_BTH_REQ_ACK; 376 if (++qp->s_cur == qp->s_size) 377 qp->s_cur = 0; 378 break; 379 380 case IB_WR_RDMA_READ: 381 /* 382 * Don't allow more operations to be started 383 * than the QP limits allow. 384 */ 385 if (newreq) { 386 if (qp->s_num_rd_atomic >= 387 qp->s_max_rd_atomic) { 388 qp->s_flags |= RVT_S_WAIT_RDMAR; 389 goto bail; 390 } 391 qp->s_num_rd_atomic++; 392 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) 393 qp->s_lsn++; 394 } 395 396 ohdr->u.rc.reth.vaddr = 397 cpu_to_be64(wqe->rdma_wr.remote_addr); 398 ohdr->u.rc.reth.rkey = 399 cpu_to_be32(wqe->rdma_wr.rkey); 400 ohdr->u.rc.reth.length = cpu_to_be32(len); 401 qp->s_state = OP(RDMA_READ_REQUEST); 402 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 403 ss = NULL; 404 len = 0; 405 bth2 |= IB_BTH_REQ_ACK; 406 if (++qp->s_cur == qp->s_size) 407 qp->s_cur = 0; 408 break; 409 410 case IB_WR_ATOMIC_CMP_AND_SWP: 411 case IB_WR_ATOMIC_FETCH_AND_ADD: 412 /* 413 * Don't allow more operations to be started 414 * than the QP limits allow. 415 */ 416 if (newreq) { 417 if (qp->s_num_rd_atomic >= 418 qp->s_max_rd_atomic) { 419 qp->s_flags |= RVT_S_WAIT_RDMAR; 420 goto bail; 421 } 422 qp->s_num_rd_atomic++; 423 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) 424 qp->s_lsn++; 425 } 426 if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { 427 qp->s_state = OP(COMPARE_SWAP); 428 put_ib_ateth_swap(wqe->atomic_wr.swap, 429 &ohdr->u.atomic_eth); 430 put_ib_ateth_compare(wqe->atomic_wr.compare_add, 431 &ohdr->u.atomic_eth); 432 } else { 433 qp->s_state = OP(FETCH_ADD); 434 put_ib_ateth_swap(wqe->atomic_wr.compare_add, 435 &ohdr->u.atomic_eth); 436 put_ib_ateth_compare(0, &ohdr->u.atomic_eth); 437 } 438 put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, 439 &ohdr->u.atomic_eth); 440 ohdr->u.atomic_eth.rkey = cpu_to_be32( 441 wqe->atomic_wr.rkey); 442 hwords += sizeof(struct ib_atomic_eth) / sizeof(u32); 443 ss = NULL; 444 len = 0; 445 bth2 |= IB_BTH_REQ_ACK; 446 if (++qp->s_cur == qp->s_size) 447 qp->s_cur = 0; 448 break; 449 450 default: 451 goto bail; 452 } 453 qp->s_sge.sge = wqe->sg_list[0]; 454 qp->s_sge.sg_list = wqe->sg_list + 1; 455 qp->s_sge.num_sge = wqe->wr.num_sge; 456 qp->s_sge.total_len = wqe->length; 457 qp->s_len = wqe->length; 458 if (newreq) { 459 qp->s_tail++; 460 if (qp->s_tail >= qp->s_size) 461 qp->s_tail = 0; 462 } 463 if (wqe->wr.opcode == IB_WR_RDMA_READ) 464 qp->s_psn = wqe->lpsn + 1; 465 else 466 qp->s_psn++; 467 break; 468 469 case OP(RDMA_READ_RESPONSE_FIRST): 470 /* 471 * qp->s_state is normally set to the opcode of the 472 * last packet constructed for new requests and therefore 473 * is never set to RDMA read response. 474 * RDMA_READ_RESPONSE_FIRST is used by the ACK processing 475 * thread to indicate a SEND needs to be restarted from an 476 * earlier PSN without interferring with the sending thread. 477 * See qib_restart_rc(). 478 */ 479 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 480 /* FALLTHROUGH */ 481 case OP(SEND_FIRST): 482 qp->s_state = OP(SEND_MIDDLE); 483 /* FALLTHROUGH */ 484 case OP(SEND_MIDDLE): 485 bth2 = qp->s_psn++ & QIB_PSN_MASK; 486 ss = &qp->s_sge; 487 len = qp->s_len; 488 if (len > pmtu) { 489 len = pmtu; 490 break; 491 } 492 if (wqe->wr.opcode == IB_WR_SEND) 493 qp->s_state = OP(SEND_LAST); 494 else { 495 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 496 /* Immediate data comes after the BTH */ 497 ohdr->u.imm_data = wqe->wr.ex.imm_data; 498 hwords += 1; 499 } 500 if (wqe->wr.send_flags & IB_SEND_SOLICITED) 501 bth0 |= IB_BTH_SOLICITED; 502 bth2 |= IB_BTH_REQ_ACK; 503 qp->s_cur++; 504 if (qp->s_cur >= qp->s_size) 505 qp->s_cur = 0; 506 break; 507 508 case OP(RDMA_READ_RESPONSE_LAST): 509 /* 510 * qp->s_state is normally set to the opcode of the 511 * last packet constructed for new requests and therefore 512 * is never set to RDMA read response. 513 * RDMA_READ_RESPONSE_LAST is used by the ACK processing 514 * thread to indicate a RDMA write needs to be restarted from 515 * an earlier PSN without interferring with the sending thread. 516 * See qib_restart_rc(). 517 */ 518 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 519 /* FALLTHROUGH */ 520 case OP(RDMA_WRITE_FIRST): 521 qp->s_state = OP(RDMA_WRITE_MIDDLE); 522 /* FALLTHROUGH */ 523 case OP(RDMA_WRITE_MIDDLE): 524 bth2 = qp->s_psn++ & QIB_PSN_MASK; 525 ss = &qp->s_sge; 526 len = qp->s_len; 527 if (len > pmtu) { 528 len = pmtu; 529 break; 530 } 531 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) 532 qp->s_state = OP(RDMA_WRITE_LAST); 533 else { 534 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 535 /* Immediate data comes after the BTH */ 536 ohdr->u.imm_data = wqe->wr.ex.imm_data; 537 hwords += 1; 538 if (wqe->wr.send_flags & IB_SEND_SOLICITED) 539 bth0 |= IB_BTH_SOLICITED; 540 } 541 bth2 |= IB_BTH_REQ_ACK; 542 qp->s_cur++; 543 if (qp->s_cur >= qp->s_size) 544 qp->s_cur = 0; 545 break; 546 547 case OP(RDMA_READ_RESPONSE_MIDDLE): 548 /* 549 * qp->s_state is normally set to the opcode of the 550 * last packet constructed for new requests and therefore 551 * is never set to RDMA read response. 552 * RDMA_READ_RESPONSE_MIDDLE is used by the ACK processing 553 * thread to indicate a RDMA read needs to be restarted from 554 * an earlier PSN without interferring with the sending thread. 555 * See qib_restart_rc(). 556 */ 557 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; 558 ohdr->u.rc.reth.vaddr = 559 cpu_to_be64(wqe->rdma_wr.remote_addr + len); 560 ohdr->u.rc.reth.rkey = 561 cpu_to_be32(wqe->rdma_wr.rkey); 562 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); 563 qp->s_state = OP(RDMA_READ_REQUEST); 564 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 565 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; 566 qp->s_psn = wqe->lpsn + 1; 567 ss = NULL; 568 len = 0; 569 qp->s_cur++; 570 if (qp->s_cur == qp->s_size) 571 qp->s_cur = 0; 572 break; 573 } 574 qp->s_sending_hpsn = bth2; 575 delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8; 576 if (delta && delta % QIB_PSN_CREDIT == 0) 577 bth2 |= IB_BTH_REQ_ACK; 578 if (qp->s_flags & RVT_S_SEND_ONE) { 579 qp->s_flags &= ~RVT_S_SEND_ONE; 580 qp->s_flags |= RVT_S_WAIT_ACK; 581 bth2 |= IB_BTH_REQ_ACK; 582 } 583 qp->s_len -= len; 584 qp->s_hdrwords = hwords; 585 qp->s_cur_sge = ss; 586 qp->s_cur_size = len; 587 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); 588 done: 589 return 1; 590 bail: 591 qp->s_flags &= ~RVT_S_BUSY; 592 return ret; 593 } 594 595 /** 596 * qib_send_rc_ack - Construct an ACK packet and send it 597 * @qp: a pointer to the QP 598 * 599 * This is called from qib_rc_rcv() and qib_kreceive(). 600 * Note that RDMA reads and atomics are handled in the 601 * send side QP state and tasklet. 602 */ 603 void qib_send_rc_ack(struct rvt_qp *qp) 604 { 605 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); 606 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 607 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 608 u64 pbc; 609 u16 lrh0; 610 u32 bth0; 611 u32 hwords; 612 u32 pbufn; 613 u32 __iomem *piobuf; 614 struct ib_header hdr; 615 struct ib_other_headers *ohdr; 616 u32 control; 617 unsigned long flags; 618 619 spin_lock_irqsave(&qp->s_lock, flags); 620 621 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) 622 goto unlock; 623 624 /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ 625 if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt) 626 goto queue_ack; 627 628 /* Construct the header with s_lock held so APM doesn't change it. */ 629 ohdr = &hdr.u.oth; 630 lrh0 = QIB_LRH_BTH; 631 /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ 632 hwords = 6; 633 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & 634 IB_AH_GRH)) { 635 hwords += qib_make_grh(ibp, &hdr.u.l.grh, 636 rdma_ah_read_grh(&qp->remote_ah_attr), 637 hwords, 0); 638 ohdr = &hdr.u.l.oth; 639 lrh0 = QIB_LRH_GRH; 640 } 641 /* read pkey_index w/o lock (its atomic) */ 642 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); 643 if (qp->s_mig_state == IB_MIG_MIGRATED) 644 bth0 |= IB_BTH_MIG_REQ; 645 if (qp->r_nak_state) 646 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | 647 (qp->r_nak_state << 648 IB_AETH_CREDIT_SHIFT)); 649 else 650 ohdr->u.aeth = rvt_compute_aeth(qp); 651 lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 | 652 rdma_ah_get_sl(&qp->remote_ah_attr) << 4; 653 hdr.lrh[0] = cpu_to_be16(lrh0); 654 hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); 655 hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC); 656 hdr.lrh[3] = cpu_to_be16(ppd->lid | 657 rdma_ah_get_path_bits(&qp->remote_ah_attr)); 658 ohdr->bth[0] = cpu_to_be32(bth0); 659 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); 660 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); 661 662 spin_unlock_irqrestore(&qp->s_lock, flags); 663 664 /* Don't try to send ACKs if the link isn't ACTIVE */ 665 if (!(ppd->lflags & QIBL_LINKACTIVE)) 666 goto done; 667 668 control = dd->f_setpbc_control(ppd, hwords + SIZE_OF_CRC, 669 qp->s_srate, lrh0 >> 12); 670 /* length is + 1 for the control dword */ 671 pbc = ((u64) control << 32) | (hwords + 1); 672 673 piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn); 674 if (!piobuf) { 675 /* 676 * We are out of PIO buffers at the moment. 677 * Pass responsibility for sending the ACK to the 678 * send tasklet so that when a PIO buffer becomes 679 * available, the ACK is sent ahead of other outgoing 680 * packets. 681 */ 682 spin_lock_irqsave(&qp->s_lock, flags); 683 goto queue_ack; 684 } 685 686 /* 687 * Write the pbc. 688 * We have to flush after the PBC for correctness 689 * on some cpus or WC buffer can be written out of order. 690 */ 691 writeq(pbc, piobuf); 692 693 if (dd->flags & QIB_PIO_FLUSH_WC) { 694 u32 *hdrp = (u32 *) &hdr; 695 696 qib_flush_wc(); 697 qib_pio_copy(piobuf + 2, hdrp, hwords - 1); 698 qib_flush_wc(); 699 __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1); 700 } else 701 qib_pio_copy(piobuf + 2, (u32 *) &hdr, hwords); 702 703 if (dd->flags & QIB_USE_SPCL_TRIG) { 704 u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023; 705 706 qib_flush_wc(); 707 __raw_writel(0xaebecede, piobuf + spcl_off); 708 } 709 710 qib_flush_wc(); 711 qib_sendbuf_done(dd, pbufn); 712 713 this_cpu_inc(ibp->pmastats->n_unicast_xmit); 714 goto done; 715 716 queue_ack: 717 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { 718 this_cpu_inc(*ibp->rvp.rc_qacks); 719 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; 720 qp->s_nak_state = qp->r_nak_state; 721 qp->s_ack_psn = qp->r_ack_psn; 722 723 /* Schedule the send tasklet. */ 724 qib_schedule_send(qp); 725 } 726 unlock: 727 spin_unlock_irqrestore(&qp->s_lock, flags); 728 done: 729 return; 730 } 731 732 /** 733 * reset_psn - reset the QP state to send starting from PSN 734 * @qp: the QP 735 * @psn: the packet sequence number to restart at 736 * 737 * This is called from qib_rc_rcv() to process an incoming RC ACK 738 * for the given QP. 739 * Called at interrupt level with the QP s_lock held. 740 */ 741 static void reset_psn(struct rvt_qp *qp, u32 psn) 742 { 743 u32 n = qp->s_acked; 744 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); 745 u32 opcode; 746 747 qp->s_cur = n; 748 749 /* 750 * If we are starting the request from the beginning, 751 * let the normal send code handle initialization. 752 */ 753 if (qib_cmp24(psn, wqe->psn) <= 0) { 754 qp->s_state = OP(SEND_LAST); 755 goto done; 756 } 757 758 /* Find the work request opcode corresponding to the given PSN. */ 759 opcode = wqe->wr.opcode; 760 for (;;) { 761 int diff; 762 763 if (++n == qp->s_size) 764 n = 0; 765 if (n == qp->s_tail) 766 break; 767 wqe = rvt_get_swqe_ptr(qp, n); 768 diff = qib_cmp24(psn, wqe->psn); 769 if (diff < 0) 770 break; 771 qp->s_cur = n; 772 /* 773 * If we are starting the request from the beginning, 774 * let the normal send code handle initialization. 775 */ 776 if (diff == 0) { 777 qp->s_state = OP(SEND_LAST); 778 goto done; 779 } 780 opcode = wqe->wr.opcode; 781 } 782 783 /* 784 * Set the state to restart in the middle of a request. 785 * Don't change the s_sge, s_cur_sge, or s_cur_size. 786 * See qib_make_rc_req(). 787 */ 788 switch (opcode) { 789 case IB_WR_SEND: 790 case IB_WR_SEND_WITH_IMM: 791 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); 792 break; 793 794 case IB_WR_RDMA_WRITE: 795 case IB_WR_RDMA_WRITE_WITH_IMM: 796 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); 797 break; 798 799 case IB_WR_RDMA_READ: 800 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); 801 break; 802 803 default: 804 /* 805 * This case shouldn't happen since its only 806 * one PSN per req. 807 */ 808 qp->s_state = OP(SEND_LAST); 809 } 810 done: 811 qp->s_psn = psn; 812 /* 813 * Set RVT_S_WAIT_PSN as qib_rc_complete() may start the timer 814 * asynchronously before the send tasklet can get scheduled. 815 * Doing it in qib_make_rc_req() is too late. 816 */ 817 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && 818 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) 819 qp->s_flags |= RVT_S_WAIT_PSN; 820 } 821 822 /* 823 * Back up requester to resend the last un-ACKed request. 824 * The QP r_lock and s_lock should be held and interrupts disabled. 825 */ 826 void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait) 827 { 828 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 829 struct qib_ibport *ibp; 830 831 if (qp->s_retry == 0) { 832 if (qp->s_mig_state == IB_MIG_ARMED) { 833 qib_migrate_qp(qp); 834 qp->s_retry = qp->s_retry_cnt; 835 } else if (qp->s_last == qp->s_acked) { 836 rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); 837 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 838 return; 839 } else /* XXX need to handle delayed completion */ 840 return; 841 } else 842 qp->s_retry--; 843 844 ibp = to_iport(qp->ibqp.device, qp->port_num); 845 if (wqe->wr.opcode == IB_WR_RDMA_READ) 846 ibp->rvp.n_rc_resends++; 847 else 848 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; 849 850 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | 851 RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_PSN | 852 RVT_S_WAIT_ACK); 853 if (wait) 854 qp->s_flags |= RVT_S_SEND_ONE; 855 reset_psn(qp, psn); 856 } 857 858 /* 859 * Set qp->s_sending_psn to the next PSN after the given one. 860 * This would be psn+1 except when RDMA reads are present. 861 */ 862 static void reset_sending_psn(struct rvt_qp *qp, u32 psn) 863 { 864 struct rvt_swqe *wqe; 865 u32 n = qp->s_last; 866 867 /* Find the work request corresponding to the given PSN. */ 868 for (;;) { 869 wqe = rvt_get_swqe_ptr(qp, n); 870 if (qib_cmp24(psn, wqe->lpsn) <= 0) { 871 if (wqe->wr.opcode == IB_WR_RDMA_READ) 872 qp->s_sending_psn = wqe->lpsn + 1; 873 else 874 qp->s_sending_psn = psn + 1; 875 break; 876 } 877 if (++n == qp->s_size) 878 n = 0; 879 if (n == qp->s_tail) 880 break; 881 } 882 } 883 884 /* 885 * This should be called with the QP s_lock held and interrupts disabled. 886 */ 887 void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr) 888 { 889 struct ib_other_headers *ohdr; 890 struct rvt_swqe *wqe; 891 u32 opcode; 892 u32 psn; 893 894 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) 895 return; 896 897 /* Find out where the BTH is */ 898 if ((be16_to_cpu(hdr->lrh[0]) & 3) == QIB_LRH_BTH) 899 ohdr = &hdr->u.oth; 900 else 901 ohdr = &hdr->u.l.oth; 902 903 opcode = be32_to_cpu(ohdr->bth[0]) >> 24; 904 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 905 opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 906 WARN_ON(!qp->s_rdma_ack_cnt); 907 qp->s_rdma_ack_cnt--; 908 return; 909 } 910 911 psn = be32_to_cpu(ohdr->bth[2]); 912 reset_sending_psn(qp, psn); 913 914 /* 915 * Start timer after a packet requesting an ACK has been sent and 916 * there are still requests that haven't been acked. 917 */ 918 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && 919 !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) && 920 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) 921 rvt_add_retry_timer(qp); 922 923 while (qp->s_last != qp->s_acked) { 924 u32 s_last; 925 926 wqe = rvt_get_swqe_ptr(qp, qp->s_last); 927 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && 928 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) 929 break; 930 s_last = qp->s_last; 931 if (++s_last >= qp->s_size) 932 s_last = 0; 933 qp->s_last = s_last; 934 /* see post_send() */ 935 barrier(); 936 rvt_put_qp_swqe(qp, wqe); 937 rvt_qp_swqe_complete(qp, 938 wqe, 939 ib_qib_wc_opcode[wqe->wr.opcode], 940 IB_WC_SUCCESS); 941 } 942 /* 943 * If we were waiting for sends to complete before resending, 944 * and they are now complete, restart sending. 945 */ 946 if (qp->s_flags & RVT_S_WAIT_PSN && 947 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 948 qp->s_flags &= ~RVT_S_WAIT_PSN; 949 qp->s_sending_psn = qp->s_psn; 950 qp->s_sending_hpsn = qp->s_psn - 1; 951 qib_schedule_send(qp); 952 } 953 } 954 955 static inline void update_last_psn(struct rvt_qp *qp, u32 psn) 956 { 957 qp->s_last_psn = psn; 958 } 959 960 /* 961 * Generate a SWQE completion. 962 * This is similar to qib_send_complete but has to check to be sure 963 * that the SGEs are not being referenced if the SWQE is being resent. 964 */ 965 static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, 966 struct rvt_swqe *wqe, 967 struct qib_ibport *ibp) 968 { 969 /* 970 * Don't decrement refcount and don't generate a 971 * completion if the SWQE is being resent until the send 972 * is finished. 973 */ 974 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || 975 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 976 u32 s_last; 977 978 rvt_put_qp_swqe(qp, wqe); 979 s_last = qp->s_last; 980 if (++s_last >= qp->s_size) 981 s_last = 0; 982 qp->s_last = s_last; 983 /* see post_send() */ 984 barrier(); 985 rvt_qp_swqe_complete(qp, 986 wqe, 987 ib_qib_wc_opcode[wqe->wr.opcode], 988 IB_WC_SUCCESS); 989 } else 990 this_cpu_inc(*ibp->rvp.rc_delayed_comp); 991 992 qp->s_retry = qp->s_retry_cnt; 993 update_last_psn(qp, wqe->lpsn); 994 995 /* 996 * If we are completing a request which is in the process of 997 * being resent, we can stop resending it since we know the 998 * responder has already seen it. 999 */ 1000 if (qp->s_acked == qp->s_cur) { 1001 if (++qp->s_cur >= qp->s_size) 1002 qp->s_cur = 0; 1003 qp->s_acked = qp->s_cur; 1004 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); 1005 if (qp->s_acked != qp->s_tail) { 1006 qp->s_state = OP(SEND_LAST); 1007 qp->s_psn = wqe->psn; 1008 } 1009 } else { 1010 if (++qp->s_acked >= qp->s_size) 1011 qp->s_acked = 0; 1012 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) 1013 qp->s_draining = 0; 1014 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 1015 } 1016 return wqe; 1017 } 1018 1019 /** 1020 * do_rc_ack - process an incoming RC ACK 1021 * @qp: the QP the ACK came in on 1022 * @psn: the packet sequence number of the ACK 1023 * @opcode: the opcode of the request that resulted in the ACK 1024 * 1025 * This is called from qib_rc_rcv_resp() to process an incoming RC ACK 1026 * for the given QP. 1027 * Called at interrupt level with the QP s_lock held. 1028 * Returns 1 if OK, 0 if current operation should be aborted (NAK). 1029 */ 1030 static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, 1031 u64 val, struct qib_ctxtdata *rcd) 1032 { 1033 struct qib_ibport *ibp; 1034 enum ib_wc_status status; 1035 struct rvt_swqe *wqe; 1036 int ret = 0; 1037 u32 ack_psn; 1038 int diff; 1039 1040 /* 1041 * Note that NAKs implicitly ACK outstanding SEND and RDMA write 1042 * requests and implicitly NAK RDMA read and atomic requests issued 1043 * before the NAK'ed request. The MSN won't include the NAK'ed 1044 * request but will include an ACK'ed request(s). 1045 */ 1046 ack_psn = psn; 1047 if (aeth >> IB_AETH_NAK_SHIFT) 1048 ack_psn--; 1049 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 1050 ibp = to_iport(qp->ibqp.device, qp->port_num); 1051 1052 /* 1053 * The MSN might be for a later WQE than the PSN indicates so 1054 * only complete WQEs that the PSN finishes. 1055 */ 1056 while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) { 1057 /* 1058 * RDMA_READ_RESPONSE_ONLY is a special case since 1059 * we want to generate completion events for everything 1060 * before the RDMA read, copy the data, then generate 1061 * the completion for the read. 1062 */ 1063 if (wqe->wr.opcode == IB_WR_RDMA_READ && 1064 opcode == OP(RDMA_READ_RESPONSE_ONLY) && 1065 diff == 0) { 1066 ret = 1; 1067 goto bail; 1068 } 1069 /* 1070 * If this request is a RDMA read or atomic, and the ACK is 1071 * for a later operation, this ACK NAKs the RDMA read or 1072 * atomic. In other words, only a RDMA_READ_LAST or ONLY 1073 * can ACK a RDMA read and likewise for atomic ops. Note 1074 * that the NAK case can only happen if relaxed ordering is 1075 * used and requests are sent after an RDMA read or atomic 1076 * is sent but before the response is received. 1077 */ 1078 if ((wqe->wr.opcode == IB_WR_RDMA_READ && 1079 (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) || 1080 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1081 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && 1082 (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) { 1083 /* Retry this request. */ 1084 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { 1085 qp->r_flags |= RVT_R_RDMAR_SEQ; 1086 qib_restart_rc(qp, qp->s_last_psn + 1, 0); 1087 if (list_empty(&qp->rspwait)) { 1088 qp->r_flags |= RVT_R_RSP_SEND; 1089 rvt_get_qp(qp); 1090 list_add_tail(&qp->rspwait, 1091 &rcd->qp_wait_list); 1092 } 1093 } 1094 /* 1095 * No need to process the ACK/NAK since we are 1096 * restarting an earlier request. 1097 */ 1098 goto bail; 1099 } 1100 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1101 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 1102 u64 *vaddr = wqe->sg_list[0].vaddr; 1103 *vaddr = val; 1104 } 1105 if (qp->s_num_rd_atomic && 1106 (wqe->wr.opcode == IB_WR_RDMA_READ || 1107 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1108 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { 1109 qp->s_num_rd_atomic--; 1110 /* Restart sending task if fence is complete */ 1111 if ((qp->s_flags & RVT_S_WAIT_FENCE) && 1112 !qp->s_num_rd_atomic) { 1113 qp->s_flags &= ~(RVT_S_WAIT_FENCE | 1114 RVT_S_WAIT_ACK); 1115 qib_schedule_send(qp); 1116 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { 1117 qp->s_flags &= ~(RVT_S_WAIT_RDMAR | 1118 RVT_S_WAIT_ACK); 1119 qib_schedule_send(qp); 1120 } 1121 } 1122 wqe = do_rc_completion(qp, wqe, ibp); 1123 if (qp->s_acked == qp->s_tail) 1124 break; 1125 } 1126 1127 switch (aeth >> IB_AETH_NAK_SHIFT) { 1128 case 0: /* ACK */ 1129 this_cpu_inc(*ibp->rvp.rc_acks); 1130 if (qp->s_acked != qp->s_tail) { 1131 /* 1132 * We are expecting more ACKs so 1133 * reset the retransmit timer. 1134 */ 1135 rvt_mod_retry_timer(qp); 1136 /* 1137 * We can stop resending the earlier packets and 1138 * continue with the next packet the receiver wants. 1139 */ 1140 if (qib_cmp24(qp->s_psn, psn) <= 0) 1141 reset_psn(qp, psn + 1); 1142 } else { 1143 /* No more acks - kill all timers */ 1144 rvt_stop_rc_timers(qp); 1145 if (qib_cmp24(qp->s_psn, psn) <= 0) { 1146 qp->s_state = OP(SEND_LAST); 1147 qp->s_psn = psn + 1; 1148 } 1149 } 1150 if (qp->s_flags & RVT_S_WAIT_ACK) { 1151 qp->s_flags &= ~RVT_S_WAIT_ACK; 1152 qib_schedule_send(qp); 1153 } 1154 rvt_get_credit(qp, aeth); 1155 qp->s_rnr_retry = qp->s_rnr_retry_cnt; 1156 qp->s_retry = qp->s_retry_cnt; 1157 update_last_psn(qp, psn); 1158 return 1; 1159 1160 case 1: /* RNR NAK */ 1161 ibp->rvp.n_rnr_naks++; 1162 if (qp->s_acked == qp->s_tail) 1163 goto bail; 1164 if (qp->s_flags & RVT_S_WAIT_RNR) 1165 goto bail; 1166 if (qp->s_rnr_retry == 0) { 1167 status = IB_WC_RNR_RETRY_EXC_ERR; 1168 goto class_b; 1169 } 1170 if (qp->s_rnr_retry_cnt < 7) 1171 qp->s_rnr_retry--; 1172 1173 /* The last valid PSN is the previous PSN. */ 1174 update_last_psn(qp, psn - 1); 1175 1176 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; 1177 1178 reset_psn(qp, psn); 1179 1180 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); 1181 rvt_stop_rc_timers(qp); 1182 rvt_add_rnr_timer(qp, aeth); 1183 return 0; 1184 1185 case 3: /* NAK */ 1186 if (qp->s_acked == qp->s_tail) 1187 goto bail; 1188 /* The last valid PSN is the previous PSN. */ 1189 update_last_psn(qp, psn - 1); 1190 switch ((aeth >> IB_AETH_CREDIT_SHIFT) & 1191 IB_AETH_CREDIT_MASK) { 1192 case 0: /* PSN sequence error */ 1193 ibp->rvp.n_seq_naks++; 1194 /* 1195 * Back up to the responder's expected PSN. 1196 * Note that we might get a NAK in the middle of an 1197 * RDMA READ response which terminates the RDMA 1198 * READ. 1199 */ 1200 qib_restart_rc(qp, psn, 0); 1201 qib_schedule_send(qp); 1202 break; 1203 1204 case 1: /* Invalid Request */ 1205 status = IB_WC_REM_INV_REQ_ERR; 1206 ibp->rvp.n_other_naks++; 1207 goto class_b; 1208 1209 case 2: /* Remote Access Error */ 1210 status = IB_WC_REM_ACCESS_ERR; 1211 ibp->rvp.n_other_naks++; 1212 goto class_b; 1213 1214 case 3: /* Remote Operation Error */ 1215 status = IB_WC_REM_OP_ERR; 1216 ibp->rvp.n_other_naks++; 1217 class_b: 1218 if (qp->s_last == qp->s_acked) { 1219 rvt_send_complete(qp, wqe, status); 1220 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1221 } 1222 break; 1223 1224 default: 1225 /* Ignore other reserved NAK error codes */ 1226 goto reserved; 1227 } 1228 qp->s_retry = qp->s_retry_cnt; 1229 qp->s_rnr_retry = qp->s_rnr_retry_cnt; 1230 goto bail; 1231 1232 default: /* 2: reserved */ 1233 reserved: 1234 /* Ignore reserved NAK codes. */ 1235 goto bail; 1236 } 1237 1238 bail: 1239 rvt_stop_rc_timers(qp); 1240 return ret; 1241 } 1242 1243 /* 1244 * We have seen an out of sequence RDMA read middle or last packet. 1245 * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE. 1246 */ 1247 static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn, 1248 struct qib_ctxtdata *rcd) 1249 { 1250 struct rvt_swqe *wqe; 1251 1252 /* Remove QP from retry timer */ 1253 rvt_stop_rc_timers(qp); 1254 1255 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 1256 1257 while (qib_cmp24(psn, wqe->lpsn) > 0) { 1258 if (wqe->wr.opcode == IB_WR_RDMA_READ || 1259 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1260 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 1261 break; 1262 wqe = do_rc_completion(qp, wqe, ibp); 1263 } 1264 1265 ibp->rvp.n_rdma_seq++; 1266 qp->r_flags |= RVT_R_RDMAR_SEQ; 1267 qib_restart_rc(qp, qp->s_last_psn + 1, 0); 1268 if (list_empty(&qp->rspwait)) { 1269 qp->r_flags |= RVT_R_RSP_SEND; 1270 rvt_get_qp(qp); 1271 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 1272 } 1273 } 1274 1275 /** 1276 * qib_rc_rcv_resp - process an incoming RC response packet 1277 * @ibp: the port this packet came in on 1278 * @ohdr: the other headers for this packet 1279 * @data: the packet data 1280 * @tlen: the packet length 1281 * @qp: the QP for this packet 1282 * @opcode: the opcode for this packet 1283 * @psn: the packet sequence number for this packet 1284 * @hdrsize: the header length 1285 * @pmtu: the path MTU 1286 * 1287 * This is called from qib_rc_rcv() to process an incoming RC response 1288 * packet for the given QP. 1289 * Called at interrupt level. 1290 */ 1291 static void qib_rc_rcv_resp(struct qib_ibport *ibp, 1292 struct ib_other_headers *ohdr, 1293 void *data, u32 tlen, 1294 struct rvt_qp *qp, 1295 u32 opcode, 1296 u32 psn, u32 hdrsize, u32 pmtu, 1297 struct qib_ctxtdata *rcd) 1298 { 1299 struct rvt_swqe *wqe; 1300 struct qib_pportdata *ppd = ppd_from_ibp(ibp); 1301 enum ib_wc_status status; 1302 unsigned long flags; 1303 int diff; 1304 u32 pad; 1305 u32 aeth; 1306 u64 val; 1307 1308 if (opcode != OP(RDMA_READ_RESPONSE_MIDDLE)) { 1309 /* 1310 * If ACK'd PSN on SDMA busy list try to make progress to 1311 * reclaim SDMA credits. 1312 */ 1313 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && 1314 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { 1315 1316 /* 1317 * If send tasklet not running attempt to progress 1318 * SDMA queue. 1319 */ 1320 if (!(qp->s_flags & RVT_S_BUSY)) { 1321 /* Acquire SDMA Lock */ 1322 spin_lock_irqsave(&ppd->sdma_lock, flags); 1323 /* Invoke sdma make progress */ 1324 qib_sdma_make_progress(ppd); 1325 /* Release SDMA Lock */ 1326 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 1327 } 1328 } 1329 } 1330 1331 spin_lock_irqsave(&qp->s_lock, flags); 1332 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) 1333 goto ack_done; 1334 1335 /* Ignore invalid responses. */ 1336 if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0) 1337 goto ack_done; 1338 1339 /* Ignore duplicate responses. */ 1340 diff = qib_cmp24(psn, qp->s_last_psn); 1341 if (unlikely(diff <= 0)) { 1342 /* Update credits for "ghost" ACKs */ 1343 if (diff == 0 && opcode == OP(ACKNOWLEDGE)) { 1344 aeth = be32_to_cpu(ohdr->u.aeth); 1345 if ((aeth >> IB_AETH_NAK_SHIFT) == 0) 1346 rvt_get_credit(qp, aeth); 1347 } 1348 goto ack_done; 1349 } 1350 1351 /* 1352 * Skip everything other than the PSN we expect, if we are waiting 1353 * for a reply to a restarted RDMA read or atomic op. 1354 */ 1355 if (qp->r_flags & RVT_R_RDMAR_SEQ) { 1356 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) 1357 goto ack_done; 1358 qp->r_flags &= ~RVT_R_RDMAR_SEQ; 1359 } 1360 1361 if (unlikely(qp->s_acked == qp->s_tail)) 1362 goto ack_done; 1363 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 1364 status = IB_WC_SUCCESS; 1365 1366 switch (opcode) { 1367 case OP(ACKNOWLEDGE): 1368 case OP(ATOMIC_ACKNOWLEDGE): 1369 case OP(RDMA_READ_RESPONSE_FIRST): 1370 aeth = be32_to_cpu(ohdr->u.aeth); 1371 if (opcode == OP(ATOMIC_ACKNOWLEDGE)) 1372 val = ib_u64_get(&ohdr->u.at.atomic_ack_eth); 1373 else 1374 val = 0; 1375 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || 1376 opcode != OP(RDMA_READ_RESPONSE_FIRST)) 1377 goto ack_done; 1378 hdrsize += 4; 1379 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 1380 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1381 goto ack_op_err; 1382 /* 1383 * If this is a response to a resent RDMA read, we 1384 * have to be careful to copy the data to the right 1385 * location. 1386 */ 1387 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 1388 wqe, psn, pmtu); 1389 goto read_middle; 1390 1391 case OP(RDMA_READ_RESPONSE_MIDDLE): 1392 /* no AETH, no ACK */ 1393 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) 1394 goto ack_seq_err; 1395 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1396 goto ack_op_err; 1397 read_middle: 1398 if (unlikely(tlen != (hdrsize + pmtu + 4))) 1399 goto ack_len_err; 1400 if (unlikely(pmtu >= qp->s_rdma_read_len)) 1401 goto ack_len_err; 1402 1403 /* 1404 * We got a response so update the timeout. 1405 * 4.096 usec. * (1 << qp->timeout) 1406 */ 1407 rvt_mod_retry_timer(qp); 1408 if (qp->s_flags & RVT_S_WAIT_ACK) { 1409 qp->s_flags &= ~RVT_S_WAIT_ACK; 1410 qib_schedule_send(qp); 1411 } 1412 1413 if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE)) 1414 qp->s_retry = qp->s_retry_cnt; 1415 1416 /* 1417 * Update the RDMA receive state but do the copy w/o 1418 * holding the locks and blocking interrupts. 1419 */ 1420 qp->s_rdma_read_len -= pmtu; 1421 update_last_psn(qp, psn); 1422 spin_unlock_irqrestore(&qp->s_lock, flags); 1423 rvt_copy_sge(qp, &qp->s_rdma_read_sge, 1424 data, pmtu, false, false); 1425 goto bail; 1426 1427 case OP(RDMA_READ_RESPONSE_ONLY): 1428 aeth = be32_to_cpu(ohdr->u.aeth); 1429 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) 1430 goto ack_done; 1431 /* Get the number of bytes the message was padded by. */ 1432 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1433 /* 1434 * Check that the data size is >= 0 && <= pmtu. 1435 * Remember to account for the AETH header (4) and 1436 * ICRC (4). 1437 */ 1438 if (unlikely(tlen < (hdrsize + pad + 8))) 1439 goto ack_len_err; 1440 /* 1441 * If this is a response to a resent RDMA read, we 1442 * have to be careful to copy the data to the right 1443 * location. 1444 */ 1445 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 1446 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 1447 wqe, psn, pmtu); 1448 goto read_last; 1449 1450 case OP(RDMA_READ_RESPONSE_LAST): 1451 /* ACKs READ req. */ 1452 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) 1453 goto ack_seq_err; 1454 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 1455 goto ack_op_err; 1456 /* Get the number of bytes the message was padded by. */ 1457 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1458 /* 1459 * Check that the data size is >= 1 && <= pmtu. 1460 * Remember to account for the AETH header (4) and 1461 * ICRC (4). 1462 */ 1463 if (unlikely(tlen <= (hdrsize + pad + 8))) 1464 goto ack_len_err; 1465 read_last: 1466 tlen -= hdrsize + pad + 8; 1467 if (unlikely(tlen != qp->s_rdma_read_len)) 1468 goto ack_len_err; 1469 aeth = be32_to_cpu(ohdr->u.aeth); 1470 rvt_copy_sge(qp, &qp->s_rdma_read_sge, 1471 data, tlen, false, false); 1472 WARN_ON(qp->s_rdma_read_sge.num_sge); 1473 (void) do_rc_ack(qp, aeth, psn, 1474 OP(RDMA_READ_RESPONSE_LAST), 0, rcd); 1475 goto ack_done; 1476 } 1477 1478 ack_op_err: 1479 status = IB_WC_LOC_QP_OP_ERR; 1480 goto ack_err; 1481 1482 ack_seq_err: 1483 rdma_seq_err(qp, ibp, psn, rcd); 1484 goto ack_done; 1485 1486 ack_len_err: 1487 status = IB_WC_LOC_LEN_ERR; 1488 ack_err: 1489 if (qp->s_last == qp->s_acked) { 1490 rvt_send_complete(qp, wqe, status); 1491 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 1492 } 1493 ack_done: 1494 spin_unlock_irqrestore(&qp->s_lock, flags); 1495 bail: 1496 return; 1497 } 1498 1499 /** 1500 * qib_rc_rcv_error - process an incoming duplicate or error RC packet 1501 * @ohdr: the other headers for this packet 1502 * @data: the packet data 1503 * @qp: the QP for this packet 1504 * @opcode: the opcode for this packet 1505 * @psn: the packet sequence number for this packet 1506 * @diff: the difference between the PSN and the expected PSN 1507 * 1508 * This is called from qib_rc_rcv() to process an unexpected 1509 * incoming RC packet for the given QP. 1510 * Called at interrupt level. 1511 * Return 1 if no more processing is needed; otherwise return 0 to 1512 * schedule a response to be sent. 1513 */ 1514 static int qib_rc_rcv_error(struct ib_other_headers *ohdr, 1515 void *data, 1516 struct rvt_qp *qp, 1517 u32 opcode, 1518 u32 psn, 1519 int diff, 1520 struct qib_ctxtdata *rcd) 1521 { 1522 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 1523 struct rvt_ack_entry *e; 1524 unsigned long flags; 1525 u8 i, prev; 1526 int old_req; 1527 1528 if (diff > 0) { 1529 /* 1530 * Packet sequence error. 1531 * A NAK will ACK earlier sends and RDMA writes. 1532 * Don't queue the NAK if we already sent one. 1533 */ 1534 if (!qp->r_nak_state) { 1535 ibp->rvp.n_rc_seqnak++; 1536 qp->r_nak_state = IB_NAK_PSN_ERROR; 1537 /* Use the expected PSN. */ 1538 qp->r_ack_psn = qp->r_psn; 1539 /* 1540 * Wait to send the sequence NAK until all packets 1541 * in the receive queue have been processed. 1542 * Otherwise, we end up propagating congestion. 1543 */ 1544 if (list_empty(&qp->rspwait)) { 1545 qp->r_flags |= RVT_R_RSP_NAK; 1546 rvt_get_qp(qp); 1547 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 1548 } 1549 } 1550 goto done; 1551 } 1552 1553 /* 1554 * Handle a duplicate request. Don't re-execute SEND, RDMA 1555 * write or atomic op. Don't NAK errors, just silently drop 1556 * the duplicate request. Note that r_sge, r_len, and 1557 * r_rcv_len may be in use so don't modify them. 1558 * 1559 * We are supposed to ACK the earliest duplicate PSN but we 1560 * can coalesce an outstanding duplicate ACK. We have to 1561 * send the earliest so that RDMA reads can be restarted at 1562 * the requester's expected PSN. 1563 * 1564 * First, find where this duplicate PSN falls within the 1565 * ACKs previously sent. 1566 * old_req is true if there is an older response that is scheduled 1567 * to be sent before sending this one. 1568 */ 1569 e = NULL; 1570 old_req = 1; 1571 ibp->rvp.n_rc_dupreq++; 1572 1573 spin_lock_irqsave(&qp->s_lock, flags); 1574 1575 for (i = qp->r_head_ack_queue; ; i = prev) { 1576 if (i == qp->s_tail_ack_queue) 1577 old_req = 0; 1578 if (i) 1579 prev = i - 1; 1580 else 1581 prev = QIB_MAX_RDMA_ATOMIC; 1582 if (prev == qp->r_head_ack_queue) { 1583 e = NULL; 1584 break; 1585 } 1586 e = &qp->s_ack_queue[prev]; 1587 if (!e->opcode) { 1588 e = NULL; 1589 break; 1590 } 1591 if (qib_cmp24(psn, e->psn) >= 0) { 1592 if (prev == qp->s_tail_ack_queue && 1593 qib_cmp24(psn, e->lpsn) <= 0) 1594 old_req = 0; 1595 break; 1596 } 1597 } 1598 switch (opcode) { 1599 case OP(RDMA_READ_REQUEST): { 1600 struct ib_reth *reth; 1601 u32 offset; 1602 u32 len; 1603 1604 /* 1605 * If we didn't find the RDMA read request in the ack queue, 1606 * we can ignore this request. 1607 */ 1608 if (!e || e->opcode != OP(RDMA_READ_REQUEST)) 1609 goto unlock_done; 1610 /* RETH comes after BTH */ 1611 reth = &ohdr->u.rc.reth; 1612 /* 1613 * Address range must be a subset of the original 1614 * request and start on pmtu boundaries. 1615 * We reuse the old ack_queue slot since the requester 1616 * should not back up and request an earlier PSN for the 1617 * same request. 1618 */ 1619 offset = ((psn - e->psn) & QIB_PSN_MASK) * 1620 qp->pmtu; 1621 len = be32_to_cpu(reth->length); 1622 if (unlikely(offset + len != e->rdma_sge.sge_length)) 1623 goto unlock_done; 1624 if (e->rdma_sge.mr) { 1625 rvt_put_mr(e->rdma_sge.mr); 1626 e->rdma_sge.mr = NULL; 1627 } 1628 if (len != 0) { 1629 u32 rkey = be32_to_cpu(reth->rkey); 1630 u64 vaddr = be64_to_cpu(reth->vaddr); 1631 int ok; 1632 1633 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, 1634 IB_ACCESS_REMOTE_READ); 1635 if (unlikely(!ok)) 1636 goto unlock_done; 1637 } else { 1638 e->rdma_sge.vaddr = NULL; 1639 e->rdma_sge.length = 0; 1640 e->rdma_sge.sge_length = 0; 1641 } 1642 e->psn = psn; 1643 if (old_req) 1644 goto unlock_done; 1645 qp->s_tail_ack_queue = prev; 1646 break; 1647 } 1648 1649 case OP(COMPARE_SWAP): 1650 case OP(FETCH_ADD): { 1651 /* 1652 * If we didn't find the atomic request in the ack queue 1653 * or the send tasklet is already backed up to send an 1654 * earlier entry, we can ignore this request. 1655 */ 1656 if (!e || e->opcode != (u8) opcode || old_req) 1657 goto unlock_done; 1658 qp->s_tail_ack_queue = prev; 1659 break; 1660 } 1661 1662 default: 1663 /* 1664 * Ignore this operation if it doesn't request an ACK 1665 * or an earlier RDMA read or atomic is going to be resent. 1666 */ 1667 if (!(psn & IB_BTH_REQ_ACK) || old_req) 1668 goto unlock_done; 1669 /* 1670 * Resend the most recent ACK if this request is 1671 * after all the previous RDMA reads and atomics. 1672 */ 1673 if (i == qp->r_head_ack_queue) { 1674 spin_unlock_irqrestore(&qp->s_lock, flags); 1675 qp->r_nak_state = 0; 1676 qp->r_ack_psn = qp->r_psn - 1; 1677 goto send_ack; 1678 } 1679 /* 1680 * Try to send a simple ACK to work around a Mellanox bug 1681 * which doesn't accept a RDMA read response or atomic 1682 * response as an ACK for earlier SENDs or RDMA writes. 1683 */ 1684 if (!(qp->s_flags & RVT_S_RESP_PENDING)) { 1685 spin_unlock_irqrestore(&qp->s_lock, flags); 1686 qp->r_nak_state = 0; 1687 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; 1688 goto send_ack; 1689 } 1690 /* 1691 * Resend the RDMA read or atomic op which 1692 * ACKs this duplicate request. 1693 */ 1694 qp->s_tail_ack_queue = i; 1695 break; 1696 } 1697 qp->s_ack_state = OP(ACKNOWLEDGE); 1698 qp->s_flags |= RVT_S_RESP_PENDING; 1699 qp->r_nak_state = 0; 1700 qib_schedule_send(qp); 1701 1702 unlock_done: 1703 spin_unlock_irqrestore(&qp->s_lock, flags); 1704 done: 1705 return 1; 1706 1707 send_ack: 1708 return 0; 1709 } 1710 1711 static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n) 1712 { 1713 unsigned next; 1714 1715 next = n + 1; 1716 if (next > QIB_MAX_RDMA_ATOMIC) 1717 next = 0; 1718 qp->s_tail_ack_queue = next; 1719 qp->s_ack_state = OP(ACKNOWLEDGE); 1720 } 1721 1722 /** 1723 * qib_rc_rcv - process an incoming RC packet 1724 * @rcd: the context pointer 1725 * @hdr: the header of this packet 1726 * @has_grh: true if the header has a GRH 1727 * @data: the packet data 1728 * @tlen: the packet length 1729 * @qp: the QP for this packet 1730 * 1731 * This is called from qib_qp_rcv() to process an incoming RC packet 1732 * for the given QP. 1733 * Called at interrupt level. 1734 */ 1735 void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr, 1736 int has_grh, void *data, u32 tlen, struct rvt_qp *qp) 1737 { 1738 struct qib_ibport *ibp = &rcd->ppd->ibport_data; 1739 struct ib_other_headers *ohdr; 1740 u32 opcode; 1741 u32 hdrsize; 1742 u32 psn; 1743 u32 pad; 1744 struct ib_wc wc; 1745 u32 pmtu = qp->pmtu; 1746 int diff; 1747 struct ib_reth *reth; 1748 unsigned long flags; 1749 int ret; 1750 1751 /* Check for GRH */ 1752 if (!has_grh) { 1753 ohdr = &hdr->u.oth; 1754 hdrsize = 8 + 12; /* LRH + BTH */ 1755 } else { 1756 ohdr = &hdr->u.l.oth; 1757 hdrsize = 8 + 40 + 12; /* LRH + GRH + BTH */ 1758 } 1759 1760 opcode = be32_to_cpu(ohdr->bth[0]); 1761 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) 1762 return; 1763 1764 psn = be32_to_cpu(ohdr->bth[2]); 1765 opcode >>= 24; 1766 1767 /* 1768 * Process responses (ACKs) before anything else. Note that the 1769 * packet sequence number will be for something in the send work 1770 * queue rather than the expected receive packet sequence number. 1771 * In other words, this QP is the requester. 1772 */ 1773 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 1774 opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 1775 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, 1776 hdrsize, pmtu, rcd); 1777 return; 1778 } 1779 1780 /* Compute 24 bits worth of difference. */ 1781 diff = qib_cmp24(psn, qp->r_psn); 1782 if (unlikely(diff)) { 1783 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) 1784 return; 1785 goto send_ack; 1786 } 1787 1788 /* Check for opcode sequence errors. */ 1789 switch (qp->r_state) { 1790 case OP(SEND_FIRST): 1791 case OP(SEND_MIDDLE): 1792 if (opcode == OP(SEND_MIDDLE) || 1793 opcode == OP(SEND_LAST) || 1794 opcode == OP(SEND_LAST_WITH_IMMEDIATE)) 1795 break; 1796 goto nack_inv; 1797 1798 case OP(RDMA_WRITE_FIRST): 1799 case OP(RDMA_WRITE_MIDDLE): 1800 if (opcode == OP(RDMA_WRITE_MIDDLE) || 1801 opcode == OP(RDMA_WRITE_LAST) || 1802 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 1803 break; 1804 goto nack_inv; 1805 1806 default: 1807 if (opcode == OP(SEND_MIDDLE) || 1808 opcode == OP(SEND_LAST) || 1809 opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 1810 opcode == OP(RDMA_WRITE_MIDDLE) || 1811 opcode == OP(RDMA_WRITE_LAST) || 1812 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 1813 goto nack_inv; 1814 /* 1815 * Note that it is up to the requester to not send a new 1816 * RDMA read or atomic operation before receiving an ACK 1817 * for the previous operation. 1818 */ 1819 break; 1820 } 1821 1822 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) 1823 rvt_comm_est(qp); 1824 1825 /* OK, process the packet. */ 1826 switch (opcode) { 1827 case OP(SEND_FIRST): 1828 ret = rvt_get_rwqe(qp, false); 1829 if (ret < 0) 1830 goto nack_op_err; 1831 if (!ret) 1832 goto rnr_nak; 1833 qp->r_rcv_len = 0; 1834 /* FALLTHROUGH */ 1835 case OP(SEND_MIDDLE): 1836 case OP(RDMA_WRITE_MIDDLE): 1837 send_middle: 1838 /* Check for invalid length PMTU or posted rwqe len. */ 1839 if (unlikely(tlen != (hdrsize + pmtu + 4))) 1840 goto nack_inv; 1841 qp->r_rcv_len += pmtu; 1842 if (unlikely(qp->r_rcv_len > qp->r_len)) 1843 goto nack_inv; 1844 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); 1845 break; 1846 1847 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 1848 /* consume RWQE */ 1849 ret = rvt_get_rwqe(qp, true); 1850 if (ret < 0) 1851 goto nack_op_err; 1852 if (!ret) 1853 goto rnr_nak; 1854 goto send_last_imm; 1855 1856 case OP(SEND_ONLY): 1857 case OP(SEND_ONLY_WITH_IMMEDIATE): 1858 ret = rvt_get_rwqe(qp, false); 1859 if (ret < 0) 1860 goto nack_op_err; 1861 if (!ret) 1862 goto rnr_nak; 1863 qp->r_rcv_len = 0; 1864 if (opcode == OP(SEND_ONLY)) 1865 goto no_immediate_data; 1866 /* fall through -- for SEND_ONLY_WITH_IMMEDIATE */ 1867 case OP(SEND_LAST_WITH_IMMEDIATE): 1868 send_last_imm: 1869 wc.ex.imm_data = ohdr->u.imm_data; 1870 hdrsize += 4; 1871 wc.wc_flags = IB_WC_WITH_IMM; 1872 goto send_last; 1873 case OP(SEND_LAST): 1874 case OP(RDMA_WRITE_LAST): 1875 no_immediate_data: 1876 wc.wc_flags = 0; 1877 wc.ex.imm_data = 0; 1878 send_last: 1879 /* Get the number of bytes the message was padded by. */ 1880 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; 1881 /* Check for invalid length. */ 1882 /* XXX LAST len should be >= 1 */ 1883 if (unlikely(tlen < (hdrsize + pad + 4))) 1884 goto nack_inv; 1885 /* Don't count the CRC. */ 1886 tlen -= (hdrsize + pad + 4); 1887 wc.byte_len = tlen + qp->r_rcv_len; 1888 if (unlikely(wc.byte_len > qp->r_len)) 1889 goto nack_inv; 1890 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); 1891 rvt_put_ss(&qp->r_sge); 1892 qp->r_msn++; 1893 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) 1894 break; 1895 wc.wr_id = qp->r_wr_id; 1896 wc.status = IB_WC_SUCCESS; 1897 if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) || 1898 opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 1899 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 1900 else 1901 wc.opcode = IB_WC_RECV; 1902 wc.qp = &qp->ibqp; 1903 wc.src_qp = qp->remote_qpn; 1904 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr); 1905 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); 1906 /* zero fields that are N/A */ 1907 wc.vendor_err = 0; 1908 wc.pkey_index = 0; 1909 wc.dlid_path_bits = 0; 1910 wc.port_num = 0; 1911 /* Signal completion event if the solicited bit is set. */ 1912 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, 1913 ib_bth_is_solicited(ohdr)); 1914 break; 1915 1916 case OP(RDMA_WRITE_FIRST): 1917 case OP(RDMA_WRITE_ONLY): 1918 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 1919 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 1920 goto nack_inv; 1921 /* consume RWQE */ 1922 reth = &ohdr->u.rc.reth; 1923 hdrsize += sizeof(*reth); 1924 qp->r_len = be32_to_cpu(reth->length); 1925 qp->r_rcv_len = 0; 1926 qp->r_sge.sg_list = NULL; 1927 if (qp->r_len != 0) { 1928 u32 rkey = be32_to_cpu(reth->rkey); 1929 u64 vaddr = be64_to_cpu(reth->vaddr); 1930 int ok; 1931 1932 /* Check rkey & NAK */ 1933 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, 1934 rkey, IB_ACCESS_REMOTE_WRITE); 1935 if (unlikely(!ok)) 1936 goto nack_acc; 1937 qp->r_sge.num_sge = 1; 1938 } else { 1939 qp->r_sge.num_sge = 0; 1940 qp->r_sge.sge.mr = NULL; 1941 qp->r_sge.sge.vaddr = NULL; 1942 qp->r_sge.sge.length = 0; 1943 qp->r_sge.sge.sge_length = 0; 1944 } 1945 if (opcode == OP(RDMA_WRITE_FIRST)) 1946 goto send_middle; 1947 else if (opcode == OP(RDMA_WRITE_ONLY)) 1948 goto no_immediate_data; 1949 ret = rvt_get_rwqe(qp, true); 1950 if (ret < 0) 1951 goto nack_op_err; 1952 if (!ret) { 1953 rvt_put_ss(&qp->r_sge); 1954 goto rnr_nak; 1955 } 1956 wc.ex.imm_data = ohdr->u.rc.imm_data; 1957 hdrsize += 4; 1958 wc.wc_flags = IB_WC_WITH_IMM; 1959 goto send_last; 1960 1961 case OP(RDMA_READ_REQUEST): { 1962 struct rvt_ack_entry *e; 1963 u32 len; 1964 u8 next; 1965 1966 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 1967 goto nack_inv; 1968 next = qp->r_head_ack_queue + 1; 1969 /* s_ack_queue is size QIB_MAX_RDMA_ATOMIC+1 so use > not >= */ 1970 if (next > QIB_MAX_RDMA_ATOMIC) 1971 next = 0; 1972 spin_lock_irqsave(&qp->s_lock, flags); 1973 if (unlikely(next == qp->s_tail_ack_queue)) { 1974 if (!qp->s_ack_queue[next].sent) 1975 goto nack_inv_unlck; 1976 qib_update_ack_queue(qp, next); 1977 } 1978 e = &qp->s_ack_queue[qp->r_head_ack_queue]; 1979 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { 1980 rvt_put_mr(e->rdma_sge.mr); 1981 e->rdma_sge.mr = NULL; 1982 } 1983 reth = &ohdr->u.rc.reth; 1984 len = be32_to_cpu(reth->length); 1985 if (len) { 1986 u32 rkey = be32_to_cpu(reth->rkey); 1987 u64 vaddr = be64_to_cpu(reth->vaddr); 1988 int ok; 1989 1990 /* Check rkey & NAK */ 1991 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, 1992 rkey, IB_ACCESS_REMOTE_READ); 1993 if (unlikely(!ok)) 1994 goto nack_acc_unlck; 1995 /* 1996 * Update the next expected PSN. We add 1 later 1997 * below, so only add the remainder here. 1998 */ 1999 qp->r_psn += rvt_div_mtu(qp, len - 1); 2000 } else { 2001 e->rdma_sge.mr = NULL; 2002 e->rdma_sge.vaddr = NULL; 2003 e->rdma_sge.length = 0; 2004 e->rdma_sge.sge_length = 0; 2005 } 2006 e->opcode = opcode; 2007 e->sent = 0; 2008 e->psn = psn; 2009 e->lpsn = qp->r_psn; 2010 /* 2011 * We need to increment the MSN here instead of when we 2012 * finish sending the result since a duplicate request would 2013 * increment it more than once. 2014 */ 2015 qp->r_msn++; 2016 qp->r_psn++; 2017 qp->r_state = opcode; 2018 qp->r_nak_state = 0; 2019 qp->r_head_ack_queue = next; 2020 2021 /* Schedule the send tasklet. */ 2022 qp->s_flags |= RVT_S_RESP_PENDING; 2023 qib_schedule_send(qp); 2024 2025 goto sunlock; 2026 } 2027 2028 case OP(COMPARE_SWAP): 2029 case OP(FETCH_ADD): { 2030 struct ib_atomic_eth *ateth; 2031 struct rvt_ack_entry *e; 2032 u64 vaddr; 2033 atomic64_t *maddr; 2034 u64 sdata; 2035 u32 rkey; 2036 u8 next; 2037 2038 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) 2039 goto nack_inv; 2040 next = qp->r_head_ack_queue + 1; 2041 if (next > QIB_MAX_RDMA_ATOMIC) 2042 next = 0; 2043 spin_lock_irqsave(&qp->s_lock, flags); 2044 if (unlikely(next == qp->s_tail_ack_queue)) { 2045 if (!qp->s_ack_queue[next].sent) 2046 goto nack_inv_unlck; 2047 qib_update_ack_queue(qp, next); 2048 } 2049 e = &qp->s_ack_queue[qp->r_head_ack_queue]; 2050 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { 2051 rvt_put_mr(e->rdma_sge.mr); 2052 e->rdma_sge.mr = NULL; 2053 } 2054 ateth = &ohdr->u.atomic_eth; 2055 vaddr = get_ib_ateth_vaddr(ateth); 2056 if (unlikely(vaddr & (sizeof(u64) - 1))) 2057 goto nack_inv_unlck; 2058 rkey = be32_to_cpu(ateth->rkey); 2059 /* Check rkey & NAK */ 2060 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), 2061 vaddr, rkey, 2062 IB_ACCESS_REMOTE_ATOMIC))) 2063 goto nack_acc_unlck; 2064 /* Perform atomic OP and save result. */ 2065 maddr = (atomic64_t *) qp->r_sge.sge.vaddr; 2066 sdata = get_ib_ateth_swap(ateth); 2067 e->atomic_data = (opcode == OP(FETCH_ADD)) ? 2068 (u64) atomic64_add_return(sdata, maddr) - sdata : 2069 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, 2070 get_ib_ateth_compare(ateth), 2071 sdata); 2072 rvt_put_mr(qp->r_sge.sge.mr); 2073 qp->r_sge.num_sge = 0; 2074 e->opcode = opcode; 2075 e->sent = 0; 2076 e->psn = psn; 2077 e->lpsn = psn; 2078 qp->r_msn++; 2079 qp->r_psn++; 2080 qp->r_state = opcode; 2081 qp->r_nak_state = 0; 2082 qp->r_head_ack_queue = next; 2083 2084 /* Schedule the send tasklet. */ 2085 qp->s_flags |= RVT_S_RESP_PENDING; 2086 qib_schedule_send(qp); 2087 2088 goto sunlock; 2089 } 2090 2091 default: 2092 /* NAK unknown opcodes. */ 2093 goto nack_inv; 2094 } 2095 qp->r_psn++; 2096 qp->r_state = opcode; 2097 qp->r_ack_psn = psn; 2098 qp->r_nak_state = 0; 2099 /* Send an ACK if requested or required. */ 2100 if (psn & (1 << 31)) 2101 goto send_ack; 2102 return; 2103 2104 rnr_nak: 2105 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; 2106 qp->r_ack_psn = qp->r_psn; 2107 /* Queue RNR NAK for later */ 2108 if (list_empty(&qp->rspwait)) { 2109 qp->r_flags |= RVT_R_RSP_NAK; 2110 rvt_get_qp(qp); 2111 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2112 } 2113 return; 2114 2115 nack_op_err: 2116 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 2117 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; 2118 qp->r_ack_psn = qp->r_psn; 2119 /* Queue NAK for later */ 2120 if (list_empty(&qp->rspwait)) { 2121 qp->r_flags |= RVT_R_RSP_NAK; 2122 rvt_get_qp(qp); 2123 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2124 } 2125 return; 2126 2127 nack_inv_unlck: 2128 spin_unlock_irqrestore(&qp->s_lock, flags); 2129 nack_inv: 2130 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 2131 qp->r_nak_state = IB_NAK_INVALID_REQUEST; 2132 qp->r_ack_psn = qp->r_psn; 2133 /* Queue NAK for later */ 2134 if (list_empty(&qp->rspwait)) { 2135 qp->r_flags |= RVT_R_RSP_NAK; 2136 rvt_get_qp(qp); 2137 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2138 } 2139 return; 2140 2141 nack_acc_unlck: 2142 spin_unlock_irqrestore(&qp->s_lock, flags); 2143 nack_acc: 2144 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); 2145 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; 2146 qp->r_ack_psn = qp->r_psn; 2147 send_ack: 2148 qib_send_rc_ack(qp); 2149 return; 2150 2151 sunlock: 2152 spin_unlock_irqrestore(&qp->s_lock, flags); 2153 } 2154