1 /* 2 * Copyright (c) 2009-2013, 2016 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_inet.h" 36 37 #ifdef TCP_OFFLOAD 38 #include <sys/types.h> 39 #include <sys/malloc.h> 40 #include <sys/socket.h> 41 #include <sys/socketvar.h> 42 #include <sys/sockio.h> 43 #include <sys/taskqueue.h> 44 #include <netinet/in.h> 45 #include <net/route.h> 46 47 #include <netinet/in_systm.h> 48 #include <netinet/in_pcb.h> 49 #include <netinet/ip.h> 50 #include <netinet/in_fib.h> 51 #include <netinet/ip_var.h> 52 #include <netinet/tcp_var.h> 53 #include <netinet/tcp.h> 54 #include <netinet/tcpip.h> 55 56 #include <netinet/toecore.h> 57 58 struct sge_iq; 59 struct rss_header; 60 struct cpl_set_tcb_rpl; 61 #include <linux/types.h> 62 #include "offload.h" 63 #include "tom/t4_tom.h" 64 65 #define TOEPCB(so) ((struct toepcb *)(so_sototcpcb((so))->t_toe)) 66 67 #include "iw_cxgbe.h" 68 #include <linux/module.h> 69 #include <linux/workqueue.h> 70 #include <linux/notifier.h> 71 #include <linux/inetdevice.h> 72 #include <linux/if_vlan.h> 73 #include <net/netevent.h> 74 75 static spinlock_t req_lock; 76 static TAILQ_HEAD(c4iw_ep_list, c4iw_ep_common) req_list; 77 static struct work_struct c4iw_task; 78 static struct workqueue_struct *c4iw_taskq; 79 static LIST_HEAD(err_cqe_list); 80 static spinlock_t err_cqe_lock; 81 82 static void process_req(struct work_struct *ctx); 83 static void start_ep_timer(struct c4iw_ep *ep); 84 static int stop_ep_timer(struct c4iw_ep *ep); 85 static int set_tcpinfo(struct c4iw_ep *ep); 86 static void process_timeout(struct c4iw_ep *ep); 87 static void process_err_cqes(void); 88 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc); 89 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 90 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); 91 static void *alloc_ep(int size, gfp_t flags); 92 static int find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 93 __be16 peer_port, u8 tos, struct nhop4_extended *pnh4); 94 static void close_socket(struct socket *so); 95 static int send_mpa_req(struct c4iw_ep *ep); 96 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen); 97 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen); 98 static void close_complete_upcall(struct c4iw_ep *ep, int status); 99 static int send_abort(struct c4iw_ep *ep); 100 static void peer_close_upcall(struct c4iw_ep *ep); 101 static void peer_abort_upcall(struct c4iw_ep *ep); 102 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 103 static int connect_request_upcall(struct c4iw_ep *ep); 104 static void established_upcall(struct c4iw_ep *ep); 105 static int process_mpa_reply(struct c4iw_ep *ep); 106 static int process_mpa_request(struct c4iw_ep *ep); 107 static void process_peer_close(struct c4iw_ep *ep); 108 static void process_conn_error(struct c4iw_ep *ep); 109 static void process_close_complete(struct c4iw_ep *ep); 110 static void ep_timeout(unsigned long arg); 111 static void init_iwarp_socket(struct socket *so, void *arg); 112 static void uninit_iwarp_socket(struct socket *so); 113 static void process_data(struct c4iw_ep *ep); 114 static void process_connected(struct c4iw_ep *ep); 115 static int c4iw_so_upcall(struct socket *so, void *arg, int waitflag); 116 static void process_socket_event(struct c4iw_ep *ep); 117 static void release_ep_resources(struct c4iw_ep *ep); 118 static int process_terminate(struct c4iw_ep *ep); 119 static int terminate(struct sge_iq *iq, const struct rss_header *rss, 120 struct mbuf *m); 121 static int add_ep_to_req_list(struct c4iw_ep *ep, int ep_events); 122 #define START_EP_TIMER(ep) \ 123 do { \ 124 CTR3(KTR_IW_CXGBE, "start_ep_timer (%s:%d) ep %p", \ 125 __func__, __LINE__, (ep)); \ 126 start_ep_timer(ep); \ 127 } while (0) 128 129 #define STOP_EP_TIMER(ep) \ 130 ({ \ 131 CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \ 132 __func__, __LINE__, (ep)); \ 133 stop_ep_timer(ep); \ 134 }) 135 136 #ifdef KTR 137 static char *states[] = { 138 "idle", 139 "listen", 140 "connecting", 141 "mpa_wait_req", 142 "mpa_req_sent", 143 "mpa_req_rcvd", 144 "mpa_rep_sent", 145 "fpdu_mode", 146 "aborting", 147 "closing", 148 "moribund", 149 "dead", 150 NULL, 151 }; 152 #endif 153 154 155 static void deref_cm_id(struct c4iw_ep_common *epc) 156 { 157 epc->cm_id->rem_ref(epc->cm_id); 158 epc->cm_id = NULL; 159 set_bit(CM_ID_DEREFED, &epc->history); 160 } 161 162 static void ref_cm_id(struct c4iw_ep_common *epc) 163 { 164 set_bit(CM_ID_REFED, &epc->history); 165 epc->cm_id->add_ref(epc->cm_id); 166 } 167 168 static void deref_qp(struct c4iw_ep *ep) 169 { 170 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 171 clear_bit(QP_REFERENCED, &ep->com.flags); 172 set_bit(QP_DEREFED, &ep->com.history); 173 } 174 175 static void ref_qp(struct c4iw_ep *ep) 176 { 177 set_bit(QP_REFERENCED, &ep->com.flags); 178 set_bit(QP_REFED, &ep->com.history); 179 c4iw_qp_add_ref(&ep->com.qp->ibqp); 180 } 181 182 static void process_timeout(struct c4iw_ep *ep) 183 { 184 struct c4iw_qp_attributes attrs; 185 int abort = 1; 186 187 mutex_lock(&ep->com.mutex); 188 CTR4(KTR_IW_CXGBE, "%s ep :%p, tid:%u, state %d", __func__, 189 ep, ep->hwtid, ep->com.state); 190 set_bit(TIMEDOUT, &ep->com.history); 191 switch (ep->com.state) { 192 case MPA_REQ_SENT: 193 connect_reply_upcall(ep, -ETIMEDOUT); 194 break; 195 case MPA_REQ_WAIT: 196 case MPA_REQ_RCVD: 197 case MPA_REP_SENT: 198 case FPDU_MODE: 199 break; 200 case CLOSING: 201 case MORIBUND: 202 if (ep->com.cm_id && ep->com.qp) { 203 attrs.next_state = C4IW_QP_STATE_ERROR; 204 c4iw_modify_qp(ep->com.dev, ep->com.qp, 205 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 206 } 207 close_complete_upcall(ep, -ETIMEDOUT); 208 break; 209 case ABORTING: 210 case DEAD: 211 /* 212 * These states are expected if the ep timed out at the same 213 * time as another thread was calling stop_ep_timer(). 214 * So we silently do nothing for these states. 215 */ 216 abort = 0; 217 break; 218 default: 219 CTR4(KTR_IW_CXGBE, "%s unexpected state ep %p tid %u state %u\n" 220 , __func__, ep, ep->hwtid, ep->com.state); 221 abort = 0; 222 } 223 mutex_unlock(&ep->com.mutex); 224 if (abort) 225 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 226 c4iw_put_ep(&ep->com); 227 return; 228 } 229 230 struct cqe_list_entry { 231 struct list_head entry; 232 struct c4iw_dev *rhp; 233 struct t4_cqe err_cqe; 234 }; 235 236 static void 237 process_err_cqes(void) 238 { 239 unsigned long flag; 240 struct cqe_list_entry *cle; 241 242 spin_lock_irqsave(&err_cqe_lock, flag); 243 while (!list_empty(&err_cqe_list)) { 244 struct list_head *tmp; 245 tmp = err_cqe_list.next; 246 list_del(tmp); 247 tmp->next = tmp->prev = NULL; 248 spin_unlock_irqrestore(&err_cqe_lock, flag); 249 cle = list_entry(tmp, struct cqe_list_entry, entry); 250 c4iw_ev_dispatch(cle->rhp, &cle->err_cqe); 251 free(cle, M_CXGBE); 252 spin_lock_irqsave(&err_cqe_lock, flag); 253 } 254 spin_unlock_irqrestore(&err_cqe_lock, flag); 255 256 return; 257 } 258 259 static void 260 process_req(struct work_struct *ctx) 261 { 262 struct c4iw_ep_common *epc; 263 unsigned long flag; 264 int ep_events; 265 266 process_err_cqes(); 267 spin_lock_irqsave(&req_lock, flag); 268 while (!TAILQ_EMPTY(&req_list)) { 269 epc = TAILQ_FIRST(&req_list); 270 TAILQ_REMOVE(&req_list, epc, entry); 271 epc->entry.tqe_prev = NULL; 272 ep_events = epc->ep_events; 273 epc->ep_events = 0; 274 spin_unlock_irqrestore(&req_lock, flag); 275 CTR4(KTR_IW_CXGBE, "%s: so %p, ep %p, events 0x%x", __func__, 276 epc->so, epc, ep_events); 277 if (ep_events & C4IW_EVENT_TERM) 278 process_terminate((struct c4iw_ep *)epc); 279 if (ep_events & C4IW_EVENT_TIMEOUT) 280 process_timeout((struct c4iw_ep *)epc); 281 if (ep_events & C4IW_EVENT_SOCKET) 282 process_socket_event((struct c4iw_ep *)epc); 283 c4iw_put_ep(epc); 284 process_err_cqes(); 285 spin_lock_irqsave(&req_lock, flag); 286 } 287 spin_unlock_irqrestore(&req_lock, flag); 288 } 289 290 /* 291 * XXX: doesn't belong here in the iWARP driver. 292 * XXX: assumes that the connection was offloaded by cxgbe/t4_tom if TF_TOE is 293 * set. Is this a valid assumption for active open? 294 */ 295 static int 296 set_tcpinfo(struct c4iw_ep *ep) 297 { 298 struct socket *so = ep->com.so; 299 struct inpcb *inp = sotoinpcb(so); 300 struct tcpcb *tp; 301 struct toepcb *toep; 302 int rc = 0; 303 304 INP_WLOCK(inp); 305 tp = intotcpcb(inp); 306 if ((tp->t_flags & TF_TOE) == 0) { 307 rc = EINVAL; 308 log(LOG_ERR, "%s: connection not offloaded (so %p, ep %p)\n", 309 __func__, so, ep); 310 goto done; 311 } 312 toep = TOEPCB(so); 313 314 ep->hwtid = toep->tid; 315 ep->snd_seq = tp->snd_nxt; 316 ep->rcv_seq = tp->rcv_nxt; 317 ep->emss = max(tp->t_maxseg, 128); 318 done: 319 INP_WUNLOCK(inp); 320 return (rc); 321 322 } 323 324 static int 325 find_route(__be32 local_ip, __be32 peer_ip, __be16 local_port, 326 __be16 peer_port, u8 tos, struct nhop4_extended *pnh4) 327 { 328 struct in_addr addr; 329 int err; 330 331 CTR5(KTR_IW_CXGBE, "%s:frtB %x, %x, %d, %d", __func__, local_ip, 332 peer_ip, ntohs(local_port), ntohs(peer_port)); 333 334 addr.s_addr = peer_ip; 335 err = fib4_lookup_nh_ext(RT_DEFAULT_FIB, addr, NHR_REF, 0, pnh4); 336 337 CTR2(KTR_IW_CXGBE, "%s:frtE %d", __func__, err); 338 return err; 339 } 340 341 static void 342 close_socket(struct socket *so) 343 { 344 345 uninit_iwarp_socket(so); 346 sodisconnect(so); 347 } 348 349 static void 350 process_peer_close(struct c4iw_ep *ep) 351 { 352 struct c4iw_qp_attributes attrs; 353 int disconnect = 1; 354 int release = 0; 355 356 CTR4(KTR_IW_CXGBE, "%s:ppcB ep %p so %p state %s", __func__, ep, 357 ep->com.so, states[ep->com.state]); 358 359 mutex_lock(&ep->com.mutex); 360 switch (ep->com.state) { 361 362 case MPA_REQ_WAIT: 363 CTR2(KTR_IW_CXGBE, "%s:ppc1 %p MPA_REQ_WAIT CLOSING", 364 __func__, ep); 365 __state_set(&ep->com, CLOSING); 366 break; 367 368 case MPA_REQ_SENT: 369 CTR2(KTR_IW_CXGBE, "%s:ppc2 %p MPA_REQ_SENT CLOSING", 370 __func__, ep); 371 __state_set(&ep->com, DEAD); 372 connect_reply_upcall(ep, -ECONNABORTED); 373 374 disconnect = 0; 375 STOP_EP_TIMER(ep); 376 close_socket(ep->com.so); 377 deref_cm_id(&ep->com); 378 release = 1; 379 break; 380 381 case MPA_REQ_RCVD: 382 383 /* 384 * We're gonna mark this puppy DEAD, but keep 385 * the reference on it until the ULP accepts or 386 * rejects the CR. 387 */ 388 CTR2(KTR_IW_CXGBE, "%s:ppc3 %p MPA_REQ_RCVD CLOSING", 389 __func__, ep); 390 __state_set(&ep->com, CLOSING); 391 c4iw_get_ep(&ep->com); 392 break; 393 394 case MPA_REP_SENT: 395 CTR2(KTR_IW_CXGBE, "%s:ppc4 %p MPA_REP_SENT CLOSING", 396 __func__, ep); 397 __state_set(&ep->com, CLOSING); 398 break; 399 400 case FPDU_MODE: 401 CTR2(KTR_IW_CXGBE, "%s:ppc5 %p FPDU_MODE CLOSING", 402 __func__, ep); 403 START_EP_TIMER(ep); 404 __state_set(&ep->com, CLOSING); 405 attrs.next_state = C4IW_QP_STATE_CLOSING; 406 c4iw_modify_qp(ep->com.dev, ep->com.qp, 407 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 408 peer_close_upcall(ep); 409 break; 410 411 case ABORTING: 412 CTR2(KTR_IW_CXGBE, "%s:ppc6 %p ABORTING (disconn)", 413 __func__, ep); 414 disconnect = 0; 415 break; 416 417 case CLOSING: 418 CTR2(KTR_IW_CXGBE, "%s:ppc7 %p CLOSING MORIBUND", 419 __func__, ep); 420 __state_set(&ep->com, MORIBUND); 421 disconnect = 0; 422 break; 423 424 case MORIBUND: 425 CTR2(KTR_IW_CXGBE, "%s:ppc8 %p MORIBUND DEAD", __func__, 426 ep); 427 STOP_EP_TIMER(ep); 428 if (ep->com.cm_id && ep->com.qp) { 429 attrs.next_state = C4IW_QP_STATE_IDLE; 430 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 431 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 432 } 433 close_socket(ep->com.so); 434 close_complete_upcall(ep, 0); 435 __state_set(&ep->com, DEAD); 436 release = 1; 437 disconnect = 0; 438 break; 439 440 case DEAD: 441 CTR2(KTR_IW_CXGBE, "%s:ppc9 %p DEAD (disconn)", 442 __func__, ep); 443 disconnect = 0; 444 break; 445 446 default: 447 panic("%s: ep %p state %d", __func__, ep, 448 ep->com.state); 449 break; 450 } 451 452 mutex_unlock(&ep->com.mutex); 453 454 if (disconnect) { 455 456 CTR2(KTR_IW_CXGBE, "%s:ppca %p", __func__, ep); 457 c4iw_ep_disconnect(ep, 0, M_NOWAIT); 458 } 459 if (release) { 460 461 CTR2(KTR_IW_CXGBE, "%s:ppcb %p", __func__, ep); 462 c4iw_put_ep(&ep->com); 463 } 464 CTR2(KTR_IW_CXGBE, "%s:ppcE %p", __func__, ep); 465 return; 466 } 467 468 static void 469 process_conn_error(struct c4iw_ep *ep) 470 { 471 struct c4iw_qp_attributes attrs; 472 int ret; 473 int state; 474 475 mutex_lock(&ep->com.mutex); 476 state = ep->com.state; 477 CTR5(KTR_IW_CXGBE, "%s:pceB ep %p so %p so->so_error %u state %s", 478 __func__, ep, ep->com.so, ep->com.so->so_error, 479 states[ep->com.state]); 480 481 switch (state) { 482 483 case MPA_REQ_WAIT: 484 STOP_EP_TIMER(ep); 485 break; 486 487 case MPA_REQ_SENT: 488 STOP_EP_TIMER(ep); 489 connect_reply_upcall(ep, -ECONNRESET); 490 break; 491 492 case MPA_REP_SENT: 493 ep->com.rpl_err = ECONNRESET; 494 CTR1(KTR_IW_CXGBE, "waking up ep %p", ep); 495 break; 496 497 case MPA_REQ_RCVD: 498 499 /* 500 * We're gonna mark this puppy DEAD, but keep 501 * the reference on it until the ULP accepts or 502 * rejects the CR. 503 */ 504 c4iw_get_ep(&ep->com); 505 break; 506 507 case MORIBUND: 508 case CLOSING: 509 STOP_EP_TIMER(ep); 510 /*FALLTHROUGH*/ 511 case FPDU_MODE: 512 513 if (ep->com.cm_id && ep->com.qp) { 514 515 attrs.next_state = C4IW_QP_STATE_ERROR; 516 ret = c4iw_modify_qp(ep->com.qp->rhp, 517 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 518 &attrs, 1); 519 if (ret) 520 log(LOG_ERR, 521 "%s - qp <- error failed!\n", 522 __func__); 523 } 524 peer_abort_upcall(ep); 525 break; 526 527 case ABORTING: 528 break; 529 530 case DEAD: 531 CTR2(KTR_IW_CXGBE, "%s so_error %d IN DEAD STATE!!!!", 532 __func__, ep->com.so->so_error); 533 mutex_unlock(&ep->com.mutex); 534 return; 535 536 default: 537 panic("%s: ep %p state %d", __func__, ep, state); 538 break; 539 } 540 541 if (state != ABORTING) { 542 close_socket(ep->com.so); 543 __state_set(&ep->com, DEAD); 544 c4iw_put_ep(&ep->com); 545 } 546 mutex_unlock(&ep->com.mutex); 547 CTR2(KTR_IW_CXGBE, "%s:pceE %p", __func__, ep); 548 return; 549 } 550 551 static void 552 process_close_complete(struct c4iw_ep *ep) 553 { 554 struct c4iw_qp_attributes attrs; 555 int release = 0; 556 557 CTR4(KTR_IW_CXGBE, "%s:pccB ep %p so %p state %s", __func__, ep, 558 ep->com.so, states[ep->com.state]); 559 560 /* The cm_id may be null if we failed to connect */ 561 mutex_lock(&ep->com.mutex); 562 set_bit(CLOSE_CON_RPL, &ep->com.history); 563 564 switch (ep->com.state) { 565 566 case CLOSING: 567 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p CLOSING MORIBUND", 568 __func__, ep); 569 __state_set(&ep->com, MORIBUND); 570 break; 571 572 case MORIBUND: 573 CTR2(KTR_IW_CXGBE, "%s:pcc1 %p MORIBUND DEAD", __func__, 574 ep); 575 STOP_EP_TIMER(ep); 576 577 if ((ep->com.cm_id) && (ep->com.qp)) { 578 579 CTR2(KTR_IW_CXGBE, "%s:pcc2 %p QP_STATE_IDLE", 580 __func__, ep); 581 attrs.next_state = C4IW_QP_STATE_IDLE; 582 c4iw_modify_qp(ep->com.dev, 583 ep->com.qp, 584 C4IW_QP_ATTR_NEXT_STATE, 585 &attrs, 1); 586 } 587 588 close_socket(ep->com.so); 589 close_complete_upcall(ep, 0); 590 __state_set(&ep->com, DEAD); 591 release = 1; 592 break; 593 594 case ABORTING: 595 CTR2(KTR_IW_CXGBE, "%s:pcc5 %p ABORTING", __func__, ep); 596 break; 597 598 case DEAD: 599 CTR2(KTR_IW_CXGBE, "%s:pcc6 %p DEAD", __func__, ep); 600 break; 601 default: 602 CTR2(KTR_IW_CXGBE, "%s:pcc7 %p unknown ep state", 603 __func__, ep); 604 panic("%s:pcc6 %p unknown ep state", __func__, ep); 605 break; 606 } 607 mutex_unlock(&ep->com.mutex); 608 609 if (release) { 610 611 CTR2(KTR_IW_CXGBE, "%s:pcc8 %p", __func__, ep); 612 c4iw_put_ep(&ep->com); 613 } 614 CTR2(KTR_IW_CXGBE, "%s:pccE %p", __func__, ep); 615 return; 616 } 617 618 static void 619 init_iwarp_socket(struct socket *so, void *arg) 620 { 621 int rc; 622 struct sockopt sopt; 623 int on = 1; 624 625 /* Note that SOCK_LOCK(so) is same as SOCKBUF_LOCK(&so->so_rcv) */ 626 SOCK_LOCK(so); 627 soupcall_set(so, SO_RCV, c4iw_so_upcall, arg); 628 so->so_state |= SS_NBIO; 629 SOCK_UNLOCK(so); 630 sopt.sopt_dir = SOPT_SET; 631 sopt.sopt_level = IPPROTO_TCP; 632 sopt.sopt_name = TCP_NODELAY; 633 sopt.sopt_val = (caddr_t)&on; 634 sopt.sopt_valsize = sizeof on; 635 sopt.sopt_td = NULL; 636 rc = sosetopt(so, &sopt); 637 if (rc) { 638 log(LOG_ERR, "%s: can't set TCP_NODELAY on so %p (%d)\n", 639 __func__, so, rc); 640 } 641 } 642 643 static void 644 uninit_iwarp_socket(struct socket *so) 645 { 646 647 SOCKBUF_LOCK(&so->so_rcv); 648 soupcall_clear(so, SO_RCV); 649 SOCKBUF_UNLOCK(&so->so_rcv); 650 } 651 652 static void 653 process_data(struct c4iw_ep *ep) 654 { 655 struct sockaddr_in *local, *remote; 656 int disconnect = 0; 657 658 CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sbused %d", __func__, 659 ep->com.so, ep, states[ep->com.state], sbused(&ep->com.so->so_rcv)); 660 661 switch (state_read(&ep->com)) { 662 case MPA_REQ_SENT: 663 disconnect = process_mpa_reply(ep); 664 break; 665 case MPA_REQ_WAIT: 666 in_getsockaddr(ep->com.so, (struct sockaddr **)&local); 667 in_getpeeraddr(ep->com.so, (struct sockaddr **)&remote); 668 ep->com.local_addr = *local; 669 ep->com.remote_addr = *remote; 670 free(local, M_SONAME); 671 free(remote, M_SONAME); 672 disconnect = process_mpa_request(ep); 673 break; 674 default: 675 if (sbused(&ep->com.so->so_rcv)) 676 log(LOG_ERR, "%s: Unexpected streaming data. ep %p, " 677 "state %d, so %p, so_state 0x%x, sbused %u\n", 678 __func__, ep, state_read(&ep->com), ep->com.so, 679 ep->com.so->so_state, sbused(&ep->com.so->so_rcv)); 680 break; 681 } 682 if (disconnect) 683 c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); 684 685 } 686 687 static void 688 process_connected(struct c4iw_ep *ep) 689 { 690 struct socket *so = ep->com.so; 691 692 if ((so->so_state & SS_ISCONNECTED) && !so->so_error) { 693 if (send_mpa_req(ep)) 694 goto err; 695 } else { 696 connect_reply_upcall(ep, -so->so_error); 697 goto err; 698 } 699 return; 700 err: 701 close_socket(so); 702 state_set(&ep->com, DEAD); 703 c4iw_put_ep(&ep->com); 704 return; 705 } 706 707 void 708 process_newconn(struct iw_cm_id *parent_cm_id, struct socket *child_so) 709 { 710 struct c4iw_ep *child_ep; 711 struct sockaddr_in *local; 712 struct sockaddr_in *remote; 713 struct c4iw_ep *parent_ep = parent_cm_id->provider_data; 714 int ret = 0; 715 716 MPASS(child_so != NULL); 717 718 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 719 720 CTR5(KTR_IW_CXGBE, 721 "%s: parent so %p, parent ep %p, child so %p, child ep %p", 722 __func__, parent_ep->com.so, parent_ep, child_so, child_ep); 723 724 in_getsockaddr(child_so, (struct sockaddr **)&local); 725 in_getpeeraddr(child_so, (struct sockaddr **)&remote); 726 727 child_ep->com.local_addr = *local; 728 child_ep->com.remote_addr = *remote; 729 child_ep->com.dev = parent_ep->com.dev; 730 child_ep->com.so = child_so; 731 child_ep->com.cm_id = NULL; 732 child_ep->com.thread = parent_ep->com.thread; 733 child_ep->parent_ep = parent_ep; 734 735 free(local, M_SONAME); 736 free(remote, M_SONAME); 737 738 init_iwarp_socket(child_so, &child_ep->com); 739 c4iw_get_ep(&parent_ep->com); 740 init_timer(&child_ep->timer); 741 state_set(&child_ep->com, MPA_REQ_WAIT); 742 START_EP_TIMER(child_ep); 743 744 /* maybe the request has already been queued up on the socket... */ 745 ret = process_mpa_request(child_ep); 746 if (ret == 2) 747 /* ABORT */ 748 c4iw_ep_disconnect(child_ep, 1, GFP_KERNEL); 749 else if (ret == 1) 750 /* CLOSE */ 751 c4iw_ep_disconnect(child_ep, 0, GFP_KERNEL); 752 753 return; 754 } 755 756 static int 757 add_ep_to_req_list(struct c4iw_ep *ep, int new_ep_event) 758 { 759 unsigned long flag; 760 761 spin_lock_irqsave(&req_lock, flag); 762 if (ep && ep->com.so) { 763 ep->com.ep_events |= new_ep_event; 764 if (!ep->com.entry.tqe_prev) { 765 c4iw_get_ep(&ep->com); 766 TAILQ_INSERT_TAIL(&req_list, &ep->com, entry); 767 queue_work(c4iw_taskq, &c4iw_task); 768 } 769 } 770 spin_unlock_irqrestore(&req_lock, flag); 771 772 return (0); 773 } 774 775 static int 776 c4iw_so_upcall(struct socket *so, void *arg, int waitflag) 777 { 778 struct c4iw_ep *ep = arg; 779 780 CTR6(KTR_IW_CXGBE, 781 "%s: so %p, so_state 0x%x, ep %p, ep_state %s, tqe_prev %p", 782 __func__, so, so->so_state, ep, states[ep->com.state], 783 ep->com.entry.tqe_prev); 784 785 MPASS(ep->com.so == so); 786 add_ep_to_req_list(ep, C4IW_EVENT_SOCKET); 787 788 return (SU_OK); 789 } 790 791 792 static int 793 terminate(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 794 { 795 struct adapter *sc = iq->adapter; 796 const struct cpl_rdma_terminate *cpl = mtod(m, const void *); 797 unsigned int tid = GET_TID(cpl); 798 struct toepcb *toep = lookup_tid(sc, tid); 799 struct socket *so; 800 struct c4iw_ep *ep; 801 802 INP_WLOCK(toep->inp); 803 so = inp_inpcbtosocket(toep->inp); 804 ep = so->so_rcv.sb_upcallarg; 805 INP_WUNLOCK(toep->inp); 806 807 CTR3(KTR_IW_CXGBE, "%s: so %p, ep %p", __func__, so, ep); 808 add_ep_to_req_list(ep, C4IW_EVENT_TERM); 809 810 return 0; 811 } 812 813 static void 814 process_socket_event(struct c4iw_ep *ep) 815 { 816 int state = state_read(&ep->com); 817 struct socket *so = ep->com.so; 818 819 CTR6(KTR_IW_CXGBE, "process_socket_event: so %p, so_state 0x%x, " 820 "so_err %d, sb_state 0x%x, ep %p, ep_state %s", so, so->so_state, 821 so->so_error, so->so_rcv.sb_state, ep, states[state]); 822 823 if (state == CONNECTING) { 824 process_connected(ep); 825 return; 826 } 827 828 if (state == LISTEN) { 829 /* socket listening events are handled at IWCM */ 830 CTR3(KTR_IW_CXGBE, "%s Invalid ep state:%u, ep:%p", __func__, 831 ep->com.state, ep); 832 BUG(); 833 return; 834 } 835 836 /* connection error */ 837 if (so->so_error) { 838 process_conn_error(ep); 839 return; 840 } 841 842 /* peer close */ 843 if ((so->so_rcv.sb_state & SBS_CANTRCVMORE) && state <= CLOSING) { 844 process_peer_close(ep); 845 /* 846 * check whether socket disconnect event is pending before 847 * returning. Fallthrough if yes. 848 */ 849 if (!(so->so_state & SS_ISDISCONNECTED)) 850 return; 851 } 852 853 /* close complete */ 854 if (so->so_state & SS_ISDISCONNECTED) { 855 process_close_complete(ep); 856 return; 857 } 858 859 /* rx data */ 860 process_data(ep); 861 } 862 863 SYSCTL_NODE(_hw, OID_AUTO, iw_cxgbe, CTLFLAG_RD, 0, "iw_cxgbe driver parameters"); 864 865 static int dack_mode = 0; 866 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RWTUN, &dack_mode, 0, 867 "Delayed ack mode (default = 0)"); 868 869 int c4iw_max_read_depth = 8; 870 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_max_read_depth, CTLFLAG_RWTUN, &c4iw_max_read_depth, 0, 871 "Per-connection max ORD/IRD (default = 8)"); 872 873 static int enable_tcp_timestamps; 874 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_timestamps, CTLFLAG_RWTUN, &enable_tcp_timestamps, 0, 875 "Enable tcp timestamps (default = 0)"); 876 877 static int enable_tcp_sack; 878 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_sack, CTLFLAG_RWTUN, &enable_tcp_sack, 0, 879 "Enable tcp SACK (default = 0)"); 880 881 static int enable_tcp_window_scaling = 1; 882 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, enable_tcp_window_scaling, CTLFLAG_RWTUN, &enable_tcp_window_scaling, 0, 883 "Enable tcp window scaling (default = 1)"); 884 885 int c4iw_debug = 1; 886 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_debug, CTLFLAG_RWTUN, &c4iw_debug, 0, 887 "Enable debug logging (default = 0)"); 888 889 static int peer2peer = 1; 890 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RWTUN, &peer2peer, 0, 891 "Support peer2peer ULPs (default = 1)"); 892 893 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 894 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RWTUN, &p2p_type, 0, 895 "RDMAP opcode to use for the RTR message: 1 = RDMA_READ 0 = RDMA_WRITE (default 1)"); 896 897 static int ep_timeout_secs = 60; 898 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, ep_timeout_secs, CTLFLAG_RWTUN, &ep_timeout_secs, 0, 899 "CM Endpoint operation timeout in seconds (default = 60)"); 900 901 static int mpa_rev = 1; 902 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, mpa_rev, CTLFLAG_RWTUN, &mpa_rev, 0, 903 "MPA Revision, 0 supports amso1100, 1 is RFC5044 spec compliant, 2 is IETF MPA Peer Connect Draft compliant (default = 1)"); 904 905 static int markers_enabled; 906 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, markers_enabled, CTLFLAG_RWTUN, &markers_enabled, 0, 907 "Enable MPA MARKERS (default(0) = disabled)"); 908 909 static int crc_enabled = 1; 910 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, crc_enabled, CTLFLAG_RWTUN, &crc_enabled, 0, 911 "Enable MPA CRC (default(1) = enabled)"); 912 913 static int rcv_win = 256 * 1024; 914 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, rcv_win, CTLFLAG_RWTUN, &rcv_win, 0, 915 "TCP receive window in bytes (default = 256KB)"); 916 917 static int snd_win = 128 * 1024; 918 SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, snd_win, CTLFLAG_RWTUN, &snd_win, 0, 919 "TCP send window in bytes (default = 128KB)"); 920 921 static void 922 start_ep_timer(struct c4iw_ep *ep) 923 { 924 925 if (timer_pending(&ep->timer)) { 926 CTR2(KTR_IW_CXGBE, "%s: ep %p, already started", __func__, ep); 927 printk(KERN_ERR "%s timer already started! ep %p\n", __func__, 928 ep); 929 return; 930 } 931 clear_bit(TIMEOUT, &ep->com.flags); 932 c4iw_get_ep(&ep->com); 933 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 934 ep->timer.data = (unsigned long)ep; 935 ep->timer.function = ep_timeout; 936 add_timer(&ep->timer); 937 } 938 939 static int 940 stop_ep_timer(struct c4iw_ep *ep) 941 { 942 943 del_timer_sync(&ep->timer); 944 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 945 c4iw_put_ep(&ep->com); 946 return 0; 947 } 948 return 1; 949 } 950 951 static enum 952 c4iw_ep_state state_read(struct c4iw_ep_common *epc) 953 { 954 enum c4iw_ep_state state; 955 956 mutex_lock(&epc->mutex); 957 state = epc->state; 958 mutex_unlock(&epc->mutex); 959 960 return (state); 961 } 962 963 static void 964 __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 965 { 966 967 epc->state = new; 968 } 969 970 static void 971 state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 972 { 973 974 mutex_lock(&epc->mutex); 975 __state_set(epc, new); 976 mutex_unlock(&epc->mutex); 977 } 978 979 static void * 980 alloc_ep(int size, gfp_t gfp) 981 { 982 struct c4iw_ep_common *epc; 983 984 epc = kzalloc(size, gfp); 985 if (epc == NULL) 986 return (NULL); 987 988 kref_init(&epc->kref); 989 mutex_init(&epc->mutex); 990 c4iw_init_wr_wait(&epc->wr_wait); 991 992 return (epc); 993 } 994 995 void _c4iw_free_ep(struct kref *kref) 996 { 997 struct c4iw_ep *ep; 998 struct c4iw_ep_common *epc; 999 1000 ep = container_of(kref, struct c4iw_ep, com.kref); 1001 epc = &ep->com; 1002 KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list", 1003 __func__, epc)); 1004 if (test_bit(QP_REFERENCED, &ep->com.flags)) 1005 deref_qp(ep); 1006 kfree(ep); 1007 } 1008 1009 static void release_ep_resources(struct c4iw_ep *ep) 1010 { 1011 CTR2(KTR_IW_CXGBE, "%s:rerB %p", __func__, ep); 1012 set_bit(RELEASE_RESOURCES, &ep->com.flags); 1013 c4iw_put_ep(&ep->com); 1014 CTR2(KTR_IW_CXGBE, "%s:rerE %p", __func__, ep); 1015 } 1016 1017 static int 1018 send_mpa_req(struct c4iw_ep *ep) 1019 { 1020 int mpalen; 1021 struct mpa_message *mpa; 1022 struct mpa_v2_conn_params mpa_v2_params; 1023 struct mbuf *m; 1024 char mpa_rev_to_use = mpa_rev; 1025 int err = 0; 1026 1027 if (ep->retry_with_mpa_v1) 1028 mpa_rev_to_use = 1; 1029 mpalen = sizeof(*mpa) + ep->plen; 1030 if (mpa_rev_to_use == 2) 1031 mpalen += sizeof(struct mpa_v2_conn_params); 1032 1033 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1034 if (mpa == NULL) { 1035 err = -ENOMEM; 1036 CTR3(KTR_IW_CXGBE, "%s:smr1 ep: %p , error: %d", 1037 __func__, ep, err); 1038 goto err; 1039 } 1040 1041 memset(mpa, 0, mpalen); 1042 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 1043 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 1044 (markers_enabled ? MPA_MARKERS : 0) | 1045 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 1046 mpa->private_data_size = htons(ep->plen); 1047 mpa->revision = mpa_rev_to_use; 1048 1049 if (mpa_rev_to_use == 1) { 1050 ep->tried_with_mpa_v1 = 1; 1051 ep->retry_with_mpa_v1 = 0; 1052 } 1053 1054 if (mpa_rev_to_use == 2) { 1055 mpa->private_data_size += 1056 htons(sizeof(struct mpa_v2_conn_params)); 1057 mpa_v2_params.ird = htons((u16)ep->ird); 1058 mpa_v2_params.ord = htons((u16)ep->ord); 1059 1060 if (peer2peer) { 1061 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1062 1063 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1064 mpa_v2_params.ord |= 1065 htons(MPA_V2_RDMA_WRITE_RTR); 1066 } else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1067 mpa_v2_params.ord |= 1068 htons(MPA_V2_RDMA_READ_RTR); 1069 } 1070 } 1071 memcpy(mpa->private_data, &mpa_v2_params, 1072 sizeof(struct mpa_v2_conn_params)); 1073 1074 if (ep->plen) { 1075 1076 memcpy(mpa->private_data + 1077 sizeof(struct mpa_v2_conn_params), 1078 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1079 } 1080 } else { 1081 1082 if (ep->plen) 1083 memcpy(mpa->private_data, 1084 ep->mpa_pkt + sizeof(*mpa), ep->plen); 1085 CTR2(KTR_IW_CXGBE, "%s:smr7 %p", __func__, ep); 1086 } 1087 1088 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1089 if (m == NULL) { 1090 err = -ENOMEM; 1091 CTR3(KTR_IW_CXGBE, "%s:smr2 ep: %p , error: %d", 1092 __func__, ep, err); 1093 free(mpa, M_CXGBE); 1094 goto err; 1095 } 1096 m_copyback(m, 0, mpalen, (void *)mpa); 1097 free(mpa, M_CXGBE); 1098 1099 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1100 ep->com.thread); 1101 if (err) { 1102 CTR3(KTR_IW_CXGBE, "%s:smr3 ep: %p , error: %d", 1103 __func__, ep, err); 1104 goto err; 1105 } 1106 1107 START_EP_TIMER(ep); 1108 state_set(&ep->com, MPA_REQ_SENT); 1109 ep->mpa_attr.initiator = 1; 1110 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1111 return 0; 1112 err: 1113 connect_reply_upcall(ep, err); 1114 CTR3(KTR_IW_CXGBE, "%s:smrE %p, error: %d", __func__, ep, err); 1115 return err; 1116 } 1117 1118 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 1119 { 1120 int mpalen ; 1121 struct mpa_message *mpa; 1122 struct mpa_v2_conn_params mpa_v2_params; 1123 struct mbuf *m; 1124 int err; 1125 1126 CTR4(KTR_IW_CXGBE, "%s:smrejB %p %u %d", __func__, ep, ep->hwtid, 1127 ep->plen); 1128 1129 mpalen = sizeof(*mpa) + plen; 1130 1131 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1132 1133 mpalen += sizeof(struct mpa_v2_conn_params); 1134 CTR4(KTR_IW_CXGBE, "%s:smrej1 %p %u %d", __func__, ep, 1135 ep->mpa_attr.version, mpalen); 1136 } 1137 1138 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1139 if (mpa == NULL) 1140 return (-ENOMEM); 1141 1142 memset(mpa, 0, mpalen); 1143 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1144 mpa->flags = MPA_REJECT; 1145 mpa->revision = mpa_rev; 1146 mpa->private_data_size = htons(plen); 1147 1148 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1149 1150 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1151 mpa->private_data_size += 1152 htons(sizeof(struct mpa_v2_conn_params)); 1153 mpa_v2_params.ird = htons(((u16)ep->ird) | 1154 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 1155 0)); 1156 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 1157 (p2p_type == 1158 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 1159 MPA_V2_RDMA_WRITE_RTR : p2p_type == 1160 FW_RI_INIT_P2PTYPE_READ_REQ ? 1161 MPA_V2_RDMA_READ_RTR : 0) : 0)); 1162 memcpy(mpa->private_data, &mpa_v2_params, 1163 sizeof(struct mpa_v2_conn_params)); 1164 1165 if (ep->plen) 1166 memcpy(mpa->private_data + 1167 sizeof(struct mpa_v2_conn_params), pdata, plen); 1168 CTR5(KTR_IW_CXGBE, "%s:smrej3 %p %d %d %d", __func__, ep, 1169 mpa_v2_params.ird, mpa_v2_params.ord, ep->plen); 1170 } else 1171 if (plen) 1172 memcpy(mpa->private_data, pdata, plen); 1173 1174 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1175 if (m == NULL) { 1176 free(mpa, M_CXGBE); 1177 return (-ENOMEM); 1178 } 1179 m_copyback(m, 0, mpalen, (void *)mpa); 1180 free(mpa, M_CXGBE); 1181 1182 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, ep->com.thread); 1183 if (!err) 1184 ep->snd_seq += mpalen; 1185 CTR4(KTR_IW_CXGBE, "%s:smrejE %p %u %d", __func__, ep, ep->hwtid, err); 1186 return err; 1187 } 1188 1189 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 1190 { 1191 int mpalen; 1192 struct mpa_message *mpa; 1193 struct mbuf *m; 1194 struct mpa_v2_conn_params mpa_v2_params; 1195 int err; 1196 1197 CTR2(KTR_IW_CXGBE, "%s:smrepB %p", __func__, ep); 1198 1199 mpalen = sizeof(*mpa) + plen; 1200 1201 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1202 1203 CTR3(KTR_IW_CXGBE, "%s:smrep1 %p %d", __func__, ep, 1204 ep->mpa_attr.version); 1205 mpalen += sizeof(struct mpa_v2_conn_params); 1206 } 1207 1208 mpa = malloc(mpalen, M_CXGBE, M_NOWAIT); 1209 if (mpa == NULL) 1210 return (-ENOMEM); 1211 1212 memset(mpa, 0, sizeof(*mpa)); 1213 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 1214 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 1215 (markers_enabled ? MPA_MARKERS : 0); 1216 mpa->revision = ep->mpa_attr.version; 1217 mpa->private_data_size = htons(plen); 1218 1219 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 1220 1221 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 1222 mpa->private_data_size += 1223 htons(sizeof(struct mpa_v2_conn_params)); 1224 mpa_v2_params.ird = htons((u16)ep->ird); 1225 mpa_v2_params.ord = htons((u16)ep->ord); 1226 CTR5(KTR_IW_CXGBE, "%s:smrep3 %p %d %d %d", __func__, ep, 1227 ep->mpa_attr.version, mpa_v2_params.ird, mpa_v2_params.ord); 1228 1229 if (peer2peer && (ep->mpa_attr.p2p_type != 1230 FW_RI_INIT_P2PTYPE_DISABLED)) { 1231 1232 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 1233 1234 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) { 1235 1236 mpa_v2_params.ord |= 1237 htons(MPA_V2_RDMA_WRITE_RTR); 1238 CTR5(KTR_IW_CXGBE, "%s:smrep4 %p %d %d %d", 1239 __func__, ep, p2p_type, mpa_v2_params.ird, 1240 mpa_v2_params.ord); 1241 } 1242 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) { 1243 1244 mpa_v2_params.ord |= 1245 htons(MPA_V2_RDMA_READ_RTR); 1246 CTR5(KTR_IW_CXGBE, "%s:smrep5 %p %d %d %d", 1247 __func__, ep, p2p_type, mpa_v2_params.ird, 1248 mpa_v2_params.ord); 1249 } 1250 } 1251 1252 memcpy(mpa->private_data, &mpa_v2_params, 1253 sizeof(struct mpa_v2_conn_params)); 1254 1255 if (ep->plen) 1256 memcpy(mpa->private_data + 1257 sizeof(struct mpa_v2_conn_params), pdata, plen); 1258 } else 1259 if (plen) 1260 memcpy(mpa->private_data, pdata, plen); 1261 1262 m = m_getm(NULL, mpalen, M_NOWAIT, MT_DATA); 1263 if (m == NULL) { 1264 free(mpa, M_CXGBE); 1265 return (-ENOMEM); 1266 } 1267 m_copyback(m, 0, mpalen, (void *)mpa); 1268 free(mpa, M_CXGBE); 1269 1270 1271 state_set(&ep->com, MPA_REP_SENT); 1272 ep->snd_seq += mpalen; 1273 err = -sosend(ep->com.so, NULL, NULL, m, NULL, MSG_DONTWAIT, 1274 ep->com.thread); 1275 CTR3(KTR_IW_CXGBE, "%s:smrepE %p %d", __func__, ep, err); 1276 return err; 1277 } 1278 1279 1280 1281 static void close_complete_upcall(struct c4iw_ep *ep, int status) 1282 { 1283 struct iw_cm_event event; 1284 1285 CTR2(KTR_IW_CXGBE, "%s:ccuB %p", __func__, ep); 1286 memset(&event, 0, sizeof(event)); 1287 event.event = IW_CM_EVENT_CLOSE; 1288 event.status = status; 1289 1290 if (ep->com.cm_id) { 1291 1292 CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep); 1293 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1294 deref_cm_id(&ep->com); 1295 set_bit(CLOSE_UPCALL, &ep->com.history); 1296 } 1297 CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep); 1298 } 1299 1300 static int 1301 send_abort(struct c4iw_ep *ep) 1302 { 1303 struct socket *so = ep->com.so; 1304 struct sockopt sopt; 1305 int rc; 1306 struct linger l; 1307 1308 CTR5(KTR_IW_CXGBE, "%s ep %p so %p state %s tid %d", __func__, ep, so, 1309 states[ep->com.state], ep->hwtid); 1310 1311 l.l_onoff = 1; 1312 l.l_linger = 0; 1313 1314 /* linger_time of 0 forces RST to be sent */ 1315 sopt.sopt_dir = SOPT_SET; 1316 sopt.sopt_level = SOL_SOCKET; 1317 sopt.sopt_name = SO_LINGER; 1318 sopt.sopt_val = (caddr_t)&l; 1319 sopt.sopt_valsize = sizeof l; 1320 sopt.sopt_td = NULL; 1321 rc = sosetopt(so, &sopt); 1322 if (rc != 0) { 1323 log(LOG_ERR, "%s: sosetopt(%p, linger = 0) failed with %d.\n", 1324 __func__, so, rc); 1325 } 1326 1327 uninit_iwarp_socket(so); 1328 sodisconnect(so); 1329 set_bit(ABORT_CONN, &ep->com.history); 1330 1331 /* 1332 * TBD: iw_cxgbe driver should receive ABORT reply for every ABORT 1333 * request it has sent. But the current TOE driver is not propagating 1334 * this ABORT reply event (via do_abort_rpl) to iw_cxgbe. So as a work- 1335 * around de-refer 'ep' (which was refered before sending ABORT request) 1336 * here instead of doing it in abort_rpl() handler of iw_cxgbe driver. 1337 */ 1338 c4iw_put_ep(&ep->com); 1339 1340 return (0); 1341 } 1342 1343 static void peer_close_upcall(struct c4iw_ep *ep) 1344 { 1345 struct iw_cm_event event; 1346 1347 CTR2(KTR_IW_CXGBE, "%s:pcuB %p", __func__, ep); 1348 memset(&event, 0, sizeof(event)); 1349 event.event = IW_CM_EVENT_DISCONNECT; 1350 1351 if (ep->com.cm_id) { 1352 1353 CTR2(KTR_IW_CXGBE, "%s:pcu1 %p", __func__, ep); 1354 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1355 set_bit(DISCONN_UPCALL, &ep->com.history); 1356 } 1357 CTR2(KTR_IW_CXGBE, "%s:pcuE %p", __func__, ep); 1358 } 1359 1360 static void peer_abort_upcall(struct c4iw_ep *ep) 1361 { 1362 struct iw_cm_event event; 1363 1364 CTR2(KTR_IW_CXGBE, "%s:pauB %p", __func__, ep); 1365 memset(&event, 0, sizeof(event)); 1366 event.event = IW_CM_EVENT_CLOSE; 1367 event.status = -ECONNRESET; 1368 1369 if (ep->com.cm_id) { 1370 1371 CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep); 1372 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1373 deref_cm_id(&ep->com); 1374 set_bit(ABORT_UPCALL, &ep->com.history); 1375 } 1376 CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep); 1377 } 1378 1379 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 1380 { 1381 struct iw_cm_event event; 1382 1383 CTR3(KTR_IW_CXGBE, "%s:cruB %p, status: %d", __func__, ep, status); 1384 memset(&event, 0, sizeof(event)); 1385 event.event = IW_CM_EVENT_CONNECT_REPLY; 1386 event.status = ((status == -ECONNABORTED) || (status == -EPIPE)) ? 1387 -ECONNRESET : status; 1388 event.local_addr = ep->com.local_addr; 1389 event.remote_addr = ep->com.remote_addr; 1390 1391 if ((status == 0) || (status == -ECONNREFUSED)) { 1392 1393 if (!ep->tried_with_mpa_v1) { 1394 1395 CTR2(KTR_IW_CXGBE, "%s:cru1 %p", __func__, ep); 1396 /* this means MPA_v2 is used */ 1397 event.private_data_len = ep->plen - 1398 sizeof(struct mpa_v2_conn_params); 1399 event.private_data = ep->mpa_pkt + 1400 sizeof(struct mpa_message) + 1401 sizeof(struct mpa_v2_conn_params); 1402 } else { 1403 1404 CTR2(KTR_IW_CXGBE, "%s:cru2 %p", __func__, ep); 1405 /* this means MPA_v1 is used */ 1406 event.private_data_len = ep->plen; 1407 event.private_data = ep->mpa_pkt + 1408 sizeof(struct mpa_message); 1409 } 1410 } 1411 1412 if (ep->com.cm_id) { 1413 1414 CTR2(KTR_IW_CXGBE, "%s:cru3 %p", __func__, ep); 1415 set_bit(CONN_RPL_UPCALL, &ep->com.history); 1416 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1417 } 1418 1419 if(status == -ECONNABORTED) { 1420 1421 CTR3(KTR_IW_CXGBE, "%s:cruE %p %d", __func__, ep, status); 1422 return; 1423 } 1424 1425 if (status < 0) { 1426 1427 CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status); 1428 deref_cm_id(&ep->com); 1429 } 1430 1431 CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep); 1432 } 1433 1434 static int connect_request_upcall(struct c4iw_ep *ep) 1435 { 1436 struct iw_cm_event event; 1437 int ret; 1438 1439 CTR3(KTR_IW_CXGBE, "%s: ep %p, mpa_v1 %d", __func__, ep, 1440 ep->tried_with_mpa_v1); 1441 1442 memset(&event, 0, sizeof(event)); 1443 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1444 event.local_addr = ep->com.local_addr; 1445 event.remote_addr = ep->com.remote_addr; 1446 event.provider_data = ep; 1447 event.so = ep->com.so; 1448 1449 if (!ep->tried_with_mpa_v1) { 1450 /* this means MPA_v2 is used */ 1451 event.ord = ep->ord; 1452 event.ird = ep->ird; 1453 event.private_data_len = ep->plen - 1454 sizeof(struct mpa_v2_conn_params); 1455 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1456 sizeof(struct mpa_v2_conn_params); 1457 } else { 1458 1459 /* this means MPA_v1 is used. Send max supported */ 1460 event.ord = c4iw_max_read_depth; 1461 event.ird = c4iw_max_read_depth; 1462 event.private_data_len = ep->plen; 1463 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1464 } 1465 1466 c4iw_get_ep(&ep->com); 1467 ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, 1468 &event); 1469 if(ret) 1470 c4iw_put_ep(&ep->com); 1471 1472 set_bit(CONNREQ_UPCALL, &ep->com.history); 1473 c4iw_put_ep(&ep->parent_ep->com); 1474 return ret; 1475 } 1476 1477 static void established_upcall(struct c4iw_ep *ep) 1478 { 1479 struct iw_cm_event event; 1480 1481 CTR2(KTR_IW_CXGBE, "%s:euB %p", __func__, ep); 1482 memset(&event, 0, sizeof(event)); 1483 event.event = IW_CM_EVENT_ESTABLISHED; 1484 event.ird = ep->ird; 1485 event.ord = ep->ord; 1486 1487 if (ep->com.cm_id) { 1488 1489 CTR2(KTR_IW_CXGBE, "%s:eu1 %p", __func__, ep); 1490 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1491 set_bit(ESTAB_UPCALL, &ep->com.history); 1492 } 1493 CTR2(KTR_IW_CXGBE, "%s:euE %p", __func__, ep); 1494 } 1495 1496 1497 /* 1498 * process_mpa_reply - process streaming mode MPA reply 1499 * 1500 * Returns: 1501 * 1502 * 0 upon success indicating a connect request was delivered to the ULP 1503 * or the mpa request is incomplete but valid so far. 1504 * 1505 * 1 if a failure requires the caller to close the connection. 1506 * 1507 * 2 if a failure requires the caller to abort the connection. 1508 */ 1509 static int process_mpa_reply(struct c4iw_ep *ep) 1510 { 1511 struct mpa_message *mpa; 1512 struct mpa_v2_conn_params *mpa_v2_params; 1513 u16 plen; 1514 u16 resp_ird, resp_ord; 1515 u8 rtr_mismatch = 0, insuff_ird = 0; 1516 struct c4iw_qp_attributes attrs; 1517 enum c4iw_qp_attr_mask mask; 1518 int err; 1519 struct mbuf *top, *m; 1520 int flags = MSG_DONTWAIT; 1521 struct uio uio; 1522 int disconnect = 0; 1523 1524 CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep); 1525 1526 /* 1527 * Stop mpa timer. If it expired, then 1528 * we ignore the MPA reply. process_timeout() 1529 * will abort the connection. 1530 */ 1531 if (STOP_EP_TIMER(ep)) 1532 return 0; 1533 1534 uio.uio_resid = 1000000; 1535 uio.uio_td = ep->com.thread; 1536 err = soreceive(ep->com.so, NULL, &uio, &top, NULL, &flags); 1537 1538 if (err) { 1539 1540 if (err == EWOULDBLOCK) { 1541 1542 CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep); 1543 START_EP_TIMER(ep); 1544 return 0; 1545 } 1546 err = -err; 1547 CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep); 1548 goto err; 1549 } 1550 1551 if (ep->com.so->so_rcv.sb_mb) { 1552 1553 CTR2(KTR_IW_CXGBE, "%s:pmr3 %p", __func__, ep); 1554 printf("%s data after soreceive called! so %p sb_mb %p top %p\n", 1555 __func__, ep->com.so, ep->com.so->so_rcv.sb_mb, top); 1556 } 1557 1558 m = top; 1559 1560 do { 1561 1562 CTR2(KTR_IW_CXGBE, "%s:pmr4 %p", __func__, ep); 1563 /* 1564 * If we get more than the supported amount of private data 1565 * then we must fail this connection. 1566 */ 1567 if (ep->mpa_pkt_len + m->m_len > sizeof(ep->mpa_pkt)) { 1568 1569 CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep, 1570 ep->mpa_pkt_len + m->m_len); 1571 err = (-EINVAL); 1572 goto err_stop_timer; 1573 } 1574 1575 /* 1576 * copy the new data into our accumulation buffer. 1577 */ 1578 m_copydata(m, 0, m->m_len, &(ep->mpa_pkt[ep->mpa_pkt_len])); 1579 ep->mpa_pkt_len += m->m_len; 1580 if (!m->m_next) 1581 m = m->m_nextpkt; 1582 else 1583 m = m->m_next; 1584 } while (m); 1585 1586 m_freem(top); 1587 /* 1588 * if we don't even have the mpa message, then bail. 1589 */ 1590 if (ep->mpa_pkt_len < sizeof(*mpa)) { 1591 return 0; 1592 } 1593 mpa = (struct mpa_message *) ep->mpa_pkt; 1594 1595 /* Validate MPA header. */ 1596 if (mpa->revision > mpa_rev) { 1597 1598 CTR4(KTR_IW_CXGBE, "%s:pmr6 %p %d %d", __func__, ep, 1599 mpa->revision, mpa_rev); 1600 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d, " 1601 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1602 err = -EPROTO; 1603 goto err_stop_timer; 1604 } 1605 1606 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1607 1608 CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep); 1609 err = -EPROTO; 1610 goto err_stop_timer; 1611 } 1612 1613 plen = ntohs(mpa->private_data_size); 1614 1615 /* 1616 * Fail if there's too much private data. 1617 */ 1618 if (plen > MPA_MAX_PRIVATE_DATA) { 1619 1620 CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep); 1621 err = -EPROTO; 1622 goto err_stop_timer; 1623 } 1624 1625 /* 1626 * If plen does not account for pkt size 1627 */ 1628 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1629 1630 CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep); 1631 STOP_EP_TIMER(ep); 1632 err = -EPROTO; 1633 goto err_stop_timer; 1634 } 1635 1636 ep->plen = (u8) plen; 1637 1638 /* 1639 * If we don't have all the pdata yet, then bail. 1640 * We'll continue process when more data arrives. 1641 */ 1642 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { 1643 1644 CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep); 1645 return 0; 1646 } 1647 1648 if (mpa->flags & MPA_REJECT) { 1649 1650 CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep); 1651 err = -ECONNREFUSED; 1652 goto err_stop_timer; 1653 } 1654 1655 /* 1656 * If we get here we have accumulated the entire mpa 1657 * start reply message including private data. And 1658 * the MPA header is valid. 1659 */ 1660 state_set(&ep->com, FPDU_MODE); 1661 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1662 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1663 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1664 ep->mpa_attr.version = mpa->revision; 1665 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1666 1667 if (mpa->revision == 2) { 1668 1669 CTR2(KTR_IW_CXGBE, "%s:pmrc %p", __func__, ep); 1670 ep->mpa_attr.enhanced_rdma_conn = 1671 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1672 1673 if (ep->mpa_attr.enhanced_rdma_conn) { 1674 1675 CTR2(KTR_IW_CXGBE, "%s:pmrd %p", __func__, ep); 1676 mpa_v2_params = (struct mpa_v2_conn_params *) 1677 (ep->mpa_pkt + sizeof(*mpa)); 1678 resp_ird = ntohs(mpa_v2_params->ird) & 1679 MPA_V2_IRD_ORD_MASK; 1680 resp_ord = ntohs(mpa_v2_params->ord) & 1681 MPA_V2_IRD_ORD_MASK; 1682 1683 /* 1684 * This is a double-check. Ideally, below checks are 1685 * not required since ird/ord stuff has been taken 1686 * care of in c4iw_accept_cr 1687 */ 1688 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1689 1690 CTR2(KTR_IW_CXGBE, "%s:pmre %p", __func__, ep); 1691 err = -ENOMEM; 1692 ep->ird = resp_ord; 1693 ep->ord = resp_ird; 1694 insuff_ird = 1; 1695 } 1696 1697 if (ntohs(mpa_v2_params->ird) & 1698 MPA_V2_PEER2PEER_MODEL) { 1699 1700 CTR2(KTR_IW_CXGBE, "%s:pmrf %p", __func__, ep); 1701 if (ntohs(mpa_v2_params->ord) & 1702 MPA_V2_RDMA_WRITE_RTR) { 1703 1704 CTR2(KTR_IW_CXGBE, "%s:pmrg %p", __func__, ep); 1705 ep->mpa_attr.p2p_type = 1706 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1707 } 1708 else if (ntohs(mpa_v2_params->ord) & 1709 MPA_V2_RDMA_READ_RTR) { 1710 1711 CTR2(KTR_IW_CXGBE, "%s:pmrh %p", __func__, ep); 1712 ep->mpa_attr.p2p_type = 1713 FW_RI_INIT_P2PTYPE_READ_REQ; 1714 } 1715 } 1716 } 1717 } else { 1718 1719 CTR2(KTR_IW_CXGBE, "%s:pmri %p", __func__, ep); 1720 1721 if (mpa->revision == 1) { 1722 1723 CTR2(KTR_IW_CXGBE, "%s:pmrj %p", __func__, ep); 1724 1725 if (peer2peer) { 1726 1727 CTR2(KTR_IW_CXGBE, "%s:pmrk %p", __func__, ep); 1728 ep->mpa_attr.p2p_type = p2p_type; 1729 } 1730 } 1731 } 1732 1733 if (set_tcpinfo(ep)) { 1734 1735 CTR2(KTR_IW_CXGBE, "%s:pmrl %p", __func__, ep); 1736 printf("%s set_tcpinfo error\n", __func__); 1737 err = -ECONNRESET; 1738 goto err; 1739 } 1740 1741 CTR6(KTR_IW_CXGBE, "%s - crc_enabled = %d, recv_marker_enabled = %d, " 1742 "xmit_marker_enabled = %d, version = %d p2p_type = %d", __func__, 1743 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1744 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1745 ep->mpa_attr.p2p_type); 1746 1747 /* 1748 * If responder's RTR does not match with that of initiator, assign 1749 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1750 * generated when moving QP to RTS state. 1751 * A TERM message will be sent after QP has moved to RTS state 1752 */ 1753 if ((ep->mpa_attr.version == 2) && peer2peer && 1754 (ep->mpa_attr.p2p_type != p2p_type)) { 1755 1756 CTR2(KTR_IW_CXGBE, "%s:pmrm %p", __func__, ep); 1757 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1758 rtr_mismatch = 1; 1759 } 1760 1761 1762 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 1763 attrs.mpa_attr = ep->mpa_attr; 1764 attrs.max_ird = ep->ird; 1765 attrs.max_ord = ep->ord; 1766 attrs.llp_stream_handle = ep; 1767 attrs.next_state = C4IW_QP_STATE_RTS; 1768 1769 mask = C4IW_QP_ATTR_NEXT_STATE | 1770 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1771 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1772 1773 /* bind QP and TID with INIT_WR */ 1774 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 1775 1776 if (err) { 1777 1778 CTR2(KTR_IW_CXGBE, "%s:pmrn %p", __func__, ep); 1779 goto err; 1780 } 1781 1782 /* 1783 * If responder's RTR requirement did not match with what initiator 1784 * supports, generate TERM message 1785 */ 1786 if (rtr_mismatch) { 1787 1788 CTR2(KTR_IW_CXGBE, "%s:pmro %p", __func__, ep); 1789 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1790 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1791 attrs.ecode = MPA_NOMATCH_RTR; 1792 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1793 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1794 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1795 err = -ENOMEM; 1796 disconnect = 1; 1797 goto out; 1798 } 1799 1800 /* 1801 * Generate TERM if initiator IRD is not sufficient for responder 1802 * provided ORD. Currently, we do the same behaviour even when 1803 * responder provided IRD is also not sufficient as regards to 1804 * initiator ORD. 1805 */ 1806 if (insuff_ird) { 1807 1808 CTR2(KTR_IW_CXGBE, "%s:pmrp %p", __func__, ep); 1809 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1810 __func__); 1811 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1812 attrs.ecode = MPA_INSUFF_IRD; 1813 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1814 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1815 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1816 err = -ENOMEM; 1817 disconnect = 1; 1818 goto out; 1819 } 1820 goto out; 1821 err_stop_timer: 1822 STOP_EP_TIMER(ep); 1823 err: 1824 disconnect = 2; 1825 out: 1826 connect_reply_upcall(ep, err); 1827 CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep); 1828 return disconnect; 1829 } 1830 1831 /* 1832 * process_mpa_request - process streaming mode MPA request 1833 * 1834 * Returns: 1835 * 1836 * 0 upon success indicating a connect request was delivered to the ULP 1837 * or the mpa request is incomplete but valid so far. 1838 * 1839 * 1 if a failure requires the caller to close the connection. 1840 * 1841 * 2 if a failure requires the caller to abort the connection. 1842 */ 1843 static int 1844 process_mpa_request(struct c4iw_ep *ep) 1845 { 1846 struct mpa_message *mpa; 1847 u16 plen; 1848 int flags = MSG_DONTWAIT; 1849 int rc; 1850 struct iovec iov; 1851 struct uio uio; 1852 enum c4iw_ep_state state = state_read(&ep->com); 1853 1854 CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]); 1855 1856 if (state != MPA_REQ_WAIT) 1857 return 0; 1858 1859 iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len]; 1860 iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1861 uio.uio_iov = &iov; 1862 uio.uio_iovcnt = 1; 1863 uio.uio_offset = 0; 1864 uio.uio_resid = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; 1865 uio.uio_segflg = UIO_SYSSPACE; 1866 uio.uio_rw = UIO_READ; 1867 uio.uio_td = NULL; /* uio.uio_td = ep->com.thread; */ 1868 1869 rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags); 1870 if (rc == EAGAIN) 1871 return 0; 1872 else if (rc) 1873 goto err_stop_timer; 1874 1875 KASSERT(uio.uio_offset > 0, ("%s: sorecieve on so %p read no data", 1876 __func__, ep->com.so)); 1877 ep->mpa_pkt_len += uio.uio_offset; 1878 1879 /* 1880 * If we get more than the supported amount of private data then we must 1881 * fail this connection. XXX: check so_rcv->sb_cc, or peek with another 1882 * soreceive, or increase the size of mpa_pkt by 1 and abort if the last 1883 * byte is filled by the soreceive above. 1884 */ 1885 1886 /* Don't even have the MPA message. Wait for more data to arrive. */ 1887 if (ep->mpa_pkt_len < sizeof(*mpa)) 1888 return 0; 1889 mpa = (struct mpa_message *) ep->mpa_pkt; 1890 1891 /* 1892 * Validate MPA Header. 1893 */ 1894 if (mpa->revision > mpa_rev) { 1895 log(LOG_ERR, "%s: MPA version mismatch. Local = %d," 1896 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1897 goto err_stop_timer; 1898 } 1899 1900 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) 1901 goto err_stop_timer; 1902 1903 /* 1904 * Fail if there's too much private data. 1905 */ 1906 plen = ntohs(mpa->private_data_size); 1907 if (plen > MPA_MAX_PRIVATE_DATA) 1908 goto err_stop_timer; 1909 1910 /* 1911 * If plen does not account for pkt size 1912 */ 1913 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) 1914 goto err_stop_timer; 1915 1916 ep->plen = (u8) plen; 1917 1918 /* 1919 * If we don't have all the pdata yet, then bail. 1920 */ 1921 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1922 return 0; 1923 1924 /* 1925 * If we get here we have accumulated the entire mpa 1926 * start reply message including private data. 1927 */ 1928 ep->mpa_attr.initiator = 0; 1929 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1930 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1931 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1932 ep->mpa_attr.version = mpa->revision; 1933 if (mpa->revision == 1) 1934 ep->tried_with_mpa_v1 = 1; 1935 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1936 1937 if (mpa->revision == 2) { 1938 ep->mpa_attr.enhanced_rdma_conn = 1939 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1940 if (ep->mpa_attr.enhanced_rdma_conn) { 1941 struct mpa_v2_conn_params *mpa_v2_params; 1942 u16 ird, ord; 1943 1944 mpa_v2_params = (void *)&ep->mpa_pkt[sizeof(*mpa)]; 1945 ird = ntohs(mpa_v2_params->ird); 1946 ord = ntohs(mpa_v2_params->ord); 1947 1948 ep->ird = ird & MPA_V2_IRD_ORD_MASK; 1949 ep->ord = ord & MPA_V2_IRD_ORD_MASK; 1950 if (ird & MPA_V2_PEER2PEER_MODEL && peer2peer) { 1951 if (ord & MPA_V2_RDMA_WRITE_RTR) { 1952 ep->mpa_attr.p2p_type = 1953 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1954 } else if (ord & MPA_V2_RDMA_READ_RTR) { 1955 ep->mpa_attr.p2p_type = 1956 FW_RI_INIT_P2PTYPE_READ_REQ; 1957 } 1958 } 1959 } 1960 } else if (mpa->revision == 1 && peer2peer) 1961 ep->mpa_attr.p2p_type = p2p_type; 1962 1963 if (set_tcpinfo(ep)) 1964 goto err_stop_timer; 1965 1966 CTR5(KTR_IW_CXGBE, "%s: crc_enabled = %d, recv_marker_enabled = %d, " 1967 "xmit_marker_enabled = %d, version = %d", __func__, 1968 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1969 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); 1970 1971 state_set(&ep->com, MPA_REQ_RCVD); 1972 STOP_EP_TIMER(ep); 1973 1974 /* drive upcall */ 1975 mutex_lock(&ep->parent_ep->com.mutex); 1976 if (ep->parent_ep->com.state != DEAD) { 1977 if (connect_request_upcall(ep)) 1978 goto err_unlock_parent; 1979 } else 1980 goto err_unlock_parent; 1981 mutex_unlock(&ep->parent_ep->com.mutex); 1982 return 0; 1983 1984 err_unlock_parent: 1985 mutex_unlock(&ep->parent_ep->com.mutex); 1986 goto err_out; 1987 err_stop_timer: 1988 STOP_EP_TIMER(ep); 1989 err_out: 1990 return 2; 1991 } 1992 1993 /* 1994 * Upcall from the adapter indicating data has been transmitted. 1995 * For us its just the single MPA request or reply. We can now free 1996 * the skb holding the mpa message. 1997 */ 1998 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 1999 { 2000 int err; 2001 struct c4iw_ep *ep = to_ep(cm_id); 2002 CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); 2003 int abort = 0; 2004 2005 if ((state_read(&ep->com) == DEAD) || 2006 (state_read(&ep->com) != MPA_REQ_RCVD)) { 2007 2008 CTR2(KTR_IW_CXGBE, "%s:crc1 %p", __func__, ep); 2009 c4iw_put_ep(&ep->com); 2010 return -ECONNRESET; 2011 } 2012 set_bit(ULP_REJECT, &ep->com.history); 2013 2014 if (mpa_rev == 0) { 2015 2016 CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep); 2017 abort = 1; 2018 } 2019 else { 2020 2021 CTR2(KTR_IW_CXGBE, "%s:crc3 %p", __func__, ep); 2022 abort = send_mpa_reject(ep, pdata, pdata_len); 2023 } 2024 stop_ep_timer(ep); 2025 err = c4iw_ep_disconnect(ep, abort != 0, GFP_KERNEL); 2026 c4iw_put_ep(&ep->com); 2027 CTR3(KTR_IW_CXGBE, "%s:crc4 %p, err: %d", __func__, ep, err); 2028 return 0; 2029 } 2030 2031 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2032 { 2033 int err; 2034 struct c4iw_qp_attributes attrs; 2035 enum c4iw_qp_attr_mask mask; 2036 struct c4iw_ep *ep = to_ep(cm_id); 2037 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2038 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2039 int abort = 0; 2040 2041 CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep); 2042 2043 if (state_read(&ep->com) == DEAD) { 2044 2045 CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep); 2046 err = -ECONNRESET; 2047 goto err_out; 2048 } 2049 2050 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2051 BUG_ON(!qp); 2052 2053 set_bit(ULP_ACCEPT, &ep->com.history); 2054 2055 if ((conn_param->ord > c4iw_max_read_depth) || 2056 (conn_param->ird > c4iw_max_read_depth)) { 2057 2058 CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep); 2059 err = -EINVAL; 2060 goto err_abort; 2061 } 2062 2063 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2064 2065 CTR2(KTR_IW_CXGBE, "%s:cac3 %p", __func__, ep); 2066 2067 if (conn_param->ord > ep->ird) { 2068 2069 CTR2(KTR_IW_CXGBE, "%s:cac4 %p", __func__, ep); 2070 ep->ird = conn_param->ird; 2071 ep->ord = conn_param->ord; 2072 send_mpa_reject(ep, conn_param->private_data, 2073 conn_param->private_data_len); 2074 err = -ENOMEM; 2075 goto err_abort; 2076 } 2077 2078 if (conn_param->ird > ep->ord) { 2079 2080 CTR2(KTR_IW_CXGBE, "%s:cac5 %p", __func__, ep); 2081 2082 if (!ep->ord) { 2083 2084 CTR2(KTR_IW_CXGBE, "%s:cac6 %p", __func__, ep); 2085 conn_param->ird = 1; 2086 } 2087 else { 2088 CTR2(KTR_IW_CXGBE, "%s:cac7 %p", __func__, ep); 2089 err = -ENOMEM; 2090 goto err_abort; 2091 } 2092 } 2093 2094 } 2095 ep->ird = conn_param->ird; 2096 ep->ord = conn_param->ord; 2097 2098 if (ep->mpa_attr.version != 2) { 2099 2100 CTR2(KTR_IW_CXGBE, "%s:cac8 %p", __func__, ep); 2101 2102 if (peer2peer && ep->ird == 0) { 2103 2104 CTR2(KTR_IW_CXGBE, "%s:cac9 %p", __func__, ep); 2105 ep->ird = 1; 2106 } 2107 } 2108 2109 2110 ep->com.cm_id = cm_id; 2111 ref_cm_id(&ep->com); 2112 ep->com.qp = qp; 2113 ref_qp(ep); 2114 //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; 2115 2116 /* bind QP to EP and move to RTS */ 2117 attrs.mpa_attr = ep->mpa_attr; 2118 attrs.max_ird = ep->ird; 2119 attrs.max_ord = ep->ord; 2120 attrs.llp_stream_handle = ep; 2121 attrs.next_state = C4IW_QP_STATE_RTS; 2122 2123 /* bind QP and TID with INIT_WR */ 2124 mask = C4IW_QP_ATTR_NEXT_STATE | 2125 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2126 C4IW_QP_ATTR_MPA_ATTR | 2127 C4IW_QP_ATTR_MAX_IRD | 2128 C4IW_QP_ATTR_MAX_ORD; 2129 2130 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, mask, &attrs, 1); 2131 2132 if (err) { 2133 2134 CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2135 goto err_defef_cm_id; 2136 } 2137 err = send_mpa_reply(ep, conn_param->private_data, 2138 conn_param->private_data_len); 2139 2140 if (err) { 2141 2142 CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); 2143 goto err_defef_cm_id; 2144 } 2145 2146 state_set(&ep->com, FPDU_MODE); 2147 established_upcall(ep); 2148 c4iw_put_ep(&ep->com); 2149 CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep); 2150 return 0; 2151 err_defef_cm_id: 2152 deref_cm_id(&ep->com); 2153 err_abort: 2154 abort = 1; 2155 err_out: 2156 if (abort) 2157 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 2158 c4iw_put_ep(&ep->com); 2159 CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep); 2160 return err; 2161 } 2162 2163 2164 2165 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2166 { 2167 int err = 0; 2168 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2169 struct c4iw_ep *ep = NULL; 2170 struct nhop4_extended nh4; 2171 2172 CTR2(KTR_IW_CXGBE, "%s:ccB %p", __func__, cm_id); 2173 2174 if ((conn_param->ord > c4iw_max_read_depth) || 2175 (conn_param->ird > c4iw_max_read_depth)) { 2176 2177 CTR2(KTR_IW_CXGBE, "%s:cc1 %p", __func__, cm_id); 2178 err = -EINVAL; 2179 goto out; 2180 } 2181 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2182 init_timer(&ep->timer); 2183 ep->plen = conn_param->private_data_len; 2184 2185 if (ep->plen) { 2186 2187 CTR2(KTR_IW_CXGBE, "%s:cc3 %p", __func__, ep); 2188 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2189 conn_param->private_data, ep->plen); 2190 } 2191 ep->ird = conn_param->ird; 2192 ep->ord = conn_param->ord; 2193 2194 if (peer2peer && ep->ord == 0) { 2195 2196 CTR2(KTR_IW_CXGBE, "%s:cc4 %p", __func__, ep); 2197 ep->ord = 1; 2198 } 2199 2200 ep->com.dev = dev; 2201 ep->com.cm_id = cm_id; 2202 ref_cm_id(&ep->com); 2203 ep->com.qp = get_qhp(dev, conn_param->qpn); 2204 2205 if (!ep->com.qp) { 2206 2207 CTR2(KTR_IW_CXGBE, "%s:cc5 %p", __func__, ep); 2208 err = -EINVAL; 2209 goto fail2; 2210 } 2211 ref_qp(ep); 2212 ep->com.thread = curthread; 2213 ep->com.so = cm_id->so; 2214 2215 /* find a route */ 2216 err = find_route( 2217 cm_id->local_addr.sin_addr.s_addr, 2218 cm_id->remote_addr.sin_addr.s_addr, 2219 cm_id->local_addr.sin_port, 2220 cm_id->remote_addr.sin_port, 0, &nh4); 2221 2222 if (err) { 2223 2224 CTR2(KTR_IW_CXGBE, "%s:cc7 %p", __func__, ep); 2225 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2226 err = -EHOSTUNREACH; 2227 goto fail2; 2228 } 2229 2230 if (!(nh4.nh_ifp->if_capenable & IFCAP_TOE) || 2231 TOEDEV(nh4.nh_ifp) == NULL) { 2232 err = -ENOPROTOOPT; 2233 goto fail3; 2234 } 2235 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 2236 2237 state_set(&ep->com, CONNECTING); 2238 ep->tos = 0; 2239 ep->com.local_addr = cm_id->local_addr; 2240 ep->com.remote_addr = cm_id->remote_addr; 2241 err = -soconnect(ep->com.so, (struct sockaddr *)&ep->com.remote_addr, 2242 ep->com.thread); 2243 2244 if (!err) { 2245 init_iwarp_socket(cm_id->so, &ep->com); 2246 goto out; 2247 } else { 2248 goto fail2; 2249 } 2250 2251 fail3: 2252 fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); 2253 fail2: 2254 deref_cm_id(&ep->com); 2255 c4iw_put_ep(&ep->com); 2256 ep = NULL; /* CTR shouldn't display already-freed ep. */ 2257 out: 2258 CTR2(KTR_IW_CXGBE, "%s:ccE %p", __func__, ep); 2259 return err; 2260 } 2261 2262 /* 2263 * iwcm->create_listen_ep. Returns -errno on failure. 2264 */ 2265 int 2266 c4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog) 2267 { 2268 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2269 struct c4iw_listen_ep *ep; 2270 struct socket *so = cm_id->so; 2271 2272 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2273 ep->com.cm_id = cm_id; 2274 ref_cm_id(&ep->com); 2275 ep->com.dev = dev; 2276 ep->backlog = backlog; 2277 ep->com.local_addr = cm_id->local_addr; 2278 ep->com.thread = curthread; 2279 state_set(&ep->com, LISTEN); 2280 ep->com.so = so; 2281 2282 cm_id->provider_data = ep; 2283 return (0); 2284 } 2285 2286 void 2287 c4iw_destroy_listen_ep(struct iw_cm_id *cm_id) 2288 { 2289 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2290 2291 CTR4(KTR_IW_CXGBE, "%s: cm_id %p, so %p, state %s", __func__, cm_id, 2292 cm_id->so, states[ep->com.state]); 2293 2294 state_set(&ep->com, DEAD); 2295 deref_cm_id(&ep->com); 2296 c4iw_put_ep(&ep->com); 2297 2298 return; 2299 } 2300 2301 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2302 { 2303 int ret = 0; 2304 int close = 0; 2305 int fatal = 0; 2306 struct c4iw_rdev *rdev; 2307 2308 mutex_lock(&ep->com.mutex); 2309 2310 CTR2(KTR_IW_CXGBE, "%s:cedB %p", __func__, ep); 2311 2312 rdev = &ep->com.dev->rdev; 2313 2314 if (c4iw_fatal_error(rdev)) { 2315 2316 CTR2(KTR_IW_CXGBE, "%s:ced1 %p", __func__, ep); 2317 fatal = 1; 2318 close_complete_upcall(ep, -ECONNRESET); 2319 ep->com.state = DEAD; 2320 } 2321 CTR3(KTR_IW_CXGBE, "%s:ced2 %p %s", __func__, ep, 2322 states[ep->com.state]); 2323 2324 switch (ep->com.state) { 2325 2326 case MPA_REQ_WAIT: 2327 case MPA_REQ_SENT: 2328 case MPA_REQ_RCVD: 2329 case MPA_REP_SENT: 2330 case FPDU_MODE: 2331 close = 1; 2332 if (abrupt) 2333 ep->com.state = ABORTING; 2334 else { 2335 ep->com.state = CLOSING; 2336 START_EP_TIMER(ep); 2337 } 2338 set_bit(CLOSE_SENT, &ep->com.flags); 2339 break; 2340 2341 case CLOSING: 2342 2343 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2344 2345 close = 1; 2346 if (abrupt) { 2347 STOP_EP_TIMER(ep); 2348 ep->com.state = ABORTING; 2349 } else 2350 ep->com.state = MORIBUND; 2351 } 2352 break; 2353 2354 case MORIBUND: 2355 case ABORTING: 2356 case DEAD: 2357 CTR3(KTR_IW_CXGBE, 2358 "%s ignoring disconnect ep %p state %u", __func__, 2359 ep, ep->com.state); 2360 break; 2361 2362 default: 2363 BUG(); 2364 break; 2365 } 2366 2367 mutex_unlock(&ep->com.mutex); 2368 2369 if (close) { 2370 2371 CTR2(KTR_IW_CXGBE, "%s:ced3 %p", __func__, ep); 2372 2373 if (abrupt) { 2374 2375 CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep); 2376 set_bit(EP_DISC_ABORT, &ep->com.history); 2377 close_complete_upcall(ep, -ECONNRESET); 2378 ret = send_abort(ep); 2379 if (ret) 2380 fatal = 1; 2381 } else { 2382 2383 CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep); 2384 set_bit(EP_DISC_CLOSE, &ep->com.history); 2385 2386 if (!ep->parent_ep) 2387 __state_set(&ep->com, MORIBUND); 2388 sodisconnect(ep->com.so); 2389 } 2390 2391 } 2392 2393 if (fatal) { 2394 set_bit(EP_DISC_FAIL, &ep->com.history); 2395 if (!abrupt) { 2396 STOP_EP_TIMER(ep); 2397 close_complete_upcall(ep, -EIO); 2398 } 2399 if (ep->com.qp) { 2400 struct c4iw_qp_attributes attrs; 2401 2402 attrs.next_state = C4IW_QP_STATE_ERROR; 2403 ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, 2404 C4IW_QP_ATTR_NEXT_STATE, 2405 &attrs, 1); 2406 if (ret) { 2407 CTR2(KTR_IW_CXGBE, "%s:ced7 %p", __func__, ep); 2408 printf("%s - qp <- error failed!\n", __func__); 2409 } 2410 } 2411 release_ep_resources(ep); 2412 ep->com.state = DEAD; 2413 CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep); 2414 } 2415 CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep); 2416 return ret; 2417 } 2418 2419 #ifdef C4IW_EP_REDIRECT 2420 int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new, 2421 struct l2t_entry *l2t) 2422 { 2423 struct c4iw_ep *ep = ctx; 2424 2425 if (ep->dst != old) 2426 return 0; 2427 2428 PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new, 2429 l2t); 2430 dst_hold(new); 2431 cxgb4_l2t_release(ep->l2t); 2432 ep->l2t = l2t; 2433 dst_release(old); 2434 ep->dst = new; 2435 return 1; 2436 } 2437 #endif 2438 2439 2440 2441 static void ep_timeout(unsigned long arg) 2442 { 2443 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 2444 2445 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 2446 2447 /* 2448 * Only insert if it is not already on the list. 2449 */ 2450 if (!(ep->com.ep_events & C4IW_EVENT_TIMEOUT)) { 2451 CTR2(KTR_IW_CXGBE, "%s:et1 %p", __func__, ep); 2452 add_ep_to_req_list(ep, C4IW_EVENT_TIMEOUT); 2453 } 2454 } 2455 } 2456 2457 static int fw6_wr_rpl(struct adapter *sc, const __be64 *rpl) 2458 { 2459 uint64_t val = be64toh(*rpl); 2460 int ret; 2461 struct c4iw_wr_wait *wr_waitp; 2462 2463 ret = (int)((val >> 8) & 0xff); 2464 wr_waitp = (struct c4iw_wr_wait *)rpl[1]; 2465 CTR3(KTR_IW_CXGBE, "%s wr_waitp %p ret %u", __func__, wr_waitp, ret); 2466 if (wr_waitp) 2467 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 2468 2469 return (0); 2470 } 2471 2472 static int fw6_cqe_handler(struct adapter *sc, const __be64 *rpl) 2473 { 2474 struct cqe_list_entry *cle; 2475 unsigned long flag; 2476 2477 cle = malloc(sizeof(*cle), M_CXGBE, M_NOWAIT); 2478 cle->rhp = sc->iwarp_softc; 2479 cle->err_cqe = *(const struct t4_cqe *)(&rpl[0]); 2480 2481 spin_lock_irqsave(&err_cqe_lock, flag); 2482 list_add_tail(&cle->entry, &err_cqe_list); 2483 queue_work(c4iw_taskq, &c4iw_task); 2484 spin_unlock_irqrestore(&err_cqe_lock, flag); 2485 2486 return (0); 2487 } 2488 2489 static int 2490 process_terminate(struct c4iw_ep *ep) 2491 { 2492 struct c4iw_qp_attributes attrs; 2493 2494 CTR2(KTR_IW_CXGBE, "%s:tB %p %d", __func__, ep); 2495 2496 if (ep && ep->com.qp) { 2497 2498 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", 2499 ep->hwtid, ep->com.qp->wq.sq.qid); 2500 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2501 c4iw_modify_qp(ep->com.dev, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 2502 1); 2503 } else 2504 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", 2505 ep->hwtid); 2506 CTR2(KTR_IW_CXGBE, "%s:tE %p %d", __func__, ep); 2507 2508 return 0; 2509 } 2510 2511 int __init c4iw_cm_init(void) 2512 { 2513 2514 t4_register_cpl_handler(CPL_RDMA_TERMINATE, terminate); 2515 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, fw6_wr_rpl); 2516 t4_register_fw_msg_handler(FW6_TYPE_CQE, fw6_cqe_handler); 2517 t4_register_an_handler(c4iw_ev_handler); 2518 2519 TAILQ_INIT(&req_list); 2520 spin_lock_init(&req_lock); 2521 INIT_LIST_HEAD(&err_cqe_list); 2522 spin_lock_init(&err_cqe_lock); 2523 2524 INIT_WORK(&c4iw_task, process_req); 2525 2526 c4iw_taskq = create_singlethread_workqueue("iw_cxgbe"); 2527 if (!c4iw_taskq) 2528 return -ENOMEM; 2529 2530 return 0; 2531 } 2532 2533 void __exit c4iw_cm_term(void) 2534 { 2535 WARN_ON(!TAILQ_EMPTY(&req_list)); 2536 WARN_ON(!list_empty(&err_cqe_list)); 2537 flush_workqueue(c4iw_taskq); 2538 destroy_workqueue(c4iw_taskq); 2539 2540 t4_register_cpl_handler(CPL_RDMA_TERMINATE, NULL); 2541 t4_register_fw_msg_handler(FW6_TYPE_WR_RPL, NULL); 2542 t4_register_fw_msg_handler(FW6_TYPE_CQE, NULL); 2543 t4_register_an_handler(NULL); 2544 } 2545 #endif 2546