1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2022 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "irdma_main.h" 36 37 static void irdma_cm_post_event(struct irdma_cm_event *event); 38 static void irdma_disconnect_worker(struct work_struct *work); 39 40 /** 41 * irdma_free_sqbuf - put back puda buffer if refcount is 0 42 * @vsi: The VSI structure of the device 43 * @bufp: puda buffer to free 44 */ 45 void 46 irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp) 47 { 48 struct irdma_puda_buf *buf = bufp; 49 struct irdma_puda_rsrc *ilq = vsi->ilq; 50 51 if (atomic_dec_and_test(&buf->refcount)) 52 irdma_puda_ret_bufpool(ilq, buf); 53 } 54 55 /** 56 * irdma_record_ird_ord - Record IRD/ORD passed in 57 * @cm_node: connection's node 58 * @conn_ird: connection IRD 59 * @conn_ord: connection ORD 60 */ 61 static void 62 irdma_record_ird_ord(struct irdma_cm_node *cm_node, u32 conn_ird, 63 u32 conn_ord) 64 { 65 if (conn_ird > cm_node->dev->hw_attrs.max_hw_ird) 66 conn_ird = cm_node->dev->hw_attrs.max_hw_ird; 67 68 if (conn_ord > cm_node->dev->hw_attrs.max_hw_ord) 69 conn_ord = cm_node->dev->hw_attrs.max_hw_ord; 70 else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO) 71 conn_ord = 1; 72 cm_node->ird_size = conn_ird; 73 cm_node->ord_size = conn_ord; 74 } 75 76 /** 77 * irdma_copy_ip_ntohl - copy IP address from network to host 78 * @dst: IP address in host order 79 * @src: IP address in network order (big endian) 80 */ 81 void 82 irdma_copy_ip_ntohl(u32 *dst, __be32 *src) 83 { 84 *dst++ = ntohl(*src++); 85 *dst++ = ntohl(*src++); 86 *dst++ = ntohl(*src++); 87 *dst = ntohl(*src); 88 } 89 90 /** 91 * irdma_copy_ip_htonl - copy IP address from host to network order 92 * @dst: IP address in network order (big endian) 93 * @src: IP address in host order 94 */ 95 void 96 irdma_copy_ip_htonl(__be32 *dst, u32 *src) 97 { 98 *dst++ = htonl(*src++); 99 *dst++ = htonl(*src++); 100 *dst++ = htonl(*src++); 101 *dst = htonl(*src); 102 } 103 104 /** 105 * irdma_get_addr_info 106 * @cm_node: contains ip/tcp info 107 * @cm_info: to get a copy of the cm_node ip/tcp info 108 */ 109 static void 110 irdma_get_addr_info(struct irdma_cm_node *cm_node, 111 struct irdma_cm_info *cm_info) 112 { 113 memset(cm_info, 0, sizeof(*cm_info)); 114 cm_info->ipv4 = cm_node->ipv4; 115 cm_info->vlan_id = cm_node->vlan_id; 116 memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr)); 117 memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr)); 118 cm_info->loc_port = cm_node->loc_port; 119 cm_info->rem_port = cm_node->rem_port; 120 } 121 122 /** 123 * irdma_fill_sockaddr4 - fill in addr info for IPv4 connection 124 * @cm_node: connection's node 125 * @event: upper layer's cm event 126 */ 127 static inline void 128 irdma_fill_sockaddr4(struct irdma_cm_node *cm_node, 129 struct iw_cm_event *event) 130 { 131 struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr; 132 struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr; 133 134 laddr->sin_family = AF_INET; 135 raddr->sin_family = AF_INET; 136 137 laddr->sin_port = htons(cm_node->loc_port); 138 raddr->sin_port = htons(cm_node->rem_port); 139 140 laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]); 141 raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]); 142 } 143 144 /** 145 * irdma_fill_sockaddr6 - fill in addr info for IPv6 connection 146 * @cm_node: connection's node 147 * @event: upper layer's cm event 148 */ 149 static inline void 150 irdma_fill_sockaddr6(struct irdma_cm_node *cm_node, 151 struct iw_cm_event *event) 152 { 153 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr; 154 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr; 155 156 laddr6->sin6_family = AF_INET6; 157 raddr6->sin6_family = AF_INET6; 158 159 laddr6->sin6_port = htons(cm_node->loc_port); 160 raddr6->sin6_port = htons(cm_node->rem_port); 161 162 irdma_copy_ip_htonl(laddr6->sin6_addr.__u6_addr.__u6_addr32, 163 cm_node->loc_addr); 164 irdma_copy_ip_htonl(raddr6->sin6_addr.__u6_addr.__u6_addr32, 165 cm_node->rem_addr); 166 } 167 168 /** 169 * irdma_get_cmevent_info - for cm event upcall 170 * @cm_node: connection's node 171 * @cm_id: upper layers cm struct for the event 172 * @event: upper layer's cm event 173 */ 174 static inline void 175 irdma_get_cmevent_info(struct irdma_cm_node *cm_node, 176 struct iw_cm_id *cm_id, 177 struct iw_cm_event *event) 178 { 179 memcpy(&event->local_addr, &cm_id->m_local_addr, 180 sizeof(event->local_addr)); 181 memcpy(&event->remote_addr, &cm_id->m_remote_addr, 182 sizeof(event->remote_addr)); 183 if (cm_node) { 184 event->private_data = cm_node->pdata_buf; 185 event->private_data_len = (u8)cm_node->pdata.size; 186 event->ird = cm_node->ird_size; 187 event->ord = cm_node->ord_size; 188 } 189 } 190 191 /** 192 * irdma_send_cm_event - upcall cm's event handler 193 * @cm_node: connection's node 194 * @cm_id: upper layer's cm info struct 195 * @type: Event type to indicate 196 * @status: status for the event type 197 */ 198 static int 199 irdma_send_cm_event(struct irdma_cm_node *cm_node, 200 struct iw_cm_id *cm_id, 201 enum iw_cm_event_type type, int status) 202 { 203 struct iw_cm_event event = {0}; 204 205 event.event = type; 206 event.status = status; 207 208 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 209 "cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n", 210 cm_node, cm_id, cm_node->accelerated, cm_node->state, type, 211 status); 212 213 switch (type) { 214 case IW_CM_EVENT_CONNECT_REQUEST: 215 if (cm_node->ipv4) 216 irdma_fill_sockaddr4(cm_node, &event); 217 else 218 irdma_fill_sockaddr6(cm_node, &event); 219 event.provider_data = cm_node; 220 event.private_data = cm_node->pdata_buf; 221 event.private_data_len = (u8)cm_node->pdata.size; 222 event.ird = cm_node->ird_size; 223 break; 224 case IW_CM_EVENT_CONNECT_REPLY: 225 irdma_get_cmevent_info(cm_node, cm_id, &event); 226 break; 227 case IW_CM_EVENT_ESTABLISHED: 228 event.ird = cm_node->ird_size; 229 event.ord = cm_node->ord_size; 230 break; 231 case IW_CM_EVENT_DISCONNECT: 232 case IW_CM_EVENT_CLOSE: 233 /* Wait if we are in RTS but havent issued the iwcm event upcall */ 234 if (!cm_node->accelerated) 235 wait_for_completion(&cm_node->establish_comp); 236 break; 237 default: 238 return -EINVAL; 239 } 240 241 return cm_id->event_handler(cm_id, &event); 242 } 243 244 /** 245 * irdma_timer_list_prep - add connection nodes to a list to perform timer tasks 246 * @cm_core: cm's core 247 * @timer_list: a timer list to which cm_node will be selected 248 */ 249 static void 250 irdma_timer_list_prep(struct irdma_cm_core *cm_core, 251 struct list_head *timer_list) 252 { 253 struct irdma_cm_node *cm_node; 254 int bkt; 255 256 HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { 257 if ((cm_node->close_entry || cm_node->send_entry) && 258 atomic_inc_not_zero(&cm_node->refcnt)) 259 list_add(&cm_node->timer_entry, timer_list); 260 } 261 } 262 263 /** 264 * irdma_create_event - create cm event 265 * @cm_node: connection's node 266 * @type: Event type to generate 267 */ 268 static struct irdma_cm_event * 269 irdma_create_event(struct irdma_cm_node *cm_node, 270 enum irdma_cm_event_type type) 271 { 272 struct irdma_cm_event *event; 273 274 if (!cm_node->cm_id) 275 return NULL; 276 277 event = kzalloc(sizeof(*event), GFP_ATOMIC); 278 279 if (!event) 280 return NULL; 281 282 event->type = type; 283 event->cm_node = cm_node; 284 memcpy(event->cm_info.rem_addr, cm_node->rem_addr, 285 sizeof(event->cm_info.rem_addr)); 286 memcpy(event->cm_info.loc_addr, cm_node->loc_addr, 287 sizeof(event->cm_info.loc_addr)); 288 event->cm_info.rem_port = cm_node->rem_port; 289 event->cm_info.loc_port = cm_node->loc_port; 290 event->cm_info.cm_id = cm_node->cm_id; 291 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 292 "node=%p event=%p type=%u dst=%pI4 src=%pI4\n", 293 cm_node, 294 event, type, event->cm_info.loc_addr, 295 event->cm_info.rem_addr); 296 irdma_cm_post_event(event); 297 298 return event; 299 } 300 301 /** 302 * irdma_free_retrans_entry - free send entry 303 * @cm_node: connection's node 304 */ 305 static void 306 irdma_free_retrans_entry(struct irdma_cm_node *cm_node) 307 { 308 struct irdma_device *iwdev = cm_node->iwdev; 309 struct irdma_timer_entry *send_entry; 310 311 send_entry = cm_node->send_entry; 312 if (!send_entry) 313 return; 314 315 cm_node->send_entry = NULL; 316 irdma_free_sqbuf(&iwdev->vsi, send_entry->sqbuf); 317 kfree(send_entry); 318 atomic_dec(&cm_node->refcnt); 319 } 320 321 /** 322 * irdma_cleanup_retrans_entry - free send entry with lock 323 * @cm_node: connection's node 324 */ 325 static void 326 irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node) 327 { 328 unsigned long flags; 329 330 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 331 irdma_free_retrans_entry(cm_node); 332 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); 333 } 334 335 /** 336 * irdma_form_ah_cm_frame - get a free packet and build frame with address handle 337 * @cm_node: connection's node ionfo to use in frame 338 * @options: pointer to options info 339 * @hdr: pointer mpa header 340 * @pdata: pointer to private data 341 * @flags: indicates FIN or ACK 342 */ 343 static struct irdma_puda_buf * 344 irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node, 345 struct irdma_kmem_info *options, 346 struct irdma_kmem_info *hdr, 347 struct irdma_mpa_priv_info *pdata, 348 u8 flags) 349 { 350 struct irdma_puda_buf *sqbuf; 351 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; 352 u8 *buf; 353 struct tcphdr *tcph; 354 u16 pktsize; 355 u32 opts_len = 0; 356 u32 pd_len = 0; 357 u32 hdr_len = 0; 358 359 if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) { 360 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "AH invalid\n"); 361 return NULL; 362 } 363 364 sqbuf = irdma_puda_get_bufpool(vsi->ilq); 365 if (!sqbuf) { 366 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "SQ buf NULL\n"); 367 return NULL; 368 } 369 370 sqbuf->ah_id = cm_node->ah->ah_info.ah_idx; 371 buf = sqbuf->mem.va; 372 if (options) 373 opts_len = (u32)options->size; 374 375 if (hdr) 376 hdr_len = hdr->size; 377 378 if (pdata) 379 pd_len = pdata->size; 380 381 pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len; 382 383 memset(buf, 0, pktsize); 384 385 sqbuf->totallen = pktsize; 386 sqbuf->tcphlen = sizeof(*tcph) + opts_len; 387 sqbuf->scratch = cm_node; 388 389 tcph = (struct tcphdr *)buf; 390 buf += sizeof(*tcph); 391 392 tcph->th_sport = htons(cm_node->loc_port); 393 tcph->th_dport = htons(cm_node->rem_port); 394 tcph->th_seq = htonl(cm_node->tcp_cntxt.loc_seq_num); 395 396 if (flags & SET_ACK) { 397 cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; 398 tcph->th_ack = htonl(cm_node->tcp_cntxt.loc_ack_num); 399 tcph->th_flags |= TH_ACK; 400 } else { 401 tcph->th_ack = 0; 402 } 403 404 if (flags & SET_SYN) { 405 cm_node->tcp_cntxt.loc_seq_num++; 406 tcph->th_flags |= TH_SYN; 407 } else { 408 cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len; 409 } 410 411 if (flags & SET_FIN) { 412 cm_node->tcp_cntxt.loc_seq_num++; 413 tcph->th_flags |= TH_FIN; 414 } 415 416 if (flags & SET_RST) 417 tcph->th_flags |= TH_RST; 418 419 tcph->th_off = (u16)((sizeof(*tcph) + opts_len + 3) >> 2); 420 sqbuf->tcphlen = tcph->th_off << 2; 421 tcph->th_win = htons(cm_node->tcp_cntxt.rcv_wnd); 422 tcph->th_urp = 0; 423 424 if (opts_len) { 425 memcpy(buf, options->addr, opts_len); 426 buf += opts_len; 427 } 428 429 if (hdr_len) { 430 memcpy(buf, hdr->addr, hdr_len); 431 buf += hdr_len; 432 } 433 434 if (pdata && pdata->addr) 435 memcpy(buf, pdata->addr, pdata->size); 436 437 atomic_set(&sqbuf->refcount, 1); 438 439 irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "TRANSMIT ILQ BUFFER", 440 sqbuf->mem.va, sqbuf->totallen); 441 442 return sqbuf; 443 } 444 445 /** 446 * irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet 447 * @cm_node: connection's node ionfo to use in frame 448 * @options: pointer to options info 449 * @hdr: pointer mpa header 450 * @pdata: pointer to private data 451 * @flags: indicates FIN or ACK 452 */ 453 static struct irdma_puda_buf * 454 irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node, 455 struct irdma_kmem_info *options, 456 struct irdma_kmem_info *hdr, 457 struct irdma_mpa_priv_info *pdata, 458 u8 flags) 459 { 460 struct irdma_puda_buf *sqbuf; 461 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; 462 u8 *buf; 463 464 struct tcphdr *tcph; 465 struct ip *iph; 466 struct ip6_hdr *ip6h; 467 struct ether_header *ethh; 468 u16 pktsize; 469 u16 eth_hlen = ETH_HLEN; 470 u32 opts_len = 0; 471 u32 pd_len = 0; 472 u32 hdr_len = 0; 473 474 u16 vtag; 475 476 sqbuf = irdma_puda_get_bufpool(vsi->ilq); 477 if (!sqbuf) 478 return NULL; 479 480 buf = sqbuf->mem.va; 481 482 if (options) 483 opts_len = (u32)options->size; 484 485 if (hdr) 486 hdr_len = hdr->size; 487 488 if (pdata) 489 pd_len = pdata->size; 490 491 if (cm_node->vlan_id < VLAN_N_VID) 492 eth_hlen += 4; 493 494 if (cm_node->ipv4) 495 pktsize = sizeof(*iph) + sizeof(*tcph); 496 else 497 pktsize = sizeof(*ip6h) + sizeof(*tcph); 498 pktsize += opts_len + hdr_len + pd_len; 499 500 memset(buf, 0, eth_hlen + pktsize); 501 502 sqbuf->totallen = pktsize + eth_hlen; 503 sqbuf->maclen = eth_hlen; 504 sqbuf->tcphlen = sizeof(*tcph) + opts_len; 505 sqbuf->scratch = cm_node; 506 507 ethh = (struct ether_header *)buf; 508 buf += eth_hlen; 509 510 if (cm_node->do_lpb) 511 sqbuf->do_lpb = true; 512 513 if (cm_node->ipv4) { 514 sqbuf->ipv4 = true; 515 516 iph = (struct ip *)buf; 517 buf += sizeof(*iph); 518 tcph = (struct tcphdr *)buf; 519 buf += sizeof(*tcph); 520 521 ether_addr_copy(ethh->ether_dhost, cm_node->rem_mac); 522 ether_addr_copy(ethh->ether_shost, cm_node->loc_mac); 523 if (cm_node->vlan_id < VLAN_N_VID) { 524 ((struct ether_vlan_header *)ethh)->evl_proto = 525 htons(ETH_P_8021Q); 526 vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | 527 cm_node->vlan_id; 528 ((struct ether_vlan_header *)ethh)->evl_tag = htons(vtag); 529 530 ((struct ether_vlan_header *)ethh)->evl_encap_proto = 531 htons(ETH_P_IP); 532 } else { 533 ethh->ether_type = htons(ETH_P_IP); 534 } 535 536 iph->ip_v = IPVERSION; 537 iph->ip_hl = 5; /* 5 * 4Byte words, IP headr len */ 538 iph->ip_tos = cm_node->tos; 539 iph->ip_len = htons(pktsize); 540 iph->ip_id = htons(++cm_node->tcp_cntxt.loc_id); 541 542 iph->ip_off = htons(0x4000); 543 iph->ip_ttl = 0x40; 544 iph->ip_p = IPPROTO_TCP; 545 iph->ip_src.s_addr = htonl(cm_node->loc_addr[0]); 546 iph->ip_dst.s_addr = htonl(cm_node->rem_addr[0]); 547 } else { 548 sqbuf->ipv4 = false; 549 ip6h = (struct ip6_hdr *)buf; 550 buf += sizeof(*ip6h); 551 tcph = (struct tcphdr *)buf; 552 buf += sizeof(*tcph); 553 554 ether_addr_copy(ethh->ether_dhost, cm_node->rem_mac); 555 ether_addr_copy(ethh->ether_shost, cm_node->loc_mac); 556 if (cm_node->vlan_id < VLAN_N_VID) { 557 ((struct ether_vlan_header *)ethh)->evl_proto = 558 htons(ETH_P_8021Q); 559 vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) | 560 cm_node->vlan_id; 561 ((struct ether_vlan_header *)ethh)->evl_tag = htons(vtag); 562 ((struct ether_vlan_header *)ethh)->evl_encap_proto = 563 htons(ETH_P_IPV6); 564 } else { 565 ethh->ether_type = htons(ETH_P_IPV6); 566 } 567 ip6h->ip6_vfc = 6 << 4; 568 ip6h->ip6_vfc |= cm_node->tos >> 4; 569 ip6h->ip6_flow = cm_node->tos << 20; 570 ip6h->ip6_plen = htons(pktsize - sizeof(*ip6h)); 571 ip6h->ip6_nxt = 6; 572 ip6h->ip6_hops = 128; 573 irdma_copy_ip_htonl(ip6h->ip6_src.__u6_addr.__u6_addr32, 574 cm_node->loc_addr); 575 irdma_copy_ip_htonl(ip6h->ip6_dst.__u6_addr.__u6_addr32, 576 cm_node->rem_addr); 577 } 578 579 tcph->th_sport = htons(cm_node->loc_port); 580 tcph->th_dport = htons(cm_node->rem_port); 581 tcph->th_seq = htonl(cm_node->tcp_cntxt.loc_seq_num); 582 583 if (flags & SET_ACK) { 584 cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; 585 tcph->th_ack = htonl(cm_node->tcp_cntxt.loc_ack_num); 586 tcph->th_flags |= TH_ACK; 587 } else { 588 tcph->th_ack = 0; 589 } 590 591 if (flags & SET_SYN) { 592 cm_node->tcp_cntxt.loc_seq_num++; 593 tcph->th_flags |= TH_SYN; 594 } else { 595 cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len; 596 } 597 598 if (flags & SET_FIN) { 599 cm_node->tcp_cntxt.loc_seq_num++; 600 tcph->th_flags |= TH_FIN; 601 } 602 603 if (flags & SET_RST) 604 tcph->th_flags |= TH_RST; 605 606 tcph->th_off = (u16)((sizeof(*tcph) + opts_len + 3) >> 2); 607 sqbuf->tcphlen = tcph->th_off << 2; 608 tcph->th_win = htons(cm_node->tcp_cntxt.rcv_wnd); 609 tcph->th_urp = 0; 610 611 if (opts_len) { 612 memcpy(buf, options->addr, opts_len); 613 buf += opts_len; 614 } 615 616 if (hdr_len) { 617 memcpy(buf, hdr->addr, hdr_len); 618 buf += hdr_len; 619 } 620 621 if (pdata && pdata->addr) 622 memcpy(buf, pdata->addr, pdata->size); 623 624 atomic_set(&sqbuf->refcount, 1); 625 626 irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "TRANSMIT ILQ BUFFER", 627 sqbuf->mem.va, sqbuf->totallen); 628 629 return sqbuf; 630 } 631 632 /** 633 * irdma_send_reset - Send RST packet 634 * @cm_node: connection's node 635 */ 636 int 637 irdma_send_reset(struct irdma_cm_node *cm_node) 638 { 639 struct irdma_puda_buf *sqbuf; 640 int flags = SET_RST | SET_ACK; 641 642 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, 643 flags); 644 if (!sqbuf) 645 return -ENOMEM; 646 647 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 648 "caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n", 649 __builtin_return_address(0), cm_node, cm_node->cm_id, 650 cm_node->accelerated, cm_node->state, cm_node->rem_port, 651 cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr); 652 653 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0, 654 1); 655 } 656 657 /** 658 * irdma_active_open_err - send event for active side cm error 659 * @cm_node: connection's node 660 * @reset: Flag to send reset or not 661 */ 662 static void 663 irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset) 664 { 665 irdma_cleanup_retrans_entry(cm_node); 666 cm_node->cm_core->stats_connect_errs++; 667 if (reset) { 668 irdma_debug(&cm_node->iwdev->rf->sc_dev, 669 IRDMA_DEBUG_CM, "cm_node=%p state=%d\n", cm_node, 670 cm_node->state); 671 atomic_inc(&cm_node->refcnt); 672 irdma_send_reset(cm_node); 673 } 674 675 cm_node->state = IRDMA_CM_STATE_CLOSED; 676 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); 677 } 678 679 /** 680 * irdma_passive_open_err - handle passive side cm error 681 * @cm_node: connection's node 682 * @reset: send reset or just free cm_node 683 */ 684 static void 685 irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset) 686 { 687 irdma_cleanup_retrans_entry(cm_node); 688 cm_node->cm_core->stats_passive_errs++; 689 cm_node->state = IRDMA_CM_STATE_CLOSED; 690 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 691 "cm_node=%p state =%d\n", 692 cm_node, cm_node->state); 693 if (reset) 694 irdma_send_reset(cm_node); 695 else 696 irdma_rem_ref_cm_node(cm_node); 697 } 698 699 /** 700 * irdma_event_connect_error - to create connect error event 701 * @event: cm information for connect event 702 */ 703 static void 704 irdma_event_connect_error(struct irdma_cm_event *event) 705 { 706 struct irdma_qp *iwqp; 707 struct iw_cm_id *cm_id; 708 709 cm_id = event->cm_node->cm_id; 710 if (!cm_id) 711 return; 712 713 iwqp = cm_id->provider_data; 714 715 if (!iwqp || !iwqp->iwdev) 716 return; 717 718 iwqp->cm_id = NULL; 719 cm_id->provider_data = NULL; 720 irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 721 -ECONNRESET); 722 irdma_rem_ref_cm_node(event->cm_node); 723 } 724 725 /** 726 * irdma_process_options - process options from TCP header 727 * @cm_node: connection's node 728 * @optionsloc: point to start of options 729 * @optionsize: size of all options 730 * @syn_pkt: flag if syn packet 731 */ 732 static int 733 irdma_process_options(struct irdma_cm_node *cm_node, u8 *optionsloc, 734 u32 optionsize, u32 syn_pkt) 735 { 736 u32 tmp; 737 u32 offset = 0; 738 union all_known_options *all_options; 739 char got_mss_option = 0; 740 741 while (offset < optionsize) { 742 all_options = (union all_known_options *)(optionsloc + offset); 743 switch (all_options->base.optionnum) { 744 case OPTION_NUM_EOL: 745 offset = optionsize; 746 break; 747 case OPTION_NUM_NONE: 748 offset += 1; 749 continue; 750 case OPTION_NUM_MSS: 751 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 752 "MSS Length: %d Offset: %d Size: %d\n", 753 all_options->mss.len, offset, optionsize); 754 got_mss_option = 1; 755 if (all_options->mss.len != 4) 756 return -EINVAL; 757 tmp = ntohs(all_options->mss.mss); 758 if ((cm_node->ipv4 && 759 (tmp + IRDMA_MTU_TO_MSS_IPV4) < IRDMA_MIN_MTU_IPV4) || 760 (!cm_node->ipv4 && 761 (tmp + IRDMA_MTU_TO_MSS_IPV6) < IRDMA_MIN_MTU_IPV6)) 762 return -EINVAL; 763 if (tmp < cm_node->tcp_cntxt.mss) 764 cm_node->tcp_cntxt.mss = tmp; 765 break; 766 case OPTION_NUM_WINDOW_SCALE: 767 cm_node->tcp_cntxt.snd_wscale = 768 all_options->windowscale.shiftcount; 769 break; 770 default: 771 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 772 "Unsupported TCP Option: %x\n", 773 all_options->base.optionnum); 774 break; 775 } 776 offset += all_options->base.len; 777 } 778 if (!got_mss_option && syn_pkt) 779 cm_node->tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS; 780 781 return 0; 782 } 783 784 /** 785 * irdma_handle_tcp_options - setup TCP context info after parsing TCP options 786 * @cm_node: connection's node 787 * @tcph: pointer tcp header 788 * @optionsize: size of options rcvd 789 * @passive: active or passive flag 790 */ 791 static int 792 irdma_handle_tcp_options(struct irdma_cm_node *cm_node, 793 struct tcphdr *tcph, int optionsize, 794 int passive) 795 { 796 u8 *optionsloc = (u8 *)&tcph[1]; 797 int ret; 798 799 if (optionsize) { 800 ret = irdma_process_options(cm_node, optionsloc, optionsize, 801 (u32)tcph->th_flags & TH_SYN); 802 if (ret) { 803 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 804 "Node %p, Sending Reset\n", 805 cm_node); 806 if (passive) 807 irdma_passive_open_err(cm_node, true); 808 else 809 irdma_active_open_err(cm_node, true); 810 return ret; 811 } 812 } 813 814 cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->th_win) 815 << cm_node->tcp_cntxt.snd_wscale; 816 817 if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd) 818 cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd; 819 820 return 0; 821 } 822 823 /** 824 * irdma_build_mpa_v1 - build a MPA V1 frame 825 * @cm_node: connection's node 826 * @start_addr: address where to build frame 827 * @mpa_key: to do read0 or write0 828 */ 829 static void 830 irdma_build_mpa_v1(struct irdma_cm_node *cm_node, void *start_addr, 831 u8 mpa_key) 832 { 833 struct ietf_mpa_v1 *mpa_frame = start_addr; 834 835 switch (mpa_key) { 836 case MPA_KEY_REQUEST: 837 memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE); 838 break; 839 case MPA_KEY_REPLY: 840 memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE); 841 break; 842 default: 843 break; 844 } 845 mpa_frame->flags = IETF_MPA_FLAGS_CRC; 846 mpa_frame->rev = cm_node->mpa_frame_rev; 847 mpa_frame->priv_data_len = htons(cm_node->pdata.size); 848 } 849 850 /** 851 * irdma_build_mpa_v2 - build a MPA V2 frame 852 * @cm_node: connection's node 853 * @start_addr: buffer start address 854 * @mpa_key: to do read0 or write0 855 */ 856 static void 857 irdma_build_mpa_v2(struct irdma_cm_node *cm_node, void *start_addr, 858 u8 mpa_key) 859 { 860 struct ietf_mpa_v2 *mpa_frame = start_addr; 861 struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg; 862 u16 ctrl_ird, ctrl_ord; 863 864 /* initialize the upper 5 bytes of the frame */ 865 irdma_build_mpa_v1(cm_node, start_addr, mpa_key); 866 mpa_frame->flags |= IETF_MPA_V2_FLAG; 867 if (cm_node->iwdev->iw_ooo) { 868 mpa_frame->flags |= IETF_MPA_FLAGS_MARKERS; 869 cm_node->rcv_mark_en = true; 870 } 871 mpa_frame->priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame->priv_data_len) + 872 IETF_RTR_MSG_SIZE); 873 874 /* initialize RTR msg */ 875 if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { 876 ctrl_ird = IETF_NO_IRD_ORD; 877 ctrl_ord = IETF_NO_IRD_ORD; 878 } else { 879 ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ? 880 IETF_NO_IRD_ORD : 881 cm_node->ird_size; 882 ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ? 883 IETF_NO_IRD_ORD : 884 cm_node->ord_size; 885 } 886 ctrl_ird |= IETF_PEER_TO_PEER; 887 888 switch (mpa_key) { 889 case MPA_KEY_REQUEST: 890 ctrl_ord |= IETF_RDMA0_WRITE; 891 ctrl_ord |= IETF_RDMA0_READ; 892 break; 893 case MPA_KEY_REPLY: 894 switch (cm_node->send_rdma0_op) { 895 case SEND_RDMA_WRITE_ZERO: 896 ctrl_ord |= IETF_RDMA0_WRITE; 897 break; 898 case SEND_RDMA_READ_ZERO: 899 ctrl_ord |= IETF_RDMA0_READ; 900 break; 901 } 902 break; 903 default: 904 break; 905 } 906 rtr_msg->ctrl_ird = htons(ctrl_ird); 907 rtr_msg->ctrl_ord = htons(ctrl_ord); 908 } 909 910 /** 911 * irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2 912 * @cm_node: connection's node 913 * @mpa: mpa: data buffer 914 * @mpa_key: to do read0 or write0 915 */ 916 static int 917 irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node, 918 struct irdma_kmem_info *mpa, u8 mpa_key) 919 { 920 int hdr_len = 0; 921 922 switch (cm_node->mpa_frame_rev) { 923 case IETF_MPA_V1: 924 hdr_len = sizeof(struct ietf_mpa_v1); 925 irdma_build_mpa_v1(cm_node, mpa->addr, mpa_key); 926 break; 927 case IETF_MPA_V2: 928 hdr_len = sizeof(struct ietf_mpa_v2); 929 irdma_build_mpa_v2(cm_node, mpa->addr, mpa_key); 930 break; 931 default: 932 break; 933 } 934 935 return hdr_len; 936 } 937 938 /** 939 * irdma_send_mpa_request - active node send mpa request to passive node 940 * @cm_node: connection's node 941 */ 942 static int 943 irdma_send_mpa_request(struct irdma_cm_node *cm_node) 944 { 945 struct irdma_puda_buf *sqbuf; 946 947 cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame; 948 cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node, 949 &cm_node->mpa_hdr, 950 MPA_KEY_REQUEST); 951 if (!cm_node->mpa_hdr.size) { 952 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 953 "mpa size = %d\n", 954 cm_node->mpa_hdr.size); 955 return -EINVAL; 956 } 957 958 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, 959 &cm_node->mpa_hdr, 960 &cm_node->pdata, SET_ACK); 961 if (!sqbuf) 962 return -ENOMEM; 963 964 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 965 0); 966 } 967 968 /** 969 * irdma_send_mpa_reject - 970 * @cm_node: connection's node 971 * @pdata: reject data for connection 972 * @plen: length of reject data 973 */ 974 static int 975 irdma_send_mpa_reject(struct irdma_cm_node *cm_node, 976 const void *pdata, u8 plen) 977 { 978 struct irdma_puda_buf *sqbuf; 979 struct irdma_mpa_priv_info priv_info; 980 981 cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame; 982 cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node, 983 &cm_node->mpa_hdr, 984 MPA_KEY_REPLY); 985 986 cm_node->mpa_v2_frame.flags |= IETF_MPA_FLAGS_REJECT; 987 priv_info.addr = pdata; 988 priv_info.size = plen; 989 990 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, 991 &cm_node->mpa_hdr, &priv_info, 992 SET_ACK | SET_FIN); 993 if (!sqbuf) 994 return -ENOMEM; 995 996 cm_node->state = IRDMA_CM_STATE_FIN_WAIT1; 997 998 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 999 0); 1000 } 1001 1002 /** 1003 * irdma_negotiate_mpa_v2_ird_ord - negotiate MPAv2 IRD/ORD 1004 * @cm_node: connection's node 1005 * @buf: Data pointer 1006 */ 1007 static int 1008 irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node, 1009 u8 *buf) 1010 { 1011 struct ietf_mpa_v2 *mpa_v2_frame; 1012 struct ietf_rtr_msg *rtr_msg; 1013 u16 ird_size; 1014 u16 ord_size; 1015 u16 ctrl_ord; 1016 u16 ctrl_ird; 1017 1018 mpa_v2_frame = (struct ietf_mpa_v2 *)buf; 1019 rtr_msg = &mpa_v2_frame->rtr_msg; 1020 1021 /* parse rtr message */ 1022 ctrl_ord = ntohs(rtr_msg->ctrl_ord); 1023 ctrl_ird = ntohs(rtr_msg->ctrl_ird); 1024 ird_size = ctrl_ird & IETF_NO_IRD_ORD; 1025 ord_size = ctrl_ord & IETF_NO_IRD_ORD; 1026 1027 if (!(ctrl_ird & IETF_PEER_TO_PEER)) 1028 return -EOPNOTSUPP; 1029 1030 if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) { 1031 cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD; 1032 goto negotiate_done; 1033 } 1034 1035 if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) { 1036 /* responder */ 1037 if (!ord_size && (ctrl_ord & IETF_RDMA0_READ)) 1038 cm_node->ird_size = 1; 1039 if (cm_node->ord_size > ird_size) 1040 cm_node->ord_size = ird_size; 1041 } else { 1042 /* initiator */ 1043 if (!ird_size && (ctrl_ord & IETF_RDMA0_READ)) 1044 /* Remote peer doesn't support RDMA0_READ */ 1045 return -EOPNOTSUPP; 1046 1047 if (cm_node->ord_size > ird_size) 1048 cm_node->ord_size = ird_size; 1049 1050 if (cm_node->ird_size < ord_size) 1051 /* no resources available */ 1052 return -EINVAL; 1053 } 1054 1055 negotiate_done: 1056 if (ctrl_ord & IETF_RDMA0_READ) 1057 cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; 1058 else if (ctrl_ord & IETF_RDMA0_WRITE) 1059 cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO; 1060 else 1061 /* Not supported RDMA0 operation */ 1062 return -EOPNOTSUPP; 1063 1064 irdma_debug(&cm_node->iwdev->rf->sc_dev, 1065 IRDMA_DEBUG_CM, "MPAV2 Negotiated ORD: %d, IRD: %d\n", 1066 cm_node->ord_size, cm_node->ird_size); 1067 return 0; 1068 } 1069 1070 /** 1071 * irdma_parse_mpa - process an IETF MPA frame 1072 * @cm_node: connection's node 1073 * @buf: Data pointer 1074 * @type: to return accept or reject 1075 * @len: Len of mpa buffer 1076 */ 1077 static int 1078 irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type, 1079 u32 len) 1080 { 1081 struct ietf_mpa_v1 *mpa_frame; 1082 int mpa_hdr_len, priv_data_len, ret; 1083 1084 *type = IRDMA_MPA_REQUEST_ACCEPT; 1085 1086 if (len < sizeof(struct ietf_mpa_v1)) { 1087 irdma_debug(&cm_node->iwdev->rf->sc_dev, 1088 IRDMA_DEBUG_CM, "ietf buffer small (%x)\n", len); 1089 return -EINVAL; 1090 } 1091 1092 mpa_frame = (struct ietf_mpa_v1 *)buf; 1093 mpa_hdr_len = sizeof(struct ietf_mpa_v1); 1094 priv_data_len = ntohs(mpa_frame->priv_data_len); 1095 1096 if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { 1097 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1098 "private_data too big %d\n", 1099 priv_data_len); 1100 return -EOVERFLOW; 1101 } 1102 1103 if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) { 1104 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1105 "unsupported mpa rev = %d\n", 1106 mpa_frame->rev); 1107 return -EINVAL; 1108 } 1109 1110 if (mpa_frame->rev > cm_node->mpa_frame_rev) { 1111 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1112 "rev %d\n", 1113 mpa_frame->rev); 1114 return -EINVAL; 1115 } 1116 1117 cm_node->mpa_frame_rev = mpa_frame->rev; 1118 if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) { 1119 if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, 1120 IETF_MPA_KEY_SIZE)) { 1121 irdma_debug(&cm_node->iwdev->rf->sc_dev, 1122 IRDMA_DEBUG_CM, "Unexpected MPA Key received\n"); 1123 return -EINVAL; 1124 } 1125 } else { 1126 if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, 1127 IETF_MPA_KEY_SIZE)) { 1128 irdma_debug(&cm_node->iwdev->rf->sc_dev, 1129 IRDMA_DEBUG_CM, "Unexpected MPA Key received\n"); 1130 return -EINVAL; 1131 } 1132 } 1133 1134 if (priv_data_len + mpa_hdr_len > len) { 1135 irdma_debug(&cm_node->iwdev->rf->sc_dev, 1136 IRDMA_DEBUG_CM, "ietf buffer len(%x + %x != %x)\n", 1137 priv_data_len, mpa_hdr_len, len); 1138 return -EOVERFLOW; 1139 } 1140 1141 if (len > IRDMA_MAX_CM_BUF) { 1142 irdma_debug(&cm_node->iwdev->rf->sc_dev, 1143 IRDMA_DEBUG_CM, "ietf buffer large len = %d\n", len); 1144 return -EOVERFLOW; 1145 } 1146 1147 switch (mpa_frame->rev) { 1148 case IETF_MPA_V2: 1149 mpa_hdr_len += IETF_RTR_MSG_SIZE; 1150 ret = irdma_negotiate_mpa_v2_ird_ord(cm_node, buf); 1151 if (ret) 1152 return ret; 1153 break; 1154 case IETF_MPA_V1: 1155 default: 1156 break; 1157 } 1158 1159 memcpy(cm_node->pdata_buf, buf + mpa_hdr_len, priv_data_len); 1160 cm_node->pdata.size = priv_data_len; 1161 1162 if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT) 1163 *type = IRDMA_MPA_REQUEST_REJECT; 1164 1165 if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS) 1166 cm_node->snd_mark_en = true; 1167 1168 return 0; 1169 } 1170 1171 /** 1172 * irdma_schedule_cm_timer 1173 * @cm_node: connection's node 1174 * @sqbuf: buffer to send 1175 * @type: if it is send or close 1176 * @send_retrans: if rexmits to be done 1177 * @close_when_complete: is cm_node to be removed 1178 * 1179 * note - cm_node needs to be protected before calling this. Encase in: 1180 * irdma_rem_ref_cm_node(cm_core, cm_node); 1181 * irdma_schedule_cm_timer(...) 1182 * atomic_inc(&cm_node->refcnt); 1183 */ 1184 int 1185 irdma_schedule_cm_timer(struct irdma_cm_node *cm_node, 1186 struct irdma_puda_buf *sqbuf, 1187 enum irdma_timer_type type, int send_retrans, 1188 int close_when_complete) 1189 { 1190 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; 1191 struct irdma_cm_core *cm_core = cm_node->cm_core; 1192 struct irdma_timer_entry *new_send; 1193 u32 was_timer_set; 1194 unsigned long flags; 1195 1196 new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC); 1197 if (!new_send) { 1198 if (type != IRDMA_TIMER_TYPE_CLOSE) 1199 irdma_free_sqbuf(vsi, sqbuf); 1200 return -ENOMEM; 1201 } 1202 1203 new_send->retrycount = IRDMA_DEFAULT_RETRYS; 1204 new_send->retranscount = IRDMA_DEFAULT_RETRANS; 1205 new_send->sqbuf = sqbuf; 1206 new_send->timetosend = jiffies; 1207 new_send->type = type; 1208 new_send->send_retrans = send_retrans; 1209 new_send->close_when_complete = close_when_complete; 1210 1211 if (type == IRDMA_TIMER_TYPE_CLOSE) { 1212 new_send->timetosend += (HZ / 10); 1213 if (cm_node->close_entry) { 1214 kfree(new_send); 1215 irdma_debug(&cm_node->iwdev->rf->sc_dev, 1216 IRDMA_DEBUG_CM, "already close entry\n"); 1217 return -EINVAL; 1218 } 1219 1220 cm_node->close_entry = new_send; 1221 } else { /* type == IRDMA_TIMER_TYPE_SEND */ 1222 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 1223 cm_node->send_entry = new_send; 1224 atomic_inc(&cm_node->refcnt); 1225 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); 1226 new_send->timetosend = jiffies + IRDMA_RETRY_TIMEOUT; 1227 1228 atomic_inc(&sqbuf->refcount); 1229 irdma_puda_send_buf(vsi->ilq, sqbuf); 1230 if (!send_retrans) { 1231 irdma_cleanup_retrans_entry(cm_node); 1232 if (close_when_complete) 1233 irdma_rem_ref_cm_node(cm_node); 1234 return 0; 1235 } 1236 } 1237 1238 spin_lock_irqsave(&cm_core->ht_lock, flags); 1239 was_timer_set = timer_pending(&cm_core->tcp_timer); 1240 1241 if (!was_timer_set) { 1242 cm_core->tcp_timer.expires = new_send->timetosend; 1243 add_timer(&cm_core->tcp_timer); 1244 } 1245 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 1246 1247 return 0; 1248 } 1249 1250 /** 1251 * irdma_retrans_expired - Could not rexmit the packet 1252 * @cm_node: connection's node 1253 */ 1254 static void 1255 irdma_retrans_expired(struct irdma_cm_node *cm_node) 1256 { 1257 enum irdma_cm_node_state state = cm_node->state; 1258 1259 cm_node->state = IRDMA_CM_STATE_CLOSED; 1260 switch (state) { 1261 case IRDMA_CM_STATE_SYN_RCVD: 1262 case IRDMA_CM_STATE_CLOSING: 1263 irdma_rem_ref_cm_node(cm_node); 1264 break; 1265 case IRDMA_CM_STATE_FIN_WAIT1: 1266 case IRDMA_CM_STATE_LAST_ACK: 1267 irdma_send_reset(cm_node); 1268 break; 1269 default: 1270 atomic_inc(&cm_node->refcnt); 1271 irdma_send_reset(cm_node); 1272 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); 1273 break; 1274 } 1275 } 1276 1277 /** 1278 * irdma_handle_close_entry - for handling retry/timeouts 1279 * @cm_node: connection's node 1280 * @rem_node: flag for remove cm_node 1281 */ 1282 static void 1283 irdma_handle_close_entry(struct irdma_cm_node *cm_node, 1284 u32 rem_node) 1285 { 1286 struct irdma_timer_entry *close_entry = cm_node->close_entry; 1287 struct irdma_qp *iwqp; 1288 unsigned long flags; 1289 1290 if (!close_entry) 1291 return; 1292 iwqp = (struct irdma_qp *)close_entry->sqbuf; 1293 if (iwqp) { 1294 spin_lock_irqsave(&iwqp->lock, flags); 1295 if (iwqp->cm_id) { 1296 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1297 iwqp->hw_iwarp_state = IRDMA_QP_STATE_ERROR; 1298 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1299 iwqp->ibqp_state = IB_QPS_ERR; 1300 spin_unlock_irqrestore(&iwqp->lock, flags); 1301 irdma_cm_disconn(iwqp); 1302 } else { 1303 spin_unlock_irqrestore(&iwqp->lock, flags); 1304 } 1305 } else if (rem_node) { 1306 /* TIME_WAIT state */ 1307 irdma_rem_ref_cm_node(cm_node); 1308 } 1309 1310 kfree(close_entry); 1311 cm_node->close_entry = NULL; 1312 } 1313 1314 /** 1315 * irdma_cm_timer_tick - system's timer expired callback 1316 * @t: Pointer to timer_list 1317 */ 1318 static void 1319 irdma_cm_timer_tick(struct timer_list *t) 1320 { 1321 unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME; 1322 struct irdma_cm_node *cm_node; 1323 struct irdma_timer_entry *send_entry, *close_entry; 1324 struct list_head *list_core_temp; 1325 struct list_head *list_node; 1326 struct irdma_cm_core *cm_core = from_timer(cm_core, t, tcp_timer); 1327 struct irdma_sc_vsi *vsi; 1328 u32 settimer = 0; 1329 unsigned long timetosend; 1330 unsigned long flags; 1331 struct list_head timer_list; 1332 1333 INIT_LIST_HEAD(&timer_list); 1334 1335 rcu_read_lock(); 1336 irdma_timer_list_prep(cm_core, &timer_list); 1337 rcu_read_unlock(); 1338 1339 list_for_each_safe(list_node, list_core_temp, &timer_list) { 1340 cm_node = container_of(list_node, struct irdma_cm_node, 1341 timer_entry); 1342 close_entry = cm_node->close_entry; 1343 1344 if (close_entry) { 1345 if (time_after(close_entry->timetosend, jiffies)) { 1346 if (nexttimeout > close_entry->timetosend || 1347 !settimer) { 1348 nexttimeout = close_entry->timetosend; 1349 settimer = 1; 1350 } 1351 } else { 1352 irdma_handle_close_entry(cm_node, 1); 1353 } 1354 } 1355 1356 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 1357 1358 send_entry = cm_node->send_entry; 1359 if (!send_entry) 1360 goto done; 1361 if (time_after(send_entry->timetosend, jiffies)) { 1362 if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) { 1363 if (nexttimeout > send_entry->timetosend || 1364 !settimer) { 1365 nexttimeout = send_entry->timetosend; 1366 settimer = 1; 1367 } 1368 } else { 1369 irdma_free_retrans_entry(cm_node); 1370 } 1371 goto done; 1372 } 1373 1374 if (cm_node->state == IRDMA_CM_STATE_OFFLOADED || 1375 cm_node->state == IRDMA_CM_STATE_CLOSED) { 1376 irdma_free_retrans_entry(cm_node); 1377 goto done; 1378 } 1379 1380 if (!send_entry->retranscount || !send_entry->retrycount) { 1381 irdma_free_retrans_entry(cm_node); 1382 1383 spin_unlock_irqrestore(&cm_node->retrans_list_lock, 1384 flags); 1385 irdma_retrans_expired(cm_node); 1386 cm_node->state = IRDMA_CM_STATE_CLOSED; 1387 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 1388 goto done; 1389 } 1390 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); 1391 1392 vsi = &cm_node->iwdev->vsi; 1393 if (!cm_node->ack_rcvd) { 1394 atomic_inc(&send_entry->sqbuf->refcount); 1395 irdma_puda_send_buf(vsi->ilq, send_entry->sqbuf); 1396 cm_node->cm_core->stats_pkt_retrans++; 1397 } 1398 1399 spin_lock_irqsave(&cm_node->retrans_list_lock, flags); 1400 if (send_entry->send_retrans) { 1401 send_entry->retranscount--; 1402 timetosend = (IRDMA_RETRY_TIMEOUT << 1403 (IRDMA_DEFAULT_RETRANS - 1404 send_entry->retranscount)); 1405 1406 send_entry->timetosend = jiffies + 1407 min(timetosend, IRDMA_MAX_TIMEOUT); 1408 if (nexttimeout > send_entry->timetosend || !settimer) { 1409 nexttimeout = send_entry->timetosend; 1410 settimer = 1; 1411 } 1412 } else { 1413 int close_when_complete; 1414 1415 close_when_complete = send_entry->close_when_complete; 1416 irdma_free_retrans_entry(cm_node); 1417 if (close_when_complete) 1418 irdma_rem_ref_cm_node(cm_node); 1419 } 1420 done: 1421 spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); 1422 irdma_rem_ref_cm_node(cm_node); 1423 } 1424 1425 if (settimer) { 1426 spin_lock_irqsave(&cm_core->ht_lock, flags); 1427 if (!timer_pending(&cm_core->tcp_timer)) { 1428 cm_core->tcp_timer.expires = nexttimeout; 1429 add_timer(&cm_core->tcp_timer); 1430 } 1431 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 1432 } 1433 } 1434 1435 /** 1436 * irdma_send_syn - send SYN packet 1437 * @cm_node: connection's node 1438 * @sendack: flag to set ACK bit or not 1439 */ 1440 int 1441 irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack) 1442 { 1443 struct irdma_puda_buf *sqbuf; 1444 int flags = SET_SYN; 1445 char optionsbuf[sizeof(struct option_mss) + 1446 sizeof(struct option_windowscale) + 1447 sizeof(struct option_base) + TCP_OPTIONS_PADDING]; 1448 struct irdma_kmem_info opts; 1449 int optionssize = 0; 1450 /* Sending MSS option */ 1451 union all_known_options *options; 1452 1453 opts.addr = optionsbuf; 1454 if (!cm_node) 1455 return -EINVAL; 1456 1457 options = (union all_known_options *)&optionsbuf[optionssize]; 1458 options->mss.optionnum = OPTION_NUM_MSS; 1459 options->mss.len = sizeof(struct option_mss); 1460 options->mss.mss = htons(cm_node->tcp_cntxt.mss); 1461 optionssize += sizeof(struct option_mss); 1462 1463 options = (union all_known_options *)&optionsbuf[optionssize]; 1464 options->windowscale.optionnum = OPTION_NUM_WINDOW_SCALE; 1465 options->windowscale.len = sizeof(struct option_windowscale); 1466 options->windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale; 1467 optionssize += sizeof(struct option_windowscale); 1468 options = (union all_known_options *)&optionsbuf[optionssize]; 1469 options->eol = OPTION_NUM_EOL; 1470 optionssize += 1; 1471 1472 if (sendack) 1473 flags |= SET_ACK; 1474 1475 opts.size = optionssize; 1476 1477 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, &opts, NULL, NULL, 1478 flags); 1479 if (!sqbuf) 1480 return -ENOMEM; 1481 1482 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 1483 0); 1484 } 1485 1486 /** 1487 * irdma_send_ack - Send ACK packet 1488 * @cm_node: connection's node 1489 */ 1490 void 1491 irdma_send_ack(struct irdma_cm_node *cm_node) 1492 { 1493 struct irdma_puda_buf *sqbuf; 1494 struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi; 1495 1496 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, 1497 SET_ACK); 1498 if (sqbuf) 1499 irdma_puda_send_buf(vsi->ilq, sqbuf); 1500 } 1501 1502 /** 1503 * irdma_send_fin - Send FIN pkt 1504 * @cm_node: connection's node 1505 */ 1506 static int 1507 irdma_send_fin(struct irdma_cm_node *cm_node) 1508 { 1509 struct irdma_puda_buf *sqbuf; 1510 1511 sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL, 1512 SET_ACK | SET_FIN); 1513 if (!sqbuf) 1514 return -ENOMEM; 1515 1516 return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1, 1517 0); 1518 } 1519 1520 /** 1521 * irdma_find_listener - find a cm node listening on this addr-port pair 1522 * @cm_core: cm's core 1523 * @dst_addr: listener ip addr 1524 * @ipv4: flag indicating IPv4 when true 1525 * @dst_port: listener tcp port num 1526 * @vlan_id: virtual LAN ID 1527 * @listener_state: state to match with listen node's 1528 */ 1529 static struct irdma_cm_listener * 1530 irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, bool ipv4, u16 dst_port, 1531 u16 vlan_id, enum irdma_cm_listener_state listener_state) 1532 { 1533 struct irdma_cm_listener *listen_node; 1534 static const u32 ip_zero[4] = {0, 0, 0, 0}; 1535 u32 listen_addr[4]; 1536 u16 listen_port; 1537 unsigned long flags; 1538 1539 /* walk list and find cm_node associated with this session ID */ 1540 spin_lock_irqsave(&cm_core->listen_list_lock, flags); 1541 list_for_each_entry(listen_node, &cm_core->listen_list, list) { 1542 memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr)); 1543 listen_port = listen_node->loc_port; 1544 if (listen_node->ipv4 != ipv4 || listen_port != dst_port || 1545 !(listener_state & listen_node->listener_state)) 1546 continue; 1547 /* compare node pair, return node handle if a match */ 1548 if (!memcmp(listen_addr, ip_zero, sizeof(listen_addr)) || 1549 (!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) && 1550 vlan_id == listen_node->vlan_id)) { 1551 atomic_inc(&listen_node->refcnt); 1552 spin_unlock_irqrestore(&cm_core->listen_list_lock, 1553 flags); 1554 return listen_node; 1555 } 1556 } 1557 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); 1558 1559 return NULL; 1560 } 1561 1562 /** 1563 * irdma_del_multiple_qhash - Remove qhash and child listens 1564 * @iwdev: iWarp device 1565 * @cm_info: CM info for parent listen node 1566 * @cm_parent_listen_node: The parent listen node 1567 */ 1568 static int 1569 irdma_del_multiple_qhash(struct irdma_device *iwdev, 1570 struct irdma_cm_info *cm_info, 1571 struct irdma_cm_listener *cm_parent_listen_node) 1572 { 1573 struct irdma_cm_listener *child_listen_node; 1574 struct list_head *pos, *tpos; 1575 unsigned long flags; 1576 int ret = -EINVAL; 1577 1578 spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); 1579 list_for_each_safe(pos, tpos, 1580 &cm_parent_listen_node->child_listen_list) { 1581 child_listen_node = list_entry(pos, struct irdma_cm_listener, 1582 child_listen_list); 1583 if (child_listen_node->ipv4) 1584 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1585 "removing child listen for IP=%pI4, port=%d, vlan=%d\n", 1586 child_listen_node->loc_addr, 1587 child_listen_node->loc_port, 1588 child_listen_node->vlan_id); 1589 else 1590 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1591 "removing child listen for IP=%pI6, port=%d, vlan=%d\n", 1592 child_listen_node->loc_addr, 1593 child_listen_node->loc_port, 1594 child_listen_node->vlan_id); 1595 list_del(pos); 1596 memcpy(cm_info->loc_addr, child_listen_node->loc_addr, 1597 sizeof(cm_info->loc_addr)); 1598 cm_info->vlan_id = child_listen_node->vlan_id; 1599 if (child_listen_node->qhash_set) { 1600 ret = irdma_manage_qhash(iwdev, cm_info, 1601 IRDMA_QHASH_TYPE_TCP_SYN, 1602 IRDMA_QHASH_MANAGE_TYPE_DELETE, 1603 NULL, false); 1604 child_listen_node->qhash_set = false; 1605 } else { 1606 ret = 0; 1607 } 1608 irdma_debug(&iwdev->rf->sc_dev, 1609 IRDMA_DEBUG_CM, "Child listen node freed = %p\n", 1610 child_listen_node); 1611 kfree(child_listen_node); 1612 cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++; 1613 } 1614 spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); 1615 1616 return ret; 1617 } 1618 1619 static u8 irdma_get_egress_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4){ 1620 return prio; 1621 } 1622 1623 /** 1624 * irdma_netdev_vlan_ipv6 - Gets the netdev and mac 1625 * @addr: local IPv6 address 1626 * @vlan_id: vlan id for the given IPv6 address 1627 * @mac: mac address for the given IPv6 address 1628 * 1629 * Returns the net_device of the IPv6 address and also sets the 1630 * vlan id and mac for that address. 1631 */ 1632 if_t 1633 irdma_netdev_vlan_ipv6(u32 *addr, u16 *vlan_id, u8 *mac) 1634 { 1635 if_t ip_dev = NULL; 1636 struct in6_addr laddr6; 1637 struct ifaddr *ifa; 1638 u16 scope_id = 0; 1639 1640 irdma_copy_ip_htonl(laddr6.__u6_addr.__u6_addr32, addr); 1641 if (vlan_id) 1642 *vlan_id = 0xFFFF; /* Match rdma_vlan_dev_vlan_id() */ 1643 if (mac) 1644 eth_zero_addr(mac); 1645 1646 if (IN6_IS_SCOPE_LINKLOCAL(&laddr6) || 1647 IN6_IS_ADDR_MC_INTFACELOCAL(&laddr6)) 1648 scope_id = ntohs(laddr6.__u6_addr.__u6_addr16[1]); 1649 1650 ip_dev = ip6_ifp_find(&init_net, laddr6, scope_id); 1651 if (ip_dev) { 1652 if (vlan_id) 1653 *vlan_id = rdma_vlan_dev_vlan_id(ip_dev); 1654 ifa = if_getifaddr(ip_dev); 1655 if (ifa && ifa->ifa_addr && mac) 1656 ether_addr_copy(mac, if_getlladdr(ip_dev)); 1657 } 1658 1659 return ip_dev; 1660 } 1661 1662 /** 1663 * irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address 1664 * @addr: local IPv4 address 1665 */ 1666 u16 1667 irdma_get_vlan_ipv4(u32 *addr) 1668 { 1669 if_t netdev; 1670 u16 vlan_id = 0xFFFF; 1671 1672 netdev = ip_ifp_find(&init_net, htonl(addr[0])); 1673 if (netdev) { 1674 vlan_id = rdma_vlan_dev_vlan_id(netdev); 1675 dev_put(netdev); 1676 } 1677 1678 return vlan_id; 1679 } 1680 1681 struct irdma_add_mqh_cbs { 1682 struct irdma_device *iwdev; 1683 struct irdma_cm_info *cm_info; 1684 struct irdma_cm_listener *cm_listen_node; 1685 }; 1686 1687 /** 1688 * irdma_add_mqh_ifa_cb - Adds multiple qhashes for IPV4/IPv6 1689 * @arg: Calback argument structure from irdma_add_mqh 1690 * @ifa: Current address to compute against 1691 * @count: Current cumulative output of all callbacks in this iteration 1692 * 1693 * Adds a qhash and a child listen node for a single IPv4/IPv6 address 1694 * on the adapter and adds the associated qhash filter 1695 */ 1696 static u_int 1697 irdma_add_mqh_ifa_cb(void *arg, struct ifaddr *ifa, u_int count) 1698 { 1699 struct irdma_add_mqh_cbs *cbs = arg; 1700 struct irdma_cm_listener *child_listen_node; 1701 struct irdma_cm_info *cm_info = cbs->cm_info; 1702 struct irdma_device *iwdev = cbs->iwdev; 1703 struct irdma_cm_listener *cm_parent_listen_node = cbs->cm_listen_node; 1704 if_t ip_dev = ifa->ifa_ifp; 1705 unsigned long flags; 1706 int ret; 1707 1708 if (count) 1709 return 0; 1710 1711 if (cm_info->ipv4) 1712 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1713 "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n", 1714 &ifa->ifa_addr, 1715 rdma_vlan_dev_vlan_id(ip_dev), if_getlladdr(ip_dev)); 1716 else 1717 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1718 "IP=%pI6, vlan_id=%d, MAC=%pM\n", 1719 &((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr, 1720 rdma_vlan_dev_vlan_id(ip_dev), 1721 if_getlladdr(ip_dev)); 1722 child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL); 1723 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1724 "Allocating child listener %p\n", 1725 child_listen_node); 1726 if (!child_listen_node) { 1727 irdma_debug(&iwdev->rf->sc_dev, 1728 IRDMA_DEBUG_CM, 1729 "listener memory allocation\n"); 1730 return -ENOMEM; 1731 } 1732 1733 memcpy(child_listen_node, cm_parent_listen_node, 1734 sizeof(*child_listen_node)); 1735 cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev); 1736 child_listen_node->vlan_id = cm_info->vlan_id; 1737 if (cm_info->ipv4) 1738 child_listen_node->loc_addr[0] = 1739 ntohl(((struct sockaddr_in *)ifa->ifa_addr)->sin_addr.s_addr); 1740 else 1741 irdma_copy_ip_ntohl(child_listen_node->loc_addr, 1742 ((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr.__u6_addr.__u6_addr32); 1743 memcpy(cm_info->loc_addr, child_listen_node->loc_addr, 1744 sizeof(cm_info->loc_addr)); 1745 if (!iwdev->vsi.dscp_mode) 1746 cm_info->user_pri = 1747 irdma_get_egress_vlan_prio(child_listen_node->loc_addr, 1748 cm_info->user_pri, 1749 false); 1750 ret = irdma_manage_qhash(iwdev, cm_info, 1751 IRDMA_QHASH_TYPE_TCP_SYN, 1752 IRDMA_QHASH_MANAGE_TYPE_ADD, 1753 NULL, true); 1754 if (ret) { 1755 kfree(child_listen_node); 1756 return ret; 1757 } 1758 1759 child_listen_node->qhash_set = true; 1760 spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags); 1761 list_add(&child_listen_node->child_listen_list, 1762 &cm_parent_listen_node->child_listen_list); 1763 spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags); 1764 cm_parent_listen_node->cm_core->stats_listen_nodes_created++; 1765 1766 return 0; 1767 } 1768 1769 /** 1770 * irdma_add_mqh - Adds multiple qhashes 1771 * @iwdev: iWarp device 1772 * @cm_info: CM info for parent listen node 1773 * @cm_listen_node: The parent listen node 1774 */ 1775 static int 1776 irdma_add_mqh(struct irdma_device *iwdev, 1777 struct irdma_cm_info *cm_info, 1778 struct irdma_cm_listener *cm_listen_node) 1779 { 1780 struct epoch_tracker et; 1781 struct irdma_add_mqh_cbs cbs; 1782 struct if_iter iter; 1783 if_t ifp; 1784 int err; 1785 1786 cbs.iwdev = iwdev; 1787 cbs.cm_info = cm_info; 1788 cbs.cm_listen_node = cm_listen_node; 1789 1790 VNET_ITERATOR_DECL(vnet_iter); 1791 1792 VNET_LIST_RLOCK(); 1793 NET_EPOCH_ENTER(et); 1794 VNET_FOREACH(vnet_iter) { 1795 CURVNET_SET_QUIET(vnet_iter); 1796 for (ifp = if_iter_start(&iter); ifp != NULL; ifp = if_iter_next(&iter)) { 1797 if (!(if_getflags(ifp) & IFF_UP)) 1798 continue; 1799 1800 if (((rdma_vlan_dev_vlan_id(ifp) >= VLAN_N_VID) || 1801 (rdma_vlan_dev_real_dev(ifp) != iwdev->netdev)) && 1802 ifp != iwdev->netdev) 1803 continue; 1804 1805 if_addr_rlock(ifp); 1806 if (cm_info->ipv4) 1807 err = if_foreach_addr_type(ifp, AF_INET, irdma_add_mqh_ifa_cb, &cbs); 1808 else 1809 err = if_foreach_addr_type(ifp, AF_INET6, irdma_add_mqh_ifa_cb, &cbs); 1810 if_addr_runlock(ifp); 1811 } 1812 if_iter_finish(&iter); 1813 CURVNET_RESTORE(); 1814 } 1815 NET_EPOCH_EXIT(et); 1816 VNET_LIST_RUNLOCK(); 1817 1818 return err; 1819 } 1820 1821 /** 1822 * irdma_reset_list_prep - add connection nodes slated for reset to list 1823 * @cm_core: cm's core 1824 * @listener: pointer to listener node 1825 * @reset_list: a list to which cm_node will be selected 1826 */ 1827 static void 1828 irdma_reset_list_prep(struct irdma_cm_core *cm_core, 1829 struct irdma_cm_listener *listener, 1830 struct list_head *reset_list) 1831 { 1832 struct irdma_cm_node *cm_node; 1833 int bkt; 1834 1835 HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { 1836 if (cm_node->listener == listener && 1837 !cm_node->accelerated && 1838 atomic_inc_not_zero(&cm_node->refcnt)) 1839 list_add(&cm_node->reset_entry, reset_list); 1840 } 1841 } 1842 1843 /** 1844 * irdma_dec_refcnt_listen - delete listener and associated cm nodes 1845 * @cm_core: cm's core 1846 * @listener: pointer to listener node 1847 * @free_hanging_nodes: to free associated cm_nodes 1848 * @apbvt_del: flag to delete the apbvt 1849 */ 1850 static int 1851 irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core, 1852 struct irdma_cm_listener *listener, 1853 int free_hanging_nodes, bool apbvt_del) 1854 { 1855 struct list_head *list_pos; 1856 struct list_head *list_temp; 1857 struct irdma_cm_node *cm_node; 1858 struct list_head reset_list; 1859 struct irdma_cm_info nfo; 1860 enum irdma_cm_node_state old_state; 1861 unsigned long flags; 1862 int err; 1863 1864 /* free non-accelerated child nodes for this listener */ 1865 INIT_LIST_HEAD(&reset_list); 1866 if (free_hanging_nodes) { 1867 rcu_read_lock(); 1868 irdma_reset_list_prep(cm_core, listener, &reset_list); 1869 rcu_read_unlock(); 1870 } 1871 1872 list_for_each_safe(list_pos, list_temp, &reset_list) { 1873 cm_node = container_of(list_pos, struct irdma_cm_node, 1874 reset_entry); 1875 if (cm_node->state >= IRDMA_CM_STATE_FIN_WAIT1) { 1876 irdma_rem_ref_cm_node(cm_node); 1877 continue; 1878 } 1879 1880 irdma_cleanup_retrans_entry(cm_node); 1881 err = irdma_send_reset(cm_node); 1882 if (err) { 1883 cm_node->state = IRDMA_CM_STATE_CLOSED; 1884 irdma_debug(&cm_node->iwdev->rf->sc_dev, 1885 IRDMA_DEBUG_CM, "send reset failed\n"); 1886 } else { 1887 old_state = cm_node->state; 1888 cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED; 1889 if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD) 1890 irdma_rem_ref_cm_node(cm_node); 1891 } 1892 } 1893 1894 if (atomic_dec_and_test(&listener->refcnt)) { 1895 spin_lock_irqsave(&cm_core->listen_list_lock, flags); 1896 list_del(&listener->list); 1897 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); 1898 1899 if (apbvt_del) 1900 irdma_del_apbvt(listener->iwdev, 1901 listener->apbvt_entry); 1902 memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr)); 1903 nfo.loc_port = listener->loc_port; 1904 nfo.ipv4 = listener->ipv4; 1905 nfo.vlan_id = listener->vlan_id; 1906 nfo.user_pri = listener->user_pri; 1907 nfo.qh_qpid = listener->iwdev->vsi.ilq->qp_id; 1908 1909 if (!list_empty(&listener->child_listen_list)) { 1910 irdma_del_multiple_qhash(listener->iwdev, &nfo, 1911 listener); 1912 } else { 1913 if (listener->qhash_set) 1914 irdma_manage_qhash(listener->iwdev, 1915 &nfo, 1916 IRDMA_QHASH_TYPE_TCP_SYN, 1917 IRDMA_QHASH_MANAGE_TYPE_DELETE, 1918 NULL, false); 1919 } 1920 1921 cm_core->stats_listen_destroyed++; 1922 cm_core->stats_listen_nodes_destroyed++; 1923 irdma_debug(&listener->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 1924 "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n", 1925 listener->loc_port, listener->loc_addr, listener, 1926 listener->cm_id, listener->qhash_set, 1927 listener->vlan_id, apbvt_del); 1928 kfree(listener); 1929 listener = NULL; 1930 return 0; 1931 } 1932 1933 return -EINVAL; 1934 } 1935 1936 /** 1937 * irdma_cm_del_listen - delete a listener 1938 * @cm_core: cm's core 1939 * @listener: passive connection's listener 1940 * @apbvt_del: flag to delete apbvt 1941 */ 1942 static int 1943 irdma_cm_del_listen(struct irdma_cm_core *cm_core, 1944 struct irdma_cm_listener *listener, 1945 bool apbvt_del) 1946 { 1947 listener->listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE; 1948 listener->cm_id = NULL; 1949 1950 return irdma_dec_refcnt_listen(cm_core, listener, 1, apbvt_del); 1951 } 1952 1953 /** 1954 * irdma_find_node - find a cm node that matches the reference cm node 1955 * @cm_core: cm's core 1956 * @rem_port: remote tcp port num 1957 * @rem_addr: remote ip addr 1958 * @loc_port: local tcp port num 1959 * @loc_addr: local ip addr 1960 * @vlan_id: local VLAN ID 1961 */ 1962 struct irdma_cm_node * 1963 irdma_find_node(struct irdma_cm_core *cm_core, 1964 u16 rem_port, u32 *rem_addr, u16 loc_port, 1965 u32 *loc_addr, u16 vlan_id) 1966 { 1967 struct irdma_cm_node *cm_node; 1968 u32 key = (rem_port << 16) | loc_port; 1969 1970 rcu_read_lock(); 1971 HASH_FOR_EACH_POSSIBLE_RCU(cm_core->cm_hash_tbl, cm_node, list, key) { 1972 if (cm_node->vlan_id == vlan_id && 1973 cm_node->loc_port == loc_port && cm_node->rem_port == rem_port && 1974 !memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) && 1975 !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr))) { 1976 if (!atomic_inc_not_zero(&cm_node->refcnt)) 1977 goto exit; 1978 rcu_read_unlock(); 1979 return cm_node; 1980 } 1981 } 1982 1983 exit: 1984 rcu_read_unlock(); 1985 1986 /* no owner node */ 1987 return NULL; 1988 } 1989 1990 /** 1991 * irdma_add_hte_node - add a cm node to the hash table 1992 * @cm_core: cm's core 1993 * @cm_node: connection's node 1994 */ 1995 static void 1996 irdma_add_hte_node(struct irdma_cm_core *cm_core, 1997 struct irdma_cm_node *cm_node) 1998 { 1999 unsigned long flags; 2000 u32 key = (cm_node->rem_port << 16) | cm_node->loc_port; 2001 2002 spin_lock_irqsave(&cm_core->ht_lock, flags); 2003 HASH_ADD_RCU(cm_core->cm_hash_tbl, &cm_node->list, key); 2004 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 2005 } 2006 2007 /** 2008 * irdma_ipv4_is_lpb - check if loopback 2009 * @loc_addr: local addr to compare 2010 * @rem_addr: remote address 2011 */ 2012 bool 2013 irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr) 2014 { 2015 return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr); 2016 } 2017 2018 /** 2019 * irdma_ipv6_is_lpb - check if loopback 2020 * @loc_addr: local addr to compare 2021 * @rem_addr: remote address 2022 */ 2023 bool 2024 irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr) 2025 { 2026 struct in6_addr raddr6; 2027 2028 irdma_copy_ip_htonl(raddr6.__u6_addr.__u6_addr32, rem_addr); 2029 2030 return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6); 2031 } 2032 2033 /** 2034 * irdma_cm_create_ah - create a cm address handle 2035 * @cm_node: The connection manager node to create AH for 2036 * @wait: Provides option to wait for ah creation or not 2037 */ 2038 static int 2039 irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait) 2040 { 2041 struct irdma_ah_info ah_info = {0}; 2042 struct irdma_device *iwdev = cm_node->iwdev; 2043 #ifdef VIMAGE 2044 struct rdma_cm_id *rdma_id = (struct rdma_cm_id *)cm_node->cm_id->context; 2045 struct vnet *vnet = rdma_id->route.addr.dev_addr.net; 2046 #endif 2047 2048 ether_addr_copy(ah_info.mac_addr, if_getlladdr(iwdev->netdev)); 2049 2050 ah_info.hop_ttl = 0x40; 2051 ah_info.tc_tos = cm_node->tos; 2052 ah_info.vsi = &iwdev->vsi; 2053 2054 if (cm_node->ipv4) { 2055 ah_info.ipv4_valid = true; 2056 ah_info.dest_ip_addr[0] = cm_node->rem_addr[0]; 2057 ah_info.src_ip_addr[0] = cm_node->loc_addr[0]; 2058 CURVNET_SET_QUIET(vnet); 2059 ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0], 2060 ah_info.dest_ip_addr[0]); 2061 CURVNET_RESTORE(); 2062 } else { 2063 memcpy(ah_info.dest_ip_addr, cm_node->rem_addr, 2064 sizeof(ah_info.dest_ip_addr)); 2065 memcpy(ah_info.src_ip_addr, cm_node->loc_addr, 2066 sizeof(ah_info.src_ip_addr)); 2067 ah_info.do_lpbk = irdma_ipv6_is_lpb(ah_info.src_ip_addr, 2068 ah_info.dest_ip_addr); 2069 } 2070 2071 ah_info.vlan_tag = cm_node->vlan_id; 2072 if (cm_node->vlan_id < VLAN_N_VID) { 2073 ah_info.insert_vlan_tag = 1; 2074 ah_info.vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT; 2075 } 2076 2077 ah_info.dst_arpindex = 2078 irdma_arp_table(iwdev->rf, ah_info.dest_ip_addr, 2079 NULL, IRDMA_ARP_RESOLVE); 2080 2081 if (irdma_puda_create_ah(&iwdev->rf->sc_dev, &ah_info, wait, 2082 IRDMA_PUDA_RSRC_TYPE_ILQ, cm_node, 2083 &cm_node->ah)) 2084 return -ENOMEM; 2085 2086 return 0; 2087 } 2088 2089 /** 2090 * irdma_cm_free_ah - free a cm address handle 2091 * @cm_node: The connection manager node to create AH for 2092 */ 2093 static void 2094 irdma_cm_free_ah(struct irdma_cm_node *cm_node) 2095 { 2096 struct irdma_device *iwdev = cm_node->iwdev; 2097 2098 irdma_puda_free_ah(&iwdev->rf->sc_dev, cm_node->ah); 2099 cm_node->ah = NULL; 2100 } 2101 2102 /** 2103 * irdma_make_cm_node - create a new instance of a cm node 2104 * @cm_core: cm's core 2105 * @iwdev: iwarp device structure 2106 * @cm_info: quad info for connection 2107 * @listener: passive connection's listener 2108 */ 2109 static struct irdma_cm_node * 2110 irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev, 2111 struct irdma_cm_info *cm_info, 2112 struct irdma_cm_listener *listener) 2113 { 2114 struct irdma_cm_node *cm_node; 2115 int arpindex; 2116 if_t netdev = iwdev->netdev; 2117 2118 /* create an hte and cm_node for this instance */ 2119 cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC); 2120 if (!cm_node) 2121 return NULL; 2122 2123 /* set our node specific transport info */ 2124 cm_node->ipv4 = cm_info->ipv4; 2125 cm_node->vlan_id = cm_info->vlan_id; 2126 if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 2127 cm_node->vlan_id = 0; 2128 cm_node->tos = cm_info->tos; 2129 cm_node->user_pri = cm_info->user_pri; 2130 if (listener) { 2131 if (listener->tos != cm_info->tos) 2132 irdma_dev_warn(&iwdev->ibdev, 2133 "application TOS[%d] and remote client TOS[%d] mismatch\n", 2134 listener->tos, cm_info->tos); 2135 if (iwdev->vsi.dscp_mode) { 2136 cm_node->user_pri = listener->user_pri; 2137 } else { 2138 cm_node->tos = max(listener->tos, cm_info->tos); 2139 cm_node->user_pri = rt_tos2priority(cm_node->tos); 2140 cm_node->user_pri = 2141 irdma_get_egress_vlan_prio(cm_info->loc_addr, 2142 cm_node->user_pri, 2143 cm_info->ipv4); 2144 } 2145 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, 2146 "listener: TOS:[%d] UP:[%d]\n", 2147 cm_node->tos, 2148 cm_node->user_pri); 2149 } 2150 memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr)); 2151 memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr)); 2152 cm_node->loc_port = cm_info->loc_port; 2153 cm_node->rem_port = cm_info->rem_port; 2154 2155 cm_node->mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER; 2156 cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; 2157 cm_node->iwdev = iwdev; 2158 cm_node->dev = &iwdev->rf->sc_dev; 2159 2160 cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird; 2161 cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord; 2162 2163 cm_node->listener = listener; 2164 cm_node->cm_id = cm_info->cm_id; 2165 ether_addr_copy(cm_node->loc_mac, if_getlladdr(netdev)); 2166 spin_lock_init(&cm_node->retrans_list_lock); 2167 cm_node->ack_rcvd = false; 2168 2169 init_completion(&cm_node->establish_comp); 2170 atomic_set(&cm_node->refcnt, 1); 2171 /* associate our parent CM core */ 2172 cm_node->cm_core = cm_core; 2173 cm_node->tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID; 2174 cm_node->tcp_cntxt.rcv_wscale = iwdev->rcv_wscale; 2175 cm_node->tcp_cntxt.rcv_wnd = iwdev->rcv_wnd >> cm_node->tcp_cntxt.rcv_wscale; 2176 kc_set_loc_seq_num_mss(cm_node); 2177 2178 arpindex = irdma_resolve_neigh_lpb_chk(iwdev, cm_node, cm_info); 2179 if (arpindex < 0) 2180 goto err; 2181 2182 ether_addr_copy(cm_node->rem_mac, iwdev->rf->arp_table[arpindex].mac_addr); 2183 irdma_add_hte_node(cm_core, cm_node); 2184 cm_core->stats_nodes_created++; 2185 return cm_node; 2186 2187 err: 2188 kfree(cm_node); 2189 2190 return NULL; 2191 } 2192 2193 static void 2194 irdma_destroy_connection(struct irdma_cm_node *cm_node) 2195 { 2196 struct irdma_cm_core *cm_core = cm_node->cm_core; 2197 struct irdma_qp *iwqp; 2198 struct irdma_cm_info nfo; 2199 2200 /* if the node is destroyed before connection was accelerated */ 2201 if (!cm_node->accelerated && cm_node->accept_pend) { 2202 irdma_debug(&cm_node->iwdev->rf->sc_dev, 2203 IRDMA_DEBUG_CM, "node destroyed before established\n"); 2204 atomic_dec(&cm_node->listener->pend_accepts_cnt); 2205 } 2206 if (cm_node->close_entry) 2207 irdma_handle_close_entry(cm_node, 0); 2208 if (cm_node->listener) { 2209 irdma_dec_refcnt_listen(cm_core, cm_node->listener, 0, true); 2210 } else { 2211 if (cm_node->apbvt_set) { 2212 irdma_del_apbvt(cm_node->iwdev, cm_node->apbvt_entry); 2213 cm_node->apbvt_set = 0; 2214 } 2215 irdma_get_addr_info(cm_node, &nfo); 2216 if (cm_node->qhash_set) { 2217 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; 2218 irdma_manage_qhash(cm_node->iwdev, &nfo, 2219 IRDMA_QHASH_TYPE_TCP_ESTABLISHED, 2220 IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, 2221 false); 2222 cm_node->qhash_set = 0; 2223 } 2224 } 2225 2226 iwqp = cm_node->iwqp; 2227 if (iwqp) { 2228 cm_node->cm_id->rem_ref(cm_node->cm_id); 2229 cm_node->cm_id = NULL; 2230 iwqp->cm_id = NULL; 2231 irdma_qp_rem_ref(&iwqp->ibqp); 2232 cm_node->iwqp = NULL; 2233 } else if (cm_node->qhash_set) { 2234 irdma_get_addr_info(cm_node, &nfo); 2235 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; 2236 irdma_manage_qhash(cm_node->iwdev, &nfo, 2237 IRDMA_QHASH_TYPE_TCP_ESTABLISHED, 2238 IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false); 2239 cm_node->qhash_set = 0; 2240 } 2241 2242 cm_core->cm_free_ah(cm_node); 2243 } 2244 2245 /** 2246 * irdma_rem_ref_cm_node - destroy an instance of a cm node 2247 * @cm_node: connection's node 2248 */ 2249 void 2250 irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node) 2251 { 2252 struct irdma_cm_core *cm_core = cm_node->cm_core; 2253 unsigned long flags; 2254 2255 spin_lock_irqsave(&cm_core->ht_lock, flags); 2256 2257 if (!atomic_dec_and_test(&cm_node->refcnt)) { 2258 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 2259 return; 2260 } 2261 if (cm_node->iwqp) { 2262 cm_node->iwqp->cm_node = NULL; 2263 cm_node->iwqp->cm_id = NULL; 2264 } 2265 HASH_DEL_RCU(cm_core->cm_hash_tbl, &cm_node->list); 2266 cm_node->cm_core->stats_nodes_destroyed++; 2267 2268 spin_unlock_irqrestore(&cm_core->ht_lock, flags); 2269 2270 irdma_destroy_connection(cm_node); 2271 2272 kfree_rcu(cm_node, rcu_head); 2273 } 2274 2275 /** 2276 * irdma_handle_fin_pkt - FIN packet received 2277 * @cm_node: connection's node 2278 */ 2279 static void 2280 irdma_handle_fin_pkt(struct irdma_cm_node *cm_node) 2281 { 2282 switch (cm_node->state) { 2283 case IRDMA_CM_STATE_SYN_RCVD: 2284 case IRDMA_CM_STATE_SYN_SENT: 2285 case IRDMA_CM_STATE_ESTABLISHED: 2286 case IRDMA_CM_STATE_MPAREJ_RCVD: 2287 cm_node->tcp_cntxt.rcv_nxt++; 2288 irdma_cleanup_retrans_entry(cm_node); 2289 cm_node->state = IRDMA_CM_STATE_LAST_ACK; 2290 irdma_send_fin(cm_node); 2291 break; 2292 case IRDMA_CM_STATE_MPAREQ_SENT: 2293 irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED); 2294 cm_node->tcp_cntxt.rcv_nxt++; 2295 irdma_cleanup_retrans_entry(cm_node); 2296 cm_node->state = IRDMA_CM_STATE_CLOSED; 2297 atomic_inc(&cm_node->refcnt); 2298 irdma_send_reset(cm_node); 2299 break; 2300 case IRDMA_CM_STATE_FIN_WAIT1: 2301 cm_node->tcp_cntxt.rcv_nxt++; 2302 irdma_cleanup_retrans_entry(cm_node); 2303 cm_node->state = IRDMA_CM_STATE_CLOSING; 2304 irdma_send_ack(cm_node); 2305 /* 2306 * Wait for ACK as this is simultaneous close. After we receive ACK, do not send anything. Just rm the 2307 * node. 2308 */ 2309 break; 2310 case IRDMA_CM_STATE_FIN_WAIT2: 2311 cm_node->tcp_cntxt.rcv_nxt++; 2312 irdma_cleanup_retrans_entry(cm_node); 2313 cm_node->state = IRDMA_CM_STATE_TIME_WAIT; 2314 irdma_send_ack(cm_node); 2315 irdma_schedule_cm_timer(cm_node, NULL, IRDMA_TIMER_TYPE_CLOSE, 2316 1, 0); 2317 break; 2318 case IRDMA_CM_STATE_TIME_WAIT: 2319 cm_node->tcp_cntxt.rcv_nxt++; 2320 irdma_cleanup_retrans_entry(cm_node); 2321 cm_node->state = IRDMA_CM_STATE_CLOSED; 2322 irdma_rem_ref_cm_node(cm_node); 2323 break; 2324 case IRDMA_CM_STATE_OFFLOADED: 2325 default: 2326 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2327 "bad state node state = %d\n", 2328 cm_node->state); 2329 break; 2330 } 2331 } 2332 2333 /** 2334 * irdma_handle_rst_pkt - process received RST packet 2335 * @cm_node: connection's node 2336 * @rbuf: receive buffer 2337 */ 2338 static void 2339 irdma_handle_rst_pkt(struct irdma_cm_node *cm_node, 2340 struct irdma_puda_buf *rbuf) 2341 { 2342 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2343 "caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n", 2344 __builtin_return_address(0), cm_node, cm_node->state, 2345 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, 2346 cm_node->loc_addr); 2347 2348 irdma_cleanup_retrans_entry(cm_node); 2349 switch (cm_node->state) { 2350 case IRDMA_CM_STATE_SYN_SENT: 2351 case IRDMA_CM_STATE_MPAREQ_SENT: 2352 switch (cm_node->mpa_frame_rev) { 2353 case IETF_MPA_V2: 2354 /* Drop down to MPA_V1 */ 2355 cm_node->mpa_frame_rev = IETF_MPA_V1; 2356 /* send a syn and goto syn sent state */ 2357 cm_node->state = IRDMA_CM_STATE_SYN_SENT; 2358 if (irdma_send_syn(cm_node, 0)) 2359 irdma_active_open_err(cm_node, false); 2360 break; 2361 case IETF_MPA_V1: 2362 default: 2363 irdma_active_open_err(cm_node, false); 2364 break; 2365 } 2366 break; 2367 case IRDMA_CM_STATE_MPAREQ_RCVD: 2368 atomic_inc(&cm_node->passive_state); 2369 break; 2370 case IRDMA_CM_STATE_ESTABLISHED: 2371 case IRDMA_CM_STATE_SYN_RCVD: 2372 case IRDMA_CM_STATE_LISTENING: 2373 irdma_passive_open_err(cm_node, false); 2374 break; 2375 case IRDMA_CM_STATE_OFFLOADED: 2376 irdma_active_open_err(cm_node, false); 2377 break; 2378 case IRDMA_CM_STATE_CLOSED: 2379 break; 2380 case IRDMA_CM_STATE_FIN_WAIT2: 2381 case IRDMA_CM_STATE_FIN_WAIT1: 2382 case IRDMA_CM_STATE_LAST_ACK: 2383 case IRDMA_CM_STATE_TIME_WAIT: 2384 cm_node->state = IRDMA_CM_STATE_CLOSED; 2385 irdma_rem_ref_cm_node(cm_node); 2386 break; 2387 default: 2388 break; 2389 } 2390 } 2391 2392 /** 2393 * irdma_handle_rcv_mpa - Process a recv'd mpa buffer 2394 * @cm_node: connection's node 2395 * @rbuf: receive buffer 2396 */ 2397 static void 2398 irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node, 2399 struct irdma_puda_buf *rbuf) 2400 { 2401 int err; 2402 int datasize = rbuf->datalen; 2403 u8 *dataloc = rbuf->data; 2404 2405 enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN; 2406 u32 res_type; 2407 2408 err = irdma_parse_mpa(cm_node, dataloc, &res_type, datasize); 2409 if (err) { 2410 if (cm_node->state == IRDMA_CM_STATE_MPAREQ_SENT) 2411 irdma_active_open_err(cm_node, true); 2412 else 2413 irdma_passive_open_err(cm_node, true); 2414 return; 2415 } 2416 2417 switch (cm_node->state) { 2418 case IRDMA_CM_STATE_ESTABLISHED: 2419 if (res_type == IRDMA_MPA_REQUEST_REJECT) 2420 irdma_debug(&cm_node->iwdev->rf->sc_dev, 2421 IRDMA_DEBUG_CM, "state for reject\n"); 2422 cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD; 2423 type = IRDMA_CM_EVENT_MPA_REQ; 2424 irdma_send_ack(cm_node); /* ACK received MPA request */ 2425 atomic_set(&cm_node->passive_state, 2426 IRDMA_PASSIVE_STATE_INDICATED); 2427 break; 2428 case IRDMA_CM_STATE_MPAREQ_SENT: 2429 irdma_cleanup_retrans_entry(cm_node); 2430 if (res_type == IRDMA_MPA_REQUEST_REJECT) { 2431 type = IRDMA_CM_EVENT_MPA_REJECT; 2432 cm_node->state = IRDMA_CM_STATE_MPAREJ_RCVD; 2433 } else { 2434 type = IRDMA_CM_EVENT_CONNECTED; 2435 cm_node->state = IRDMA_CM_STATE_OFFLOADED; 2436 } 2437 irdma_send_ack(cm_node); 2438 break; 2439 default: 2440 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2441 "wrong cm_node state =%d\n", 2442 cm_node->state); 2443 break; 2444 } 2445 irdma_create_event(cm_node, type); 2446 } 2447 2448 /** 2449 * irdma_check_syn - Check for error on received syn ack 2450 * @cm_node: connection's node 2451 * @tcph: pointer tcp header 2452 */ 2453 static int 2454 irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph) 2455 { 2456 if (ntohl(tcph->th_ack) != cm_node->tcp_cntxt.loc_seq_num) { 2457 irdma_active_open_err(cm_node, true); 2458 return 1; 2459 } 2460 2461 return 0; 2462 } 2463 2464 /** 2465 * irdma_check_seq - check seq numbers if OK 2466 * @cm_node: connection's node 2467 * @tcph: pointer tcp header 2468 */ 2469 static int 2470 irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph) 2471 { 2472 u32 seq; 2473 u32 ack_seq; 2474 u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num; 2475 u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; 2476 u32 rcv_wnd; 2477 int err = 0; 2478 2479 seq = ntohl(tcph->th_seq); 2480 ack_seq = ntohl(tcph->th_ack); 2481 rcv_wnd = cm_node->tcp_cntxt.rcv_wnd; 2482 if (ack_seq != loc_seq_num || 2483 !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd))) 2484 err = -1; 2485 if (err) 2486 irdma_debug(&cm_node->iwdev->rf->sc_dev, 2487 IRDMA_DEBUG_CM, "seq number err\n"); 2488 2489 return err; 2490 } 2491 2492 void 2493 irdma_add_conn_est_qh(struct irdma_cm_node *cm_node) 2494 { 2495 struct irdma_cm_info nfo; 2496 2497 irdma_get_addr_info(cm_node, &nfo); 2498 nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id; 2499 irdma_manage_qhash(cm_node->iwdev, &nfo, 2500 IRDMA_QHASH_TYPE_TCP_ESTABLISHED, 2501 IRDMA_QHASH_MANAGE_TYPE_ADD, 2502 cm_node, false); 2503 cm_node->qhash_set = true; 2504 } 2505 2506 /** 2507 * irdma_handle_syn_pkt - is for Passive node 2508 * @cm_node: connection's node 2509 * @rbuf: receive buffer 2510 */ 2511 static void 2512 irdma_handle_syn_pkt(struct irdma_cm_node *cm_node, 2513 struct irdma_puda_buf *rbuf) 2514 { 2515 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; 2516 int err; 2517 u32 inc_sequence; 2518 int optionsize; 2519 2520 optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); 2521 inc_sequence = ntohl(tcph->th_seq); 2522 2523 switch (cm_node->state) { 2524 case IRDMA_CM_STATE_SYN_SENT: 2525 case IRDMA_CM_STATE_MPAREQ_SENT: 2526 /* Rcvd syn on active open connection */ 2527 irdma_active_open_err(cm_node, 1); 2528 break; 2529 case IRDMA_CM_STATE_LISTENING: 2530 /* Passive OPEN */ 2531 if (atomic_read(&cm_node->listener->pend_accepts_cnt) > 2532 cm_node->listener->backlog) { 2533 cm_node->cm_core->stats_backlog_drops++; 2534 irdma_passive_open_err(cm_node, false); 2535 break; 2536 } 2537 err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1); 2538 if (err) { 2539 irdma_passive_open_err(cm_node, false); 2540 /* drop pkt */ 2541 break; 2542 } 2543 err = cm_node->cm_core->cm_create_ah(cm_node, false); 2544 if (err) { 2545 irdma_passive_open_err(cm_node, false); 2546 /* drop pkt */ 2547 break; 2548 } 2549 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; 2550 cm_node->accept_pend = 1; 2551 atomic_inc(&cm_node->listener->pend_accepts_cnt); 2552 2553 cm_node->state = IRDMA_CM_STATE_SYN_RCVD; 2554 break; 2555 case IRDMA_CM_STATE_CLOSED: 2556 irdma_cleanup_retrans_entry(cm_node); 2557 atomic_inc(&cm_node->refcnt); 2558 irdma_send_reset(cm_node); 2559 break; 2560 case IRDMA_CM_STATE_OFFLOADED: 2561 case IRDMA_CM_STATE_ESTABLISHED: 2562 case IRDMA_CM_STATE_FIN_WAIT1: 2563 case IRDMA_CM_STATE_FIN_WAIT2: 2564 case IRDMA_CM_STATE_MPAREQ_RCVD: 2565 case IRDMA_CM_STATE_LAST_ACK: 2566 case IRDMA_CM_STATE_CLOSING: 2567 case IRDMA_CM_STATE_UNKNOWN: 2568 default: 2569 break; 2570 } 2571 } 2572 2573 /** 2574 * irdma_handle_synack_pkt - Process SYN+ACK packet (active side) 2575 * @cm_node: connection's node 2576 * @rbuf: receive buffer 2577 */ 2578 static void 2579 irdma_handle_synack_pkt(struct irdma_cm_node *cm_node, 2580 struct irdma_puda_buf *rbuf) 2581 { 2582 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; 2583 int err; 2584 u32 inc_sequence; 2585 int optionsize; 2586 2587 optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); 2588 inc_sequence = ntohl(tcph->th_seq); 2589 switch (cm_node->state) { 2590 case IRDMA_CM_STATE_SYN_SENT: 2591 irdma_cleanup_retrans_entry(cm_node); 2592 /* active open */ 2593 if (irdma_check_syn(cm_node, tcph)) { 2594 irdma_debug(&cm_node->iwdev->rf->sc_dev, 2595 IRDMA_DEBUG_CM, "check syn fail\n"); 2596 return; 2597 } 2598 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); 2599 /* setup options */ 2600 err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0); 2601 if (err) { 2602 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2603 "cm_node=%p tcp_options failed\n", 2604 cm_node); 2605 break; 2606 } 2607 irdma_cleanup_retrans_entry(cm_node); 2608 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; 2609 irdma_send_ack(cm_node); /* ACK for the syn_ack */ 2610 err = irdma_send_mpa_request(cm_node); 2611 if (err) { 2612 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 2613 "cm_node=%p irdma_send_mpa_request failed\n", 2614 cm_node); 2615 break; 2616 } 2617 cm_node->state = IRDMA_CM_STATE_MPAREQ_SENT; 2618 break; 2619 case IRDMA_CM_STATE_MPAREQ_RCVD: 2620 irdma_passive_open_err(cm_node, true); 2621 break; 2622 case IRDMA_CM_STATE_LISTENING: 2623 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); 2624 irdma_cleanup_retrans_entry(cm_node); 2625 cm_node->state = IRDMA_CM_STATE_CLOSED; 2626 irdma_send_reset(cm_node); 2627 break; 2628 case IRDMA_CM_STATE_CLOSED: 2629 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); 2630 irdma_cleanup_retrans_entry(cm_node); 2631 atomic_inc(&cm_node->refcnt); 2632 irdma_send_reset(cm_node); 2633 break; 2634 case IRDMA_CM_STATE_ESTABLISHED: 2635 case IRDMA_CM_STATE_FIN_WAIT1: 2636 case IRDMA_CM_STATE_FIN_WAIT2: 2637 case IRDMA_CM_STATE_LAST_ACK: 2638 case IRDMA_CM_STATE_OFFLOADED: 2639 case IRDMA_CM_STATE_CLOSING: 2640 case IRDMA_CM_STATE_UNKNOWN: 2641 case IRDMA_CM_STATE_MPAREQ_SENT: 2642 default: 2643 break; 2644 } 2645 } 2646 2647 /** 2648 * irdma_handle_ack_pkt - process packet with ACK 2649 * @cm_node: connection's node 2650 * @rbuf: receive buffer 2651 */ 2652 static int 2653 irdma_handle_ack_pkt(struct irdma_cm_node *cm_node, 2654 struct irdma_puda_buf *rbuf) 2655 { 2656 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; 2657 u32 inc_sequence; 2658 int ret; 2659 int optionsize; 2660 u32 datasize = rbuf->datalen; 2661 2662 optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr); 2663 2664 if (irdma_check_seq(cm_node, tcph)) 2665 return -EINVAL; 2666 2667 inc_sequence = ntohl(tcph->th_seq); 2668 switch (cm_node->state) { 2669 case IRDMA_CM_STATE_SYN_RCVD: 2670 irdma_cleanup_retrans_entry(cm_node); 2671 ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1); 2672 if (ret) 2673 return ret; 2674 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); 2675 cm_node->state = IRDMA_CM_STATE_ESTABLISHED; 2676 if (datasize) { 2677 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; 2678 irdma_handle_rcv_mpa(cm_node, rbuf); 2679 } 2680 break; 2681 case IRDMA_CM_STATE_ESTABLISHED: 2682 irdma_cleanup_retrans_entry(cm_node); 2683 if (datasize) { 2684 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; 2685 irdma_handle_rcv_mpa(cm_node, rbuf); 2686 } 2687 break; 2688 case IRDMA_CM_STATE_MPAREQ_SENT: 2689 cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); 2690 if (datasize) { 2691 cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; 2692 cm_node->ack_rcvd = false; 2693 irdma_handle_rcv_mpa(cm_node, rbuf); 2694 } else { 2695 cm_node->ack_rcvd = true; 2696 } 2697 break; 2698 case IRDMA_CM_STATE_LISTENING: 2699 irdma_cleanup_retrans_entry(cm_node); 2700 cm_node->state = IRDMA_CM_STATE_CLOSED; 2701 irdma_send_reset(cm_node); 2702 break; 2703 case IRDMA_CM_STATE_CLOSED: 2704 irdma_cleanup_retrans_entry(cm_node); 2705 atomic_inc(&cm_node->refcnt); 2706 irdma_send_reset(cm_node); 2707 break; 2708 case IRDMA_CM_STATE_LAST_ACK: 2709 case IRDMA_CM_STATE_CLOSING: 2710 irdma_cleanup_retrans_entry(cm_node); 2711 cm_node->state = IRDMA_CM_STATE_CLOSED; 2712 irdma_rem_ref_cm_node(cm_node); 2713 break; 2714 case IRDMA_CM_STATE_FIN_WAIT1: 2715 irdma_cleanup_retrans_entry(cm_node); 2716 cm_node->state = IRDMA_CM_STATE_FIN_WAIT2; 2717 break; 2718 case IRDMA_CM_STATE_SYN_SENT: 2719 case IRDMA_CM_STATE_FIN_WAIT2: 2720 case IRDMA_CM_STATE_OFFLOADED: 2721 case IRDMA_CM_STATE_MPAREQ_RCVD: 2722 case IRDMA_CM_STATE_UNKNOWN: 2723 default: 2724 irdma_cleanup_retrans_entry(cm_node); 2725 break; 2726 } 2727 2728 return 0; 2729 } 2730 2731 /** 2732 * irdma_process_pkt - process cm packet 2733 * @cm_node: connection's node 2734 * @rbuf: receive buffer 2735 */ 2736 static void 2737 irdma_process_pkt(struct irdma_cm_node *cm_node, 2738 struct irdma_puda_buf *rbuf) 2739 { 2740 enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN; 2741 struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph; 2742 u32 fin_set = 0; 2743 int err; 2744 2745 if (tcph->th_flags & TH_RST) { 2746 pkt_type = IRDMA_PKT_TYPE_RST; 2747 } else if (tcph->th_flags & TH_SYN) { 2748 pkt_type = IRDMA_PKT_TYPE_SYN; 2749 if (tcph->th_flags & TH_ACK) 2750 pkt_type = IRDMA_PKT_TYPE_SYNACK; 2751 } else if (tcph->th_flags & TH_ACK) { 2752 pkt_type = IRDMA_PKT_TYPE_ACK; 2753 } 2754 if (tcph->th_flags & TH_FIN) 2755 fin_set = 1; 2756 2757 switch (pkt_type) { 2758 case IRDMA_PKT_TYPE_SYN: 2759 irdma_handle_syn_pkt(cm_node, rbuf); 2760 break; 2761 case IRDMA_PKT_TYPE_SYNACK: 2762 irdma_handle_synack_pkt(cm_node, rbuf); 2763 break; 2764 case IRDMA_PKT_TYPE_ACK: 2765 err = irdma_handle_ack_pkt(cm_node, rbuf); 2766 if (fin_set && !err) 2767 irdma_handle_fin_pkt(cm_node); 2768 break; 2769 case IRDMA_PKT_TYPE_RST: 2770 irdma_handle_rst_pkt(cm_node, rbuf); 2771 break; 2772 default: 2773 if (fin_set && 2774 (!irdma_check_seq(cm_node, (struct tcphdr *)rbuf->tcph))) 2775 irdma_handle_fin_pkt(cm_node); 2776 break; 2777 } 2778 } 2779 2780 /** 2781 * irdma_make_listen_node - create a listen node with params 2782 * @cm_core: cm's core 2783 * @iwdev: iwarp device structure 2784 * @cm_info: quad info for connection 2785 */ 2786 static struct irdma_cm_listener * 2787 irdma_make_listen_node(struct irdma_cm_core *cm_core, 2788 struct irdma_device *iwdev, 2789 struct irdma_cm_info *cm_info) 2790 { 2791 struct irdma_cm_listener *listener; 2792 unsigned long flags; 2793 2794 /* cannot have multiple matching listeners */ 2795 listener = irdma_find_listener(cm_core, cm_info->loc_addr, cm_info->ipv4, 2796 cm_info->loc_port, cm_info->vlan_id, 2797 IRDMA_CM_LISTENER_EITHER_STATE); 2798 if (listener && 2799 listener->listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) { 2800 atomic_dec(&listener->refcnt); 2801 return NULL; 2802 } 2803 2804 if (!listener) { 2805 /* 2806 * create a CM listen node 1/2 node to compare incoming traffic to 2807 */ 2808 listener = kzalloc(sizeof(*listener), GFP_KERNEL); 2809 if (!listener) 2810 return NULL; 2811 cm_core->stats_listen_nodes_created++; 2812 memcpy(listener->loc_addr, cm_info->loc_addr, 2813 sizeof(listener->loc_addr)); 2814 listener->loc_port = cm_info->loc_port; 2815 2816 INIT_LIST_HEAD(&listener->child_listen_list); 2817 2818 atomic_set(&listener->refcnt, 1); 2819 } else { 2820 listener->reused_node = 1; 2821 } 2822 2823 listener->cm_id = cm_info->cm_id; 2824 listener->ipv4 = cm_info->ipv4; 2825 listener->vlan_id = cm_info->vlan_id; 2826 atomic_set(&listener->pend_accepts_cnt, 0); 2827 listener->cm_core = cm_core; 2828 listener->iwdev = iwdev; 2829 2830 listener->backlog = cm_info->backlog; 2831 listener->listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE; 2832 2833 if (!listener->reused_node) { 2834 spin_lock_irqsave(&cm_core->listen_list_lock, flags); 2835 list_add(&listener->list, &cm_core->listen_list); 2836 spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); 2837 } 2838 2839 return listener; 2840 } 2841 2842 /** 2843 * irdma_create_cm_node - make a connection node with params 2844 * @cm_core: cm's core 2845 * @iwdev: iwarp device structure 2846 * @conn_param: connection parameters 2847 * @cm_info: quad info for connection 2848 * @caller_cm_node: pointer to cm_node structure to return 2849 */ 2850 static int 2851 irdma_create_cm_node(struct irdma_cm_core *cm_core, 2852 struct irdma_device *iwdev, 2853 struct iw_cm_conn_param *conn_param, 2854 struct irdma_cm_info *cm_info, 2855 struct irdma_cm_node **caller_cm_node) 2856 { 2857 struct irdma_cm_node *cm_node; 2858 u16 private_data_len = conn_param->private_data_len; 2859 const void *private_data = conn_param->private_data; 2860 2861 /* create a CM connection node */ 2862 cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL); 2863 if (!cm_node) 2864 return -ENOMEM; 2865 2866 /* set our node side to client (active) side */ 2867 cm_node->tcp_cntxt.client = 1; 2868 cm_node->tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; 2869 2870 irdma_record_ird_ord(cm_node, conn_param->ird, conn_param->ord); 2871 2872 cm_node->pdata.size = private_data_len; 2873 cm_node->pdata.addr = cm_node->pdata_buf; 2874 2875 memcpy(cm_node->pdata_buf, private_data, private_data_len); 2876 *caller_cm_node = cm_node; 2877 2878 return 0; 2879 } 2880 2881 /** 2882 * irdma_cm_reject - reject and teardown a connection 2883 * @cm_node: connection's node 2884 * @pdata: ptr to private data for reject 2885 * @plen: size of private data 2886 */ 2887 static int 2888 irdma_cm_reject(struct irdma_cm_node *cm_node, const void *pdata, 2889 u8 plen) 2890 { 2891 int ret; 2892 int passive_state; 2893 2894 if (cm_node->tcp_cntxt.client) 2895 return 0; 2896 2897 irdma_cleanup_retrans_entry(cm_node); 2898 2899 passive_state = atomic_add_return(1, &cm_node->passive_state); 2900 if (passive_state == IRDMA_SEND_RESET_EVENT) { 2901 cm_node->state = IRDMA_CM_STATE_CLOSED; 2902 irdma_rem_ref_cm_node(cm_node); 2903 return 0; 2904 } 2905 2906 if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) { 2907 irdma_rem_ref_cm_node(cm_node); 2908 return 0; 2909 } 2910 2911 ret = irdma_send_mpa_reject(cm_node, pdata, plen); 2912 if (!ret) 2913 return 0; 2914 2915 cm_node->state = IRDMA_CM_STATE_CLOSED; 2916 if (irdma_send_reset(cm_node)) 2917 irdma_debug(&cm_node->iwdev->rf->sc_dev, 2918 IRDMA_DEBUG_CM, "send reset failed\n"); 2919 2920 return ret; 2921 } 2922 2923 /** 2924 * irdma_cm_close - close of cm connection 2925 * @cm_node: connection's node 2926 */ 2927 static int 2928 irdma_cm_close(struct irdma_cm_node *cm_node) 2929 { 2930 switch (cm_node->state) { 2931 case IRDMA_CM_STATE_SYN_RCVD: 2932 case IRDMA_CM_STATE_SYN_SENT: 2933 case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED: 2934 case IRDMA_CM_STATE_ESTABLISHED: 2935 case IRDMA_CM_STATE_ACCEPTING: 2936 case IRDMA_CM_STATE_MPAREQ_SENT: 2937 case IRDMA_CM_STATE_MPAREQ_RCVD: 2938 irdma_cleanup_retrans_entry(cm_node); 2939 irdma_send_reset(cm_node); 2940 break; 2941 case IRDMA_CM_STATE_CLOSE_WAIT: 2942 cm_node->state = IRDMA_CM_STATE_LAST_ACK; 2943 irdma_send_fin(cm_node); 2944 break; 2945 case IRDMA_CM_STATE_FIN_WAIT1: 2946 case IRDMA_CM_STATE_FIN_WAIT2: 2947 case IRDMA_CM_STATE_LAST_ACK: 2948 case IRDMA_CM_STATE_TIME_WAIT: 2949 case IRDMA_CM_STATE_CLOSING: 2950 return -EINVAL; 2951 case IRDMA_CM_STATE_LISTENING: 2952 irdma_cleanup_retrans_entry(cm_node); 2953 irdma_send_reset(cm_node); 2954 break; 2955 case IRDMA_CM_STATE_MPAREJ_RCVD: 2956 case IRDMA_CM_STATE_UNKNOWN: 2957 case IRDMA_CM_STATE_INITED: 2958 case IRDMA_CM_STATE_CLOSED: 2959 case IRDMA_CM_STATE_LISTENER_DESTROYED: 2960 irdma_rem_ref_cm_node(cm_node); 2961 break; 2962 case IRDMA_CM_STATE_OFFLOADED: 2963 if (cm_node->send_entry) 2964 irdma_debug(&cm_node->iwdev->rf->sc_dev, 2965 IRDMA_DEBUG_CM, "CM send_entry in OFFLOADED state\n"); 2966 irdma_rem_ref_cm_node(cm_node); 2967 break; 2968 } 2969 2970 return 0; 2971 } 2972 2973 /** 2974 * irdma_receive_ilq - recv an ETHERNET packet, and process it 2975 * through CM 2976 * @vsi: VSI structure of dev 2977 * @rbuf: receive buffer 2978 */ 2979 void 2980 irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf) 2981 { 2982 struct irdma_cm_node *cm_node; 2983 struct irdma_cm_listener *listener; 2984 struct ip *iph; 2985 struct ip6_hdr *ip6h; 2986 struct tcphdr *tcph; 2987 struct irdma_cm_info cm_info = {0}; 2988 struct irdma_device *iwdev = vsi->back_vsi; 2989 struct irdma_cm_core *cm_core = &iwdev->cm_core; 2990 struct ether_vlan_header *ethh; 2991 u16 vtag; 2992 2993 /* if vlan, then maclen = 18 else 14 */ 2994 iph = (struct ip *)rbuf->iph; 2995 irdma_debug_buf(vsi->dev, IRDMA_DEBUG_ILQ, "RECEIVE ILQ BUFFER", 2996 rbuf->mem.va, rbuf->totallen); 2997 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 2998 if (rbuf->vlan_valid) { 2999 vtag = rbuf->vlan_id; 3000 cm_info.user_pri = (vtag & EVL_PRI_MASK) >> 3001 VLAN_PRIO_SHIFT; 3002 cm_info.vlan_id = vtag & EVL_VLID_MASK; 3003 } else { 3004 cm_info.vlan_id = 0xFFFF; 3005 } 3006 } else { 3007 ethh = rbuf->mem.va; 3008 3009 if (ethh->evl_proto == htons(ETH_P_8021Q)) { 3010 vtag = ntohs(ethh->evl_tag); 3011 cm_info.user_pri = (vtag & EVL_PRI_MASK) >> 3012 VLAN_PRIO_SHIFT; 3013 cm_info.vlan_id = vtag & EVL_VLID_MASK; 3014 irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3015 "vlan_id=%d\n", 3016 cm_info.vlan_id); 3017 } else { 3018 cm_info.vlan_id = 0xFFFF; 3019 } 3020 } 3021 tcph = (struct tcphdr *)rbuf->tcph; 3022 3023 if (rbuf->ipv4) { 3024 cm_info.loc_addr[0] = ntohl(iph->ip_dst.s_addr); 3025 cm_info.rem_addr[0] = ntohl(iph->ip_src.s_addr); 3026 cm_info.ipv4 = true; 3027 cm_info.tos = iph->ip_tos; 3028 } else { 3029 ip6h = (struct ip6_hdr *)rbuf->iph; 3030 irdma_copy_ip_ntohl(cm_info.loc_addr, 3031 ip6h->ip6_dst.__u6_addr.__u6_addr32); 3032 irdma_copy_ip_ntohl(cm_info.rem_addr, 3033 ip6h->ip6_src.__u6_addr.__u6_addr32); 3034 cm_info.ipv4 = false; 3035 cm_info.tos = (ip6h->ip6_vfc << 4) | ip6h->ip6_flow; 3036 } 3037 cm_info.loc_port = ntohs(tcph->th_dport); 3038 cm_info.rem_port = ntohs(tcph->th_sport); 3039 cm_node = irdma_find_node(cm_core, cm_info.rem_port, cm_info.rem_addr, 3040 cm_info.loc_port, cm_info.loc_addr, cm_info.vlan_id); 3041 3042 if (!cm_node) { 3043 /* 3044 * Only type of packet accepted are for the PASSIVE open (syn only) 3045 */ 3046 if (!(tcph->th_flags & TH_SYN) || tcph->th_flags & TH_ACK) 3047 return; 3048 3049 listener = irdma_find_listener(cm_core, 3050 cm_info.loc_addr, 3051 cm_info.ipv4, 3052 cm_info.loc_port, 3053 cm_info.vlan_id, 3054 IRDMA_CM_LISTENER_ACTIVE_STATE); 3055 if (!listener) { 3056 cm_info.cm_id = NULL; 3057 irdma_debug(&cm_core->iwdev->rf->sc_dev, 3058 IRDMA_DEBUG_CM, "no listener found\n"); 3059 return; 3060 } 3061 3062 cm_info.cm_id = listener->cm_id; 3063 cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info, 3064 listener); 3065 if (!cm_node) { 3066 irdma_debug(&cm_core->iwdev->rf->sc_dev, 3067 IRDMA_DEBUG_CM, "allocate node failed\n"); 3068 atomic_dec(&listener->refcnt); 3069 return; 3070 } 3071 3072 if (!(tcph->th_flags & (TH_RST | TH_FIN))) { 3073 cm_node->state = IRDMA_CM_STATE_LISTENING; 3074 } else { 3075 irdma_rem_ref_cm_node(cm_node); 3076 return; 3077 } 3078 3079 atomic_inc(&cm_node->refcnt); 3080 } else if (cm_node->state == IRDMA_CM_STATE_OFFLOADED) { 3081 irdma_rem_ref_cm_node(cm_node); 3082 return; 3083 } 3084 3085 irdma_process_pkt(cm_node, rbuf); 3086 irdma_rem_ref_cm_node(cm_node); 3087 } 3088 3089 static int 3090 irdma_add_qh(struct irdma_cm_node *cm_node, bool active) 3091 { 3092 if (!active) 3093 irdma_add_conn_est_qh(cm_node); 3094 return 0; 3095 } 3096 3097 static void 3098 irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node) 3099 { 3100 } 3101 3102 /** 3103 * irdma_setup_cm_core - setup top level instance of a cm core 3104 * @iwdev: iwarp device structure 3105 * @rdma_ver: HW version 3106 */ 3107 int 3108 irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver) 3109 { 3110 struct irdma_cm_core *cm_core = &iwdev->cm_core; 3111 3112 cm_core->iwdev = iwdev; 3113 cm_core->dev = &iwdev->rf->sc_dev; 3114 3115 /* Handles CM event work items send to Iwarp core */ 3116 cm_core->event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0); 3117 if (!cm_core->event_wq) 3118 return -ENOMEM; 3119 3120 INIT_LIST_HEAD(&cm_core->listen_list); 3121 3122 timer_setup(&cm_core->tcp_timer, irdma_cm_timer_tick, 0); 3123 3124 spin_lock_init(&cm_core->ht_lock); 3125 spin_lock_init(&cm_core->listen_list_lock); 3126 spin_lock_init(&cm_core->apbvt_lock); 3127 switch (rdma_ver) { 3128 case IRDMA_GEN_1: 3129 cm_core->form_cm_frame = irdma_form_uda_cm_frame; 3130 cm_core->cm_create_ah = irdma_add_qh; 3131 cm_core->cm_free_ah = irdma_cm_free_ah_nop; 3132 break; 3133 case IRDMA_GEN_2: 3134 default: 3135 cm_core->form_cm_frame = irdma_form_ah_cm_frame; 3136 cm_core->cm_create_ah = irdma_cm_create_ah; 3137 cm_core->cm_free_ah = irdma_cm_free_ah; 3138 } 3139 3140 return 0; 3141 } 3142 3143 /** 3144 * irdma_cleanup_cm_core - deallocate a top level instance of a 3145 * cm core 3146 * @cm_core: cm's core 3147 */ 3148 void 3149 irdma_cleanup_cm_core(struct irdma_cm_core *cm_core) 3150 { 3151 if (!cm_core) 3152 return; 3153 3154 del_timer_sync(&cm_core->tcp_timer); 3155 3156 destroy_workqueue(cm_core->event_wq); 3157 cm_core->dev->ws_reset(&cm_core->iwdev->vsi); 3158 } 3159 3160 /** 3161 * irdma_init_tcp_ctx - setup qp context 3162 * @cm_node: connection's node 3163 * @tcp_info: offload info for tcp 3164 * @iwqp: associate qp for the connection 3165 */ 3166 static void 3167 irdma_init_tcp_ctx(struct irdma_cm_node *cm_node, 3168 struct irdma_tcp_offload_info *tcp_info, 3169 struct irdma_qp *iwqp) 3170 { 3171 tcp_info->ipv4 = cm_node->ipv4; 3172 tcp_info->drop_ooo_seg = !iwqp->iwdev->iw_ooo; 3173 tcp_info->wscale = true; 3174 tcp_info->ignore_tcp_opt = true; 3175 tcp_info->ignore_tcp_uns_opt = true; 3176 tcp_info->no_nagle = false; 3177 3178 tcp_info->ttl = IRDMA_DEFAULT_TTL; 3179 tcp_info->rtt_var = IRDMA_DEFAULT_RTT_VAR; 3180 tcp_info->ss_thresh = IRDMA_DEFAULT_SS_THRESH; 3181 tcp_info->rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH; 3182 3183 tcp_info->tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 3184 tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale; 3185 tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale; 3186 3187 tcp_info->snd_nxt = cm_node->tcp_cntxt.loc_seq_num; 3188 tcp_info->snd_wnd = cm_node->tcp_cntxt.snd_wnd; 3189 tcp_info->rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; 3190 tcp_info->snd_max = cm_node->tcp_cntxt.loc_seq_num; 3191 3192 tcp_info->snd_una = cm_node->tcp_cntxt.loc_seq_num; 3193 tcp_info->cwnd = 2 * cm_node->tcp_cntxt.mss; 3194 tcp_info->snd_wl1 = cm_node->tcp_cntxt.rcv_nxt; 3195 tcp_info->snd_wl2 = cm_node->tcp_cntxt.loc_seq_num; 3196 tcp_info->max_snd_window = cm_node->tcp_cntxt.max_snd_wnd; 3197 tcp_info->rcv_wnd = cm_node->tcp_cntxt.rcv_wnd 3198 << cm_node->tcp_cntxt.rcv_wscale; 3199 3200 tcp_info->flow_label = 0; 3201 tcp_info->snd_mss = (u32)cm_node->tcp_cntxt.mss; 3202 tcp_info->tos = cm_node->tos; 3203 if (cm_node->vlan_id < VLAN_N_VID) { 3204 tcp_info->insert_vlan_tag = true; 3205 tcp_info->vlan_tag = cm_node->vlan_id; 3206 tcp_info->vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT; 3207 } 3208 tcp_info->src_port = cm_node->loc_port; 3209 tcp_info->dst_port = cm_node->rem_port; 3210 tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf, 3211 cm_node->rem_addr, NULL, 3212 IRDMA_ARP_RESOLVE); 3213 if (cm_node->ipv4) { 3214 tcp_info->dest_ip_addr[3] = cm_node->rem_addr[0]; 3215 tcp_info->local_ipaddr[3] = cm_node->loc_addr[0]; 3216 } else { 3217 memcpy(tcp_info->dest_ip_addr, cm_node->rem_addr, 3218 sizeof(tcp_info->dest_ip_addr)); 3219 memcpy(tcp_info->local_ipaddr, cm_node->loc_addr, 3220 sizeof(tcp_info->local_ipaddr)); 3221 } 3222 } 3223 3224 /** 3225 * irdma_cm_init_tsa_conn - setup qp for RTS 3226 * @iwqp: associate qp for the connection 3227 * @cm_node: connection's node 3228 */ 3229 static void 3230 irdma_cm_init_tsa_conn(struct irdma_qp *iwqp, 3231 struct irdma_cm_node *cm_node) 3232 { 3233 struct irdma_iwarp_offload_info *iwarp_info; 3234 struct irdma_qp_host_ctx_info *ctx_info; 3235 3236 iwarp_info = &iwqp->iwarp_info; 3237 ctx_info = &iwqp->ctx_info; 3238 3239 ctx_info->tcp_info = &iwqp->tcp_info; 3240 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 3241 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 3242 3243 iwarp_info->ord_size = cm_node->ord_size; 3244 iwarp_info->ird_size = cm_node->ird_size; 3245 iwarp_info->rd_en = true; 3246 iwarp_info->rdmap_ver = 1; 3247 iwarp_info->ddp_ver = 1; 3248 iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id; 3249 3250 ctx_info->tcp_info_valid = true; 3251 ctx_info->iwarp_info_valid = true; 3252 ctx_info->user_pri = cm_node->user_pri; 3253 3254 irdma_init_tcp_ctx(cm_node, &iwqp->tcp_info, iwqp); 3255 if (cm_node->snd_mark_en) { 3256 iwarp_info->snd_mark_en = true; 3257 iwarp_info->snd_mark_offset = (iwqp->tcp_info.snd_nxt & SNDMARKER_SEQNMASK) + 3258 cm_node->lsmm_size; 3259 } 3260 3261 cm_node->state = IRDMA_CM_STATE_OFFLOADED; 3262 iwqp->tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 3263 iwqp->tcp_info.src_mac_addr_idx = iwqp->iwdev->mac_ip_table_idx; 3264 3265 if (cm_node->rcv_mark_en) { 3266 iwarp_info->rcv_mark_en = true; 3267 iwarp_info->align_hdrs = true; 3268 } 3269 3270 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 3271 3272 /* once tcp_info is set, no need to do it again */ 3273 ctx_info->tcp_info_valid = false; 3274 ctx_info->iwarp_info_valid = false; 3275 } 3276 3277 /** 3278 * irdma_cm_disconn - when a connection is being closed 3279 * @iwqp: associated qp for the connection 3280 */ 3281 void 3282 irdma_cm_disconn(struct irdma_qp *iwqp) 3283 { 3284 struct irdma_device *iwdev = iwqp->iwdev; 3285 struct disconn_work *work; 3286 unsigned long flags; 3287 3288 work = kzalloc(sizeof(*work), GFP_ATOMIC); 3289 if (!work) 3290 return; 3291 3292 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 3293 if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) { 3294 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 3295 irdma_debug(&iwdev->rf->sc_dev, 3296 IRDMA_DEBUG_CM, "qp_id %d is already freed\n", 3297 iwqp->ibqp.qp_num); 3298 kfree(work); 3299 return; 3300 } 3301 irdma_qp_add_ref(&iwqp->ibqp); 3302 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 3303 3304 work->iwqp = iwqp; 3305 INIT_WORK(&work->work, irdma_disconnect_worker); 3306 queue_work(iwdev->cleanup_wq, &work->work); 3307 } 3308 3309 /** 3310 * irdma_qp_disconnect - free qp and close cm 3311 * @iwqp: associate qp for the connection 3312 */ 3313 static void 3314 irdma_qp_disconnect(struct irdma_qp *iwqp) 3315 { 3316 struct irdma_device *iwdev = iwqp->iwdev; 3317 3318 iwqp->active_conn = 0; 3319 /* close the CM node down if it is still active */ 3320 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Call close API\n"); 3321 irdma_cm_close(iwqp->cm_node); 3322 } 3323 3324 /** 3325 * irdma_cm_disconn_true - called by worker thread to disconnect qp 3326 * @iwqp: associate qp for the connection 3327 */ 3328 static void 3329 irdma_cm_disconn_true(struct irdma_qp *iwqp) 3330 { 3331 struct iw_cm_id *cm_id; 3332 struct irdma_device *iwdev; 3333 struct irdma_sc_qp *qp = &iwqp->sc_qp; 3334 u16 last_ae; 3335 u8 original_hw_tcp_state; 3336 u8 original_ibqp_state; 3337 int disconn_status = 0; 3338 int issue_disconn = 0; 3339 int issue_close = 0; 3340 int issue_flush = 0; 3341 unsigned long flags; 3342 int err; 3343 3344 iwdev = iwqp->iwdev; 3345 spin_lock_irqsave(&iwqp->lock, flags); 3346 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 3347 struct ib_qp_attr attr; 3348 3349 if (iwqp->flush_issued || iwqp->sc_qp.qp_uk.destroy_pending) { 3350 spin_unlock_irqrestore(&iwqp->lock, flags); 3351 return; 3352 } 3353 3354 spin_unlock_irqrestore(&iwqp->lock, flags); 3355 3356 attr.qp_state = IB_QPS_ERR; 3357 irdma_modify_qp_roce(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); 3358 irdma_ib_qp_event(iwqp, qp->event_type); 3359 return; 3360 } 3361 3362 cm_id = iwqp->cm_id; 3363 original_hw_tcp_state = iwqp->hw_tcp_state; 3364 original_ibqp_state = iwqp->ibqp_state; 3365 last_ae = iwqp->last_aeq; 3366 3367 if (qp->term_flags) { 3368 issue_disconn = 1; 3369 issue_close = 1; 3370 iwqp->cm_id = NULL; 3371 irdma_terminate_del_timer(qp); 3372 if (!iwqp->flush_issued) { 3373 iwqp->flush_issued = 1; 3374 issue_flush = 1; 3375 } 3376 } else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) || 3377 ((original_ibqp_state == IB_QPS_RTS) && 3378 (last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) { 3379 issue_disconn = 1; 3380 if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET) 3381 disconn_status = -ECONNRESET; 3382 } 3383 3384 if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED || 3385 original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT || 3386 last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE || 3387 last_ae == IRDMA_AE_BAD_CLOSE || 3388 last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset || !cm_id) { 3389 issue_close = 1; 3390 iwqp->cm_id = NULL; 3391 qp->term_flags = 0; 3392 if (!iwqp->flush_issued) { 3393 iwqp->flush_issued = 1; 3394 issue_flush = 1; 3395 } 3396 } 3397 3398 spin_unlock_irqrestore(&iwqp->lock, flags); 3399 if (issue_flush && !iwqp->sc_qp.qp_uk.destroy_pending) { 3400 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ | 3401 IRDMA_FLUSH_WAIT); 3402 3403 if (qp->term_flags) 3404 irdma_ib_qp_event(iwqp, qp->event_type); 3405 } 3406 3407 if (!cm_id || !cm_id->event_handler) 3408 return; 3409 3410 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 3411 if (!iwqp->cm_node) { 3412 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 3413 return; 3414 } 3415 atomic_inc(&iwqp->cm_node->refcnt); 3416 3417 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 3418 3419 if (issue_disconn) { 3420 err = irdma_send_cm_event(iwqp->cm_node, cm_id, 3421 IW_CM_EVENT_DISCONNECT, 3422 disconn_status); 3423 if (err) 3424 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3425 "disconnect event failed: - cm_id = %p\n", 3426 cm_id); 3427 } 3428 if (issue_close) { 3429 cm_id->provider_data = iwqp; 3430 err = irdma_send_cm_event(iwqp->cm_node, cm_id, 3431 IW_CM_EVENT_CLOSE, 0); 3432 if (err) 3433 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3434 "close event failed: - cm_id = %p\n", 3435 cm_id); 3436 irdma_qp_disconnect(iwqp); 3437 } 3438 irdma_rem_ref_cm_node(iwqp->cm_node); 3439 } 3440 3441 /** 3442 * irdma_disconnect_worker - worker for connection close 3443 * @work: points or disconn structure 3444 */ 3445 static void 3446 irdma_disconnect_worker(struct work_struct *work) 3447 { 3448 struct disconn_work *dwork = container_of(work, struct disconn_work, work); 3449 struct irdma_qp *iwqp = dwork->iwqp; 3450 3451 kfree(dwork); 3452 irdma_cm_disconn_true(iwqp); 3453 irdma_qp_rem_ref(&iwqp->ibqp); 3454 } 3455 3456 /** 3457 * irdma_free_lsmm_rsrc - free lsmm memory and deregister 3458 * @iwqp: associate qp for the connection 3459 */ 3460 void 3461 irdma_free_lsmm_rsrc(struct irdma_qp *iwqp) 3462 { 3463 struct irdma_device *iwdev; 3464 3465 iwdev = iwqp->iwdev; 3466 3467 if (iwqp->ietf_mem.va) { 3468 if (iwqp->lsmm_mr) 3469 kc_free_lsmm_dereg_mr(iwdev, iwqp); 3470 irdma_free_dma_mem(iwdev->rf->sc_dev.hw, 3471 &iwqp->ietf_mem); 3472 iwqp->ietf_mem.va = NULL; 3473 } 3474 } 3475 3476 /** 3477 * irdma_accept - registered call for connection to be accepted 3478 * @cm_id: cm information for passive connection 3479 * @conn_param: accpet parameters 3480 */ 3481 int 3482 irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 3483 { 3484 struct ib_qp *ibqp; 3485 struct irdma_qp *iwqp; 3486 struct irdma_device *iwdev; 3487 struct irdma_sc_dev *dev; 3488 struct irdma_cm_node *cm_node; 3489 struct ib_qp_attr attr = {0}; 3490 int passive_state; 3491 struct ib_mr *ibmr; 3492 struct irdma_pd *iwpd; 3493 u16 buf_len = 0; 3494 struct irdma_kmem_info accept; 3495 u64 tagged_offset; 3496 int wait_ret; 3497 int ret = 0; 3498 3499 ibqp = irdma_get_qp(cm_id->device, conn_param->qpn); 3500 if (!ibqp) 3501 return -EINVAL; 3502 3503 iwqp = to_iwqp(ibqp); 3504 iwdev = iwqp->iwdev; 3505 dev = &iwdev->rf->sc_dev; 3506 cm_node = cm_id->provider_data; 3507 3508 if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) { 3509 cm_node->ipv4 = true; 3510 cm_node->vlan_id = irdma_get_vlan_ipv4(cm_node->loc_addr); 3511 } else { 3512 cm_node->ipv4 = false; 3513 irdma_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id, 3514 NULL); 3515 } 3516 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Accept vlan_id=%d\n", 3517 cm_node->vlan_id); 3518 3519 if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) { 3520 ret = -EINVAL; 3521 goto error; 3522 } 3523 3524 passive_state = atomic_add_return(1, &cm_node->passive_state); 3525 if (passive_state == IRDMA_SEND_RESET_EVENT) { 3526 ret = -ECONNRESET; 3527 goto error; 3528 } 3529 3530 buf_len = conn_param->private_data_len + IRDMA_MAX_IETF_SIZE; 3531 iwqp->ietf_mem.size = buf_len; 3532 iwqp->ietf_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->ietf_mem, 3533 iwqp->ietf_mem.size, 1); 3534 if (!iwqp->ietf_mem.va) { 3535 ret = -ENOMEM; 3536 goto error; 3537 } 3538 3539 cm_node->pdata.size = conn_param->private_data_len; 3540 accept.addr = iwqp->ietf_mem.va; 3541 accept.size = irdma_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY); 3542 memcpy((u8 *)accept.addr + accept.size, conn_param->private_data, 3543 conn_param->private_data_len); 3544 3545 if (cm_node->dev->ws_add(iwqp->sc_qp.vsi, cm_node->user_pri)) { 3546 ret = -ENOMEM; 3547 goto error; 3548 } 3549 iwqp->sc_qp.user_pri = cm_node->user_pri; 3550 irdma_qp_add_qos(&iwqp->sc_qp); 3551 if (cm_node->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) 3552 iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); 3553 /* setup our first outgoing iWarp send WQE (the IETF frame response) */ 3554 iwpd = iwqp->iwpd; 3555 tagged_offset = (uintptr_t)iwqp->ietf_mem.va; 3556 ibmr = irdma_reg_phys_mr(&iwpd->ibpd, iwqp->ietf_mem.pa, buf_len, 3557 IB_ACCESS_LOCAL_WRITE, &tagged_offset); 3558 if (IS_ERR(ibmr)) { 3559 ret = -ENOMEM; 3560 goto error; 3561 } 3562 3563 ibmr->pd = &iwpd->ibpd; 3564 ibmr->device = iwpd->ibpd.device; 3565 iwqp->lsmm_mr = ibmr; 3566 if (iwqp->page) 3567 iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page); 3568 3569 cm_node->lsmm_size = accept.size + conn_param->private_data_len; 3570 irdma_sc_send_lsmm(&iwqp->sc_qp, iwqp->ietf_mem.va, cm_node->lsmm_size, 3571 ibmr->lkey); 3572 3573 if (iwqp->page) 3574 kunmap_local(iwqp->sc_qp.qp_uk.sq_base); 3575 3576 iwqp->cm_id = cm_id; 3577 cm_node->cm_id = cm_id; 3578 3579 cm_id->provider_data = iwqp; 3580 iwqp->active_conn = 0; 3581 iwqp->cm_node = cm_node; 3582 cm_node->iwqp = iwqp; 3583 irdma_cm_init_tsa_conn(iwqp, cm_node); 3584 irdma_qp_add_ref(&iwqp->ibqp); 3585 cm_id->add_ref(cm_id); 3586 3587 attr.qp_state = IB_QPS_RTS; 3588 cm_node->qhash_set = false; 3589 cm_node->cm_core->cm_free_ah(cm_node); 3590 3591 irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); 3592 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) { 3593 wait_ret = wait_event_interruptible_timeout(iwqp->waitq, 3594 iwqp->rts_ae_rcvd, 3595 IRDMA_MAX_TIMEOUT); 3596 if (!wait_ret) { 3597 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3598 "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n", 3599 cm_node, cm_node->loc_port, 3600 cm_node->rem_port, cm_node->cm_id); 3601 ret = -ECONNRESET; 3602 goto error; 3603 } 3604 } 3605 3606 irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0); 3607 cm_node->accelerated = true; 3608 complete(&cm_node->establish_comp); 3609 3610 if (cm_node->accept_pend) { 3611 atomic_dec(&cm_node->listener->pend_accepts_cnt); 3612 cm_node->accept_pend = 0; 3613 } 3614 3615 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3616 "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n", 3617 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, 3618 cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num); 3619 cm_node->cm_core->stats_accepts++; 3620 3621 return 0; 3622 error: 3623 irdma_free_lsmm_rsrc(iwqp); 3624 irdma_rem_ref_cm_node(cm_node); 3625 3626 return ret; 3627 } 3628 3629 /** 3630 * irdma_reject - registered call for connection to be rejected 3631 * @cm_id: cm information for passive connection 3632 * @pdata: private data to be sent 3633 * @pdata_len: private data length 3634 */ 3635 int 3636 irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 3637 { 3638 struct irdma_device *iwdev; 3639 struct irdma_cm_node *cm_node; 3640 3641 cm_node = cm_id->provider_data; 3642 cm_node->pdata.size = pdata_len; 3643 3644 iwdev = to_iwdev(cm_id->device); 3645 if (!iwdev) 3646 return -EINVAL; 3647 3648 cm_node->cm_core->stats_rejects++; 3649 3650 if (pdata_len + sizeof(struct ietf_mpa_v2) > IRDMA_MAX_CM_BUF) 3651 return -EINVAL; 3652 3653 return irdma_cm_reject(cm_node, pdata, pdata_len); 3654 } 3655 3656 /** 3657 * irdma_connect - registered call for connection to be established 3658 * @cm_id: cm information for passive connection 3659 * @conn_param: Information about the connection 3660 */ 3661 int 3662 irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 3663 { 3664 struct ib_qp *ibqp; 3665 struct irdma_qp *iwqp; 3666 struct irdma_device *iwdev; 3667 struct irdma_cm_node *cm_node; 3668 struct irdma_cm_info cm_info; 3669 struct sockaddr_in *laddr; 3670 struct sockaddr_in *raddr; 3671 struct sockaddr_in6 *laddr6; 3672 struct sockaddr_in6 *raddr6; 3673 int ret = 0; 3674 3675 ibqp = irdma_get_qp(cm_id->device, conn_param->qpn); 3676 if (!ibqp) 3677 return -EINVAL; 3678 iwqp = to_iwqp(ibqp); 3679 if (!iwqp) 3680 return -EINVAL; 3681 iwdev = iwqp->iwdev; 3682 if (!iwdev) 3683 return -EINVAL; 3684 3685 laddr = (struct sockaddr_in *)&cm_id->m_local_addr; 3686 raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; 3687 laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr; 3688 raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr; 3689 3690 if (!(laddr->sin_port) || !(raddr->sin_port)) 3691 return -EINVAL; 3692 3693 iwqp->active_conn = 1; 3694 iwqp->cm_id = NULL; 3695 cm_id->provider_data = iwqp; 3696 3697 /* set up the connection params for the node */ 3698 if (cm_id->remote_addr.ss_family == AF_INET) { 3699 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4) 3700 return -EINVAL; 3701 3702 cm_info.ipv4 = true; 3703 memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr)); 3704 memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr)); 3705 cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr); 3706 cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr); 3707 cm_info.loc_port = ntohs(laddr->sin_port); 3708 cm_info.rem_port = ntohs(raddr->sin_port); 3709 cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr); 3710 } else { 3711 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6) 3712 return -EINVAL; 3713 3714 cm_info.ipv4 = false; 3715 irdma_copy_ip_ntohl(cm_info.loc_addr, 3716 laddr6->sin6_addr.__u6_addr.__u6_addr32); 3717 irdma_copy_ip_ntohl(cm_info.rem_addr, 3718 raddr6->sin6_addr.__u6_addr.__u6_addr32); 3719 cm_info.loc_port = ntohs(laddr6->sin6_port); 3720 cm_info.rem_port = ntohs(raddr6->sin6_port); 3721 irdma_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id, NULL); 3722 } 3723 cm_info.cm_id = cm_id; 3724 cm_info.qh_qpid = iwdev->vsi.ilq->qp_id; 3725 cm_info.tos = cm_id->tos; 3726 if (iwdev->vsi.dscp_mode) { 3727 cm_info.user_pri = 3728 iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)]; 3729 } else { 3730 cm_info.user_pri = rt_tos2priority(cm_id->tos); 3731 cm_info.user_pri = irdma_get_egress_vlan_prio(cm_info.loc_addr, 3732 cm_info.user_pri, 3733 cm_info.ipv4); 3734 } 3735 3736 if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri)) 3737 return -ENOMEM; 3738 iwqp->sc_qp.user_pri = cm_info.user_pri; 3739 irdma_qp_add_qos(&iwqp->sc_qp); 3740 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) 3741 iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); 3742 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, 3743 "TOS:[%d] UP:[%d]\n", cm_id->tos, 3744 cm_info.user_pri); 3745 3746 ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info, 3747 &cm_node); 3748 if (ret) 3749 return ret; 3750 ret = cm_node->cm_core->cm_create_ah(cm_node, true); 3751 if (ret) 3752 goto err; 3753 if (irdma_manage_qhash(iwdev, &cm_info, 3754 IRDMA_QHASH_TYPE_TCP_ESTABLISHED, 3755 IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true)) { 3756 ret = -EINVAL; 3757 goto err; 3758 } 3759 cm_node->qhash_set = true; 3760 3761 cm_node->apbvt_entry = irdma_add_apbvt(iwdev, cm_info.loc_port); 3762 if (!cm_node->apbvt_entry) { 3763 ret = -EINVAL; 3764 goto err; 3765 } 3766 3767 cm_node->apbvt_set = true; 3768 iwqp->cm_node = cm_node; 3769 cm_node->iwqp = iwqp; 3770 iwqp->cm_id = cm_id; 3771 irdma_qp_add_ref(&iwqp->ibqp); 3772 cm_id->add_ref(cm_id); 3773 3774 if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) { 3775 cm_node->state = IRDMA_CM_STATE_SYN_SENT; 3776 ret = irdma_send_syn(cm_node, 0); 3777 if (ret) 3778 goto err; 3779 } 3780 3781 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3782 "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n", 3783 cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, 3784 cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num); 3785 3786 return 0; 3787 3788 err: 3789 if (cm_info.ipv4) 3790 irdma_debug(&iwdev->rf->sc_dev, 3791 IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI4", 3792 cm_info.rem_addr); 3793 else 3794 irdma_debug(&iwdev->rf->sc_dev, 3795 IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI6", 3796 cm_info.rem_addr); 3797 irdma_rem_ref_cm_node(cm_node); 3798 iwdev->cm_core.stats_connect_errs++; 3799 3800 return ret; 3801 } 3802 3803 /** 3804 * irdma_create_listen - registered call creating listener 3805 * @cm_id: cm information for passive connection 3806 * @backlog: to max accept pending count 3807 */ 3808 int 3809 irdma_create_listen(struct iw_cm_id *cm_id, int backlog) 3810 { 3811 struct irdma_device *iwdev; 3812 struct irdma_cm_listener *cm_listen_node; 3813 struct irdma_cm_info cm_info = {0}; 3814 struct sockaddr_in *laddr; 3815 struct sockaddr_in6 *laddr6; 3816 bool wildcard = false; 3817 int err; 3818 3819 iwdev = to_iwdev(cm_id->device); 3820 if (!iwdev) 3821 return -EINVAL; 3822 3823 laddr = (struct sockaddr_in *)&cm_id->m_local_addr; 3824 laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr; 3825 cm_info.qh_qpid = iwdev->vsi.ilq->qp_id; 3826 3827 if (laddr->sin_family == AF_INET) { 3828 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4) 3829 return -EINVAL; 3830 3831 cm_info.ipv4 = true; 3832 cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr); 3833 cm_info.loc_port = ntohs(laddr->sin_port); 3834 3835 if (laddr->sin_addr.s_addr != htonl(INADDR_ANY)) { 3836 cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr); 3837 } else { 3838 cm_info.vlan_id = 0xFFFF; 3839 wildcard = true; 3840 } 3841 } else { 3842 if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6) 3843 return -EINVAL; 3844 3845 cm_info.ipv4 = false; 3846 irdma_copy_ip_ntohl(cm_info.loc_addr, 3847 laddr6->sin6_addr.__u6_addr.__u6_addr32); 3848 cm_info.loc_port = ntohs(laddr6->sin6_port); 3849 if (!IN6_IS_ADDR_UNSPECIFIED(&laddr6->sin6_addr)) { 3850 irdma_netdev_vlan_ipv6(cm_info.loc_addr, 3851 &cm_info.vlan_id, NULL); 3852 } else { 3853 cm_info.vlan_id = 0xFFFF; 3854 wildcard = true; 3855 } 3856 } 3857 3858 if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 3859 cm_info.vlan_id = 0; 3860 cm_info.backlog = backlog; 3861 cm_info.cm_id = cm_id; 3862 3863 cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev, 3864 &cm_info); 3865 if (!cm_listen_node) { 3866 irdma_debug(&iwdev->rf->sc_dev, 3867 IRDMA_DEBUG_CM, "cm_listen_node == NULL\n"); 3868 return -ENOMEM; 3869 } 3870 3871 cm_id->provider_data = cm_listen_node; 3872 3873 cm_listen_node->tos = cm_id->tos; 3874 if (iwdev->vsi.dscp_mode) 3875 cm_listen_node->user_pri = 3876 iwdev->vsi.dscp_map[irdma_tos2dscp(cm_id->tos)]; 3877 else 3878 cm_listen_node->user_pri = rt_tos2priority(cm_id->tos); 3879 cm_info.user_pri = cm_listen_node->user_pri; 3880 if (!cm_listen_node->reused_node) { 3881 if (wildcard) { 3882 err = irdma_add_mqh(iwdev, &cm_info, cm_listen_node); 3883 if (err) 3884 goto error; 3885 } else { 3886 if (!iwdev->vsi.dscp_mode) 3887 cm_info.user_pri = cm_listen_node->user_pri = 3888 irdma_get_egress_vlan_prio(cm_info.loc_addr, 3889 cm_info.user_pri, 3890 cm_info.ipv4); 3891 err = irdma_manage_qhash(iwdev, &cm_info, 3892 IRDMA_QHASH_TYPE_TCP_SYN, 3893 IRDMA_QHASH_MANAGE_TYPE_ADD, 3894 NULL, true); 3895 if (err) 3896 goto error; 3897 3898 cm_listen_node->qhash_set = true; 3899 } 3900 3901 cm_listen_node->apbvt_entry = irdma_add_apbvt(iwdev, 3902 cm_info.loc_port); 3903 if (!cm_listen_node->apbvt_entry) 3904 goto error; 3905 } 3906 cm_id->add_ref(cm_id); 3907 cm_listen_node->cm_core->stats_listen_created++; 3908 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 3909 "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n", 3910 cm_listen_node->loc_port, cm_listen_node->loc_addr, 3911 cm_listen_node, cm_listen_node->cm_id, 3912 cm_listen_node->qhash_set, cm_listen_node->vlan_id); 3913 3914 return 0; 3915 3916 error: 3917 3918 irdma_cm_del_listen(&iwdev->cm_core, cm_listen_node, false); 3919 3920 return -EINVAL; 3921 } 3922 3923 /** 3924 * irdma_destroy_listen - registered call to destroy listener 3925 * @cm_id: cm information for passive connection 3926 */ 3927 int 3928 irdma_destroy_listen(struct iw_cm_id *cm_id) 3929 { 3930 struct irdma_device *iwdev; 3931 3932 iwdev = to_iwdev(cm_id->device); 3933 if (cm_id->provider_data) 3934 irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data, 3935 true); 3936 else 3937 irdma_debug(&iwdev->rf->sc_dev, 3938 IRDMA_DEBUG_CM, "cm_id->provider_data was NULL\n"); 3939 3940 cm_id->rem_ref(cm_id); 3941 3942 return 0; 3943 } 3944 3945 /** 3946 * irdma_iw_teardown_list_prep - add conn nodes slated for tear 3947 * down to list 3948 * @cm_core: cm's core 3949 * @teardown_list: a list to which cm_node will be selected 3950 * @ipaddr: pointer to ip address 3951 * @nfo: pointer to cm_info structure instance 3952 * @disconnect_all: flag indicating disconnect all QPs 3953 */ 3954 static void 3955 irdma_iw_teardown_list_prep(struct irdma_cm_core *cm_core, 3956 struct list_head *teardown_list, 3957 u32 *ipaddr, 3958 struct irdma_cm_info *nfo, 3959 bool disconnect_all) 3960 { 3961 struct irdma_cm_node *cm_node; 3962 int bkt; 3963 3964 HASH_FOR_EACH_RCU(cm_core->cm_hash_tbl, bkt, cm_node, list) { 3965 if ((disconnect_all || 3966 (nfo->vlan_id == cm_node->vlan_id && 3967 !memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16))) && 3968 atomic_inc_not_zero(&cm_node->refcnt)) 3969 list_add(&cm_node->teardown_entry, teardown_list); 3970 } 3971 } 3972 3973 static inline bool 3974 irdma_ip_vlan_match(u32 *ip1, u16 vlan_id1, 3975 bool check_vlan, u32 *ip2, 3976 u16 vlan_id2, bool ipv4) 3977 { 3978 return (!check_vlan || vlan_id1 == vlan_id2) && 3979 !memcmp(ip1, ip2, ipv4 ? 4 : 16); 3980 } 3981 3982 /** 3983 * irdma_roce_teardown_list_prep - add conn nodes slated for 3984 * tear down to list 3985 * @iwdev: RDMA device 3986 * @teardown_list: a list to which cm_node will be selected 3987 * @ipaddr: pointer to ip address 3988 * @nfo: pointer to cm_info structure instance 3989 * @disconnect_all: flag indicating disconnect all QPs 3990 */ 3991 static void 3992 irdma_roce_teardown_list_prep(struct irdma_device *iwdev, 3993 struct list_head *teardown_list, 3994 u32 *ipaddr, 3995 struct irdma_cm_info *nfo, 3996 bool disconnect_all) 3997 { 3998 struct irdma_sc_vsi *vsi = &iwdev->vsi; 3999 struct irdma_sc_qp *sc_qp; 4000 struct list_head *list_node; 4001 struct irdma_qp *qp; 4002 unsigned long flags; 4003 int i; 4004 4005 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 4006 mutex_lock(&vsi->qos[i].qos_mutex); 4007 list_for_each(list_node, &vsi->qos[i].qplist) { 4008 u32 qp_ip[4]; 4009 4010 sc_qp = container_of(list_node, struct irdma_sc_qp, 4011 list); 4012 if (sc_qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_RC) 4013 continue; 4014 4015 qp = sc_qp->qp_uk.back_qp; 4016 if (!disconnect_all) { 4017 if (nfo->ipv4) 4018 qp_ip[0] = qp->udp_info.local_ipaddr[3]; 4019 else 4020 memcpy(qp_ip, 4021 &qp->udp_info.local_ipaddr[0], 4022 sizeof(qp_ip)); 4023 } 4024 4025 if (disconnect_all || 4026 irdma_ip_vlan_match(qp_ip, 4027 qp->udp_info.vlan_tag & EVL_VLID_MASK, 4028 qp->udp_info.insert_vlan_tag, 4029 ipaddr, nfo->vlan_id, nfo->ipv4)) { 4030 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 4031 if (iwdev->rf->qp_table[sc_qp->qp_uk.qp_id]) { 4032 irdma_qp_add_ref(&qp->ibqp); 4033 list_add(&qp->teardown_entry, teardown_list); 4034 } 4035 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 4036 } 4037 } 4038 mutex_unlock(&vsi->qos[i].qos_mutex); 4039 } 4040 } 4041 4042 /** 4043 * irdma_cm_event_connected - handle connected active node 4044 * @event: the info for cm_node of connection 4045 */ 4046 static void 4047 irdma_cm_event_connected(struct irdma_cm_event *event) 4048 { 4049 struct irdma_qp *iwqp; 4050 struct irdma_device *iwdev; 4051 struct irdma_cm_node *cm_node; 4052 struct irdma_sc_dev *dev; 4053 struct ib_qp_attr attr = {0}; 4054 struct iw_cm_id *cm_id; 4055 int status; 4056 bool read0; 4057 int wait_ret = 0; 4058 4059 cm_node = event->cm_node; 4060 cm_id = cm_node->cm_id; 4061 iwqp = cm_id->provider_data; 4062 iwdev = iwqp->iwdev; 4063 dev = &iwdev->rf->sc_dev; 4064 if (iwqp->sc_qp.qp_uk.destroy_pending) { 4065 status = -ETIMEDOUT; 4066 goto error; 4067 } 4068 4069 irdma_cm_init_tsa_conn(iwqp, cm_node); 4070 read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO); 4071 if (iwqp->page) 4072 iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page); 4073 irdma_sc_send_rtt(&iwqp->sc_qp, read0); 4074 if (iwqp->page) 4075 kunmap_local(iwqp->sc_qp.qp_uk.sq_base); 4076 4077 attr.qp_state = IB_QPS_RTS; 4078 cm_node->qhash_set = false; 4079 irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL); 4080 if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) { 4081 wait_ret = wait_event_interruptible_timeout(iwqp->waitq, 4082 iwqp->rts_ae_rcvd, 4083 IRDMA_MAX_TIMEOUT); 4084 if (!wait_ret) 4085 irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4086 "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n", 4087 cm_node, cm_node->loc_port, 4088 cm_node->rem_port, cm_node->cm_id); 4089 } 4090 4091 irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0); 4092 cm_node->accelerated = true; 4093 complete(&cm_node->establish_comp); 4094 cm_node->cm_core->cm_free_ah(cm_node); 4095 return; 4096 4097 error: 4098 iwqp->cm_id = NULL; 4099 cm_id->provider_data = NULL; 4100 irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 4101 status); 4102 irdma_rem_ref_cm_node(event->cm_node); 4103 } 4104 4105 /** 4106 * irdma_cm_event_reset - handle reset 4107 * @event: the info for cm_node of connection 4108 */ 4109 static void 4110 irdma_cm_event_reset(struct irdma_cm_event *event) 4111 { 4112 struct irdma_cm_node *cm_node = event->cm_node; 4113 struct iw_cm_id *cm_id = cm_node->cm_id; 4114 struct irdma_qp *iwqp; 4115 4116 if (!cm_id) 4117 return; 4118 4119 iwqp = cm_id->provider_data; 4120 if (!iwqp) 4121 return; 4122 4123 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4124 "reset event %p - cm_id = %p\n", 4125 event->cm_node, cm_id); 4126 iwqp->cm_id = NULL; 4127 4128 irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT, 4129 -ECONNRESET); 4130 irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0); 4131 } 4132 4133 /** 4134 * irdma_cm_event_handler - send event to cm upper layer 4135 * @work: pointer of cm event info. 4136 */ 4137 static void 4138 irdma_cm_event_handler(struct work_struct *work) 4139 { 4140 struct irdma_cm_event *event = container_of(work, struct irdma_cm_event, event_work); 4141 struct irdma_cm_node *cm_node; 4142 4143 if (!event || !event->cm_node || !event->cm_node->cm_core) 4144 return; 4145 4146 cm_node = event->cm_node; 4147 4148 switch (event->type) { 4149 case IRDMA_CM_EVENT_MPA_REQ: 4150 irdma_send_cm_event(cm_node, cm_node->cm_id, 4151 IW_CM_EVENT_CONNECT_REQUEST, 0); 4152 break; 4153 case IRDMA_CM_EVENT_RESET: 4154 irdma_cm_event_reset(event); 4155 break; 4156 case IRDMA_CM_EVENT_CONNECTED: 4157 if (!event->cm_node->cm_id || 4158 event->cm_node->state != IRDMA_CM_STATE_OFFLOADED) 4159 break; 4160 irdma_cm_event_connected(event); 4161 break; 4162 case IRDMA_CM_EVENT_MPA_REJECT: 4163 if (!event->cm_node->cm_id || 4164 cm_node->state == IRDMA_CM_STATE_OFFLOADED) 4165 break; 4166 irdma_send_cm_event(cm_node, cm_node->cm_id, 4167 IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); 4168 break; 4169 case IRDMA_CM_EVENT_ABORTED: 4170 if (!event->cm_node->cm_id || 4171 event->cm_node->state == IRDMA_CM_STATE_OFFLOADED) 4172 break; 4173 irdma_event_connect_error(event); 4174 break; 4175 default: 4176 irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, 4177 "bad event type = %d\n", 4178 event->type); 4179 break; 4180 } 4181 4182 irdma_rem_ref_cm_node(event->cm_node); 4183 kfree(event); 4184 } 4185 4186 /** 4187 * irdma_cm_post_event - queue event request for worker thread 4188 * @event: cm node's info for up event call 4189 */ 4190 static void 4191 irdma_cm_post_event(struct irdma_cm_event *event) 4192 { 4193 atomic_inc(&event->cm_node->refcnt); 4194 INIT_WORK(&event->event_work, irdma_cm_event_handler); 4195 queue_work(event->cm_node->cm_core->event_wq, &event->event_work); 4196 } 4197 4198 /** 4199 * irdma_cm_teardown_connections - teardown QPs 4200 * @iwdev: device pointer 4201 * @ipaddr: Pointer to IPv4 or IPv6 address 4202 * @nfo: Connection info 4203 * @disconnect_all: flag indicating disconnect all QPs 4204 * 4205 * teardown QPs where source or destination addr matches ip addr 4206 */ 4207 static void __unused 4208 irdma_cm_teardown_connections(struct irdma_device *iwdev, 4209 u32 *ipaddr, 4210 struct irdma_cm_info *nfo, 4211 bool disconnect_all) 4212 { 4213 struct irdma_cm_core *cm_core = &iwdev->cm_core; 4214 struct list_head *list_core_temp; 4215 struct list_head *list_node; 4216 struct irdma_cm_node *cm_node; 4217 struct list_head teardown_list; 4218 struct ib_qp_attr attr; 4219 struct irdma_qp *qp; 4220 4221 INIT_LIST_HEAD(&teardown_list); 4222 4223 rcu_read_lock(); 4224 irdma_iw_teardown_list_prep(cm_core, &teardown_list, ipaddr, nfo, disconnect_all); 4225 rcu_read_unlock(); 4226 4227 attr.qp_state = IB_QPS_ERR; 4228 list_for_each_safe(list_node, list_core_temp, &teardown_list) { 4229 cm_node = container_of(list_node, struct irdma_cm_node, 4230 teardown_entry); 4231 irdma_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL); 4232 if (iwdev->rf->reset) 4233 irdma_cm_disconn(cm_node->iwqp); 4234 irdma_rem_ref_cm_node(cm_node); 4235 } 4236 4237 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 4238 return; 4239 4240 INIT_LIST_HEAD(&teardown_list); 4241 irdma_roce_teardown_list_prep(iwdev, &teardown_list, ipaddr, nfo, disconnect_all); 4242 4243 list_for_each_safe(list_node, list_core_temp, &teardown_list) { 4244 qp = container_of(list_node, struct irdma_qp, teardown_entry); 4245 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 4246 irdma_ib_qp_event(qp, IRDMA_QP_EVENT_CATASTROPHIC); 4247 irdma_qp_rem_ref(&qp->ibqp); 4248 } 4249 } 4250