1 /*- 2 * Copyright (c) 2012 The FreeBSD Foundation 3 * Copyright (c) 2015 Chelsio Communications, Inc. 4 * All rights reserved. 5 * 6 * This software was developed by Edward Tomasz Napierala under sponsorship 7 * from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32 /* 33 * cxgbei implementation of iSCSI Common Layer kobj(9) interface. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include "opt_inet.h" 40 #include "opt_inet6.h" 41 42 #ifdef TCP_OFFLOAD 43 #include <sys/param.h> 44 #include <sys/bio.h> 45 #include <sys/capsicum.h> 46 #include <sys/condvar.h> 47 #include <sys/conf.h> 48 #include <sys/file.h> 49 #include <sys/kernel.h> 50 #include <sys/kthread.h> 51 #include <sys/ktr.h> 52 #include <sys/lock.h> 53 #include <sys/mbuf.h> 54 #include <sys/mutex.h> 55 #include <sys/module.h> 56 #include <sys/protosw.h> 57 #include <sys/socket.h> 58 #include <sys/socketvar.h> 59 #include <sys/sysctl.h> 60 #include <sys/systm.h> 61 #include <sys/sx.h> 62 #include <sys/uio.h> 63 #include <machine/bus.h> 64 #include <vm/vm.h> 65 #include <vm/vm_page.h> 66 #include <vm/pmap.h> 67 #include <netinet/in.h> 68 #include <netinet/in_pcb.h> 69 #include <netinet/tcp.h> 70 #include <netinet/tcp_var.h> 71 #include <netinet/toecore.h> 72 73 #include <dev/iscsi/icl.h> 74 #include <dev/iscsi/iscsi_proto.h> 75 #include <icl_conn_if.h> 76 77 #include <cam/scsi/scsi_all.h> 78 #include <cam/scsi/scsi_da.h> 79 #include <cam/ctl/ctl_io.h> 80 #include <cam/ctl/ctl.h> 81 #include <cam/ctl/ctl_backend.h> 82 #include <cam/ctl/ctl_error.h> 83 #include <cam/ctl/ctl_frontend.h> 84 #include <cam/ctl/ctl_debug.h> 85 #include <cam/ctl/ctl_ha.h> 86 #include <cam/ctl/ctl_ioctl.h> 87 88 #include <cam/cam.h> 89 #include <cam/cam_ccb.h> 90 #include <cam/cam_xpt.h> 91 #include <cam/cam_debug.h> 92 #include <cam/cam_sim.h> 93 #include <cam/cam_xpt_sim.h> 94 #include <cam/cam_xpt_periph.h> 95 #include <cam/cam_periph.h> 96 #include <cam/cam_compat.h> 97 #include <cam/scsi/scsi_message.h> 98 99 #include "common/common.h" 100 #include "common/t4_regs.h" 101 #include "common/t4_tcb.h" 102 #include "tom/t4_tom.h" 103 #include "cxgbei.h" 104 105 /* 106 * Use the page pod tag for the TT hash. 107 */ 108 #define TT_HASH(icc, tt) (G_PPOD_TAG(tt) & (icc)->cmp_hash_mask) 109 110 struct cxgbei_ddp_state { 111 struct ppod_reservation prsv; 112 struct cxgbei_cmp cmp; 113 }; 114 115 static MALLOC_DEFINE(M_CXGBEI, "cxgbei", "cxgbei(4)"); 116 117 SYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 118 "Chelsio iSCSI offload"); 119 static int first_burst_length = 8192; 120 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, first_burst_length, CTLFLAG_RWTUN, 121 &first_burst_length, 0, "First burst length"); 122 static int max_burst_length = 2 * 1024 * 1024; 123 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, max_burst_length, CTLFLAG_RWTUN, 124 &max_burst_length, 0, "Maximum burst length"); 125 static int sendspace = 1048576; 126 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN, 127 &sendspace, 0, "Default send socket buffer size"); 128 static int recvspace = 1048576; 129 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN, 130 &recvspace, 0, "Default receive socket buffer size"); 131 132 static volatile u_int icl_cxgbei_ncons; 133 134 static icl_conn_new_pdu_t icl_cxgbei_conn_new_pdu; 135 static icl_conn_pdu_data_segment_length_t 136 icl_cxgbei_conn_pdu_data_segment_length; 137 static icl_conn_pdu_append_bio_t icl_cxgbei_conn_pdu_append_bio; 138 static icl_conn_pdu_append_data_t icl_cxgbei_conn_pdu_append_data; 139 static icl_conn_pdu_get_bio_t icl_cxgbei_conn_pdu_get_bio; 140 static icl_conn_pdu_get_data_t icl_cxgbei_conn_pdu_get_data; 141 static icl_conn_pdu_queue_t icl_cxgbei_conn_pdu_queue; 142 static icl_conn_pdu_queue_cb_t icl_cxgbei_conn_pdu_queue_cb; 143 static icl_conn_handoff_t icl_cxgbei_conn_handoff; 144 static icl_conn_free_t icl_cxgbei_conn_free; 145 static icl_conn_close_t icl_cxgbei_conn_close; 146 static icl_conn_task_setup_t icl_cxgbei_conn_task_setup; 147 static icl_conn_task_done_t icl_cxgbei_conn_task_done; 148 static icl_conn_transfer_setup_t icl_cxgbei_conn_transfer_setup; 149 static icl_conn_transfer_done_t icl_cxgbei_conn_transfer_done; 150 151 static kobj_method_t icl_cxgbei_methods[] = { 152 KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu), 153 KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free), 154 KOBJMETHOD(icl_conn_pdu_data_segment_length, 155 icl_cxgbei_conn_pdu_data_segment_length), 156 KOBJMETHOD(icl_conn_pdu_append_bio, icl_cxgbei_conn_pdu_append_bio), 157 KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data), 158 KOBJMETHOD(icl_conn_pdu_get_bio, icl_cxgbei_conn_pdu_get_bio), 159 KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data), 160 KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue), 161 KOBJMETHOD(icl_conn_pdu_queue_cb, icl_cxgbei_conn_pdu_queue_cb), 162 KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff), 163 KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free), 164 KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close), 165 KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup), 166 KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done), 167 KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup), 168 KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done), 169 { 0, 0 } 170 }; 171 172 DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn)); 173 174 void 175 icl_cxgbei_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip) 176 { 177 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 178 179 KASSERT(icp->ref_cnt != 0, ("freeing deleted PDU")); 180 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 181 MPASS(ic == ip->ip_conn); 182 183 m_freem(ip->ip_ahs_mbuf); 184 m_freem(ip->ip_data_mbuf); 185 m_freem(ip->ip_bhs_mbuf); 186 187 KASSERT(ic != NULL || icp->ref_cnt == 1, 188 ("orphaned PDU has oustanding references")); 189 190 if (atomic_fetchadd_int(&icp->ref_cnt, -1) != 1) 191 return; 192 193 free(icp, M_CXGBEI); 194 #ifdef DIAGNOSTIC 195 if (__predict_true(ic != NULL)) 196 refcount_release(&ic->ic_outstanding_pdus); 197 #endif 198 } 199 200 static void 201 icl_cxgbei_pdu_call_cb(struct icl_pdu *ip) 202 { 203 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 204 205 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 206 207 if (icp->cb != NULL) 208 icp->cb(ip, icp->error); 209 #ifdef DIAGNOSTIC 210 if (__predict_true(ip->ip_conn != NULL)) 211 refcount_release(&ip->ip_conn->ic_outstanding_pdus); 212 #endif 213 free(icp, M_CXGBEI); 214 } 215 216 static void 217 icl_cxgbei_pdu_done(struct icl_pdu *ip, int error) 218 { 219 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 220 221 if (error != 0) 222 icp->error = error; 223 224 m_freem(ip->ip_ahs_mbuf); 225 ip->ip_ahs_mbuf = NULL; 226 m_freem(ip->ip_data_mbuf); 227 ip->ip_data_mbuf = NULL; 228 m_freem(ip->ip_bhs_mbuf); 229 ip->ip_bhs_mbuf = NULL; 230 231 /* 232 * All other references to this PDU should have been dropped 233 * by the m_freem() of ip_data_mbuf. 234 */ 235 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1) 236 icl_cxgbei_pdu_call_cb(ip); 237 else 238 __assert_unreachable(); 239 } 240 241 static void 242 icl_cxgbei_mbuf_done(struct mbuf *mb) 243 { 244 245 struct icl_cxgbei_pdu *icp = (struct icl_cxgbei_pdu *)mb->m_ext.ext_arg1; 246 247 /* 248 * NB: mb_free_mext() might leave ref_cnt as 1 without 249 * decrementing it if it hits the fast path in the ref_cnt 250 * check. 251 */ 252 icl_cxgbei_pdu_call_cb(&icp->ip); 253 } 254 255 struct icl_pdu * 256 icl_cxgbei_new_pdu(int flags) 257 { 258 struct icl_cxgbei_pdu *icp; 259 struct icl_pdu *ip; 260 struct mbuf *m; 261 262 icp = malloc(sizeof(*icp), M_CXGBEI, flags | M_ZERO); 263 if (__predict_false(icp == NULL)) 264 return (NULL); 265 266 icp->icp_signature = CXGBEI_PDU_SIGNATURE; 267 icp->ref_cnt = 1; 268 ip = &icp->ip; 269 270 m = m_gethdr(flags, MT_DATA); 271 if (__predict_false(m == NULL)) { 272 free(icp, M_CXGBEI); 273 return (NULL); 274 } 275 276 ip->ip_bhs_mbuf = m; 277 ip->ip_bhs = mtod(m, struct iscsi_bhs *); 278 memset(ip->ip_bhs, 0, sizeof(*ip->ip_bhs)); 279 m->m_len = sizeof(struct iscsi_bhs); 280 m->m_pkthdr.len = m->m_len; 281 282 return (ip); 283 } 284 285 void 286 icl_cxgbei_new_pdu_set_conn(struct icl_pdu *ip, struct icl_conn *ic) 287 { 288 289 ip->ip_conn = ic; 290 #ifdef DIAGNOSTIC 291 refcount_acquire(&ic->ic_outstanding_pdus); 292 #endif 293 } 294 295 /* 296 * Allocate icl_pdu with empty BHS to fill up by the caller. 297 */ 298 static struct icl_pdu * 299 icl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags) 300 { 301 struct icl_pdu *ip; 302 303 ip = icl_cxgbei_new_pdu(flags); 304 if (__predict_false(ip == NULL)) 305 return (NULL); 306 icl_cxgbei_new_pdu_set_conn(ip, ic); 307 308 return (ip); 309 } 310 311 static size_t 312 icl_pdu_data_segment_length(const struct icl_pdu *request) 313 { 314 uint32_t len = 0; 315 316 len += request->ip_bhs->bhs_data_segment_len[0]; 317 len <<= 8; 318 len += request->ip_bhs->bhs_data_segment_len[1]; 319 len <<= 8; 320 len += request->ip_bhs->bhs_data_segment_len[2]; 321 322 return (len); 323 } 324 325 size_t 326 icl_cxgbei_conn_pdu_data_segment_length(struct icl_conn *ic, 327 const struct icl_pdu *request) 328 { 329 330 return (icl_pdu_data_segment_length(request)); 331 } 332 333 static struct mbuf * 334 finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp) 335 { 336 struct icl_pdu *ip = &icp->ip; 337 uint8_t ulp_submode, padding; 338 struct mbuf *m, *last; 339 struct iscsi_bhs *bhs; 340 int data_len; 341 342 /* 343 * Fix up the data segment mbuf first. 344 */ 345 m = ip->ip_data_mbuf; 346 ulp_submode = icc->ulp_submode; 347 if (m != NULL) { 348 last = m_last(m); 349 350 /* 351 * Round up the data segment to a 4B boundary. Pad with 0 if 352 * necessary. There will definitely be room in the mbuf. 353 */ 354 padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len; 355 if (padding != 0) { 356 MPASS(padding <= M_TRAILINGSPACE(last)); 357 bzero(mtod(last, uint8_t *) + last->m_len, padding); 358 last->m_len += padding; 359 } 360 } else { 361 MPASS(ip->ip_data_len == 0); 362 ulp_submode &= ~ULP_CRC_DATA; 363 padding = 0; 364 } 365 366 /* 367 * Now the header mbuf that has the BHS. 368 */ 369 m = ip->ip_bhs_mbuf; 370 MPASS(m->m_pkthdr.len == sizeof(struct iscsi_bhs)); 371 MPASS(m->m_len == sizeof(struct iscsi_bhs)); 372 373 bhs = ip->ip_bhs; 374 data_len = ip->ip_data_len; 375 if (data_len > icc->ic.ic_max_send_data_segment_length) { 376 struct iscsi_bhs_data_in *bhsdi; 377 int flags; 378 379 KASSERT(padding == 0, ("%s: ISO with padding %d for icp %p", 380 __func__, padding, icp)); 381 switch (bhs->bhs_opcode) { 382 case ISCSI_BHS_OPCODE_SCSI_DATA_OUT: 383 flags = 1; 384 break; 385 case ISCSI_BHS_OPCODE_SCSI_DATA_IN: 386 flags = 2; 387 break; 388 default: 389 panic("invalid opcode %#x for ISO", bhs->bhs_opcode); 390 } 391 data_len = icc->ic.ic_max_send_data_segment_length; 392 bhsdi = (struct iscsi_bhs_data_in *)bhs; 393 if (bhsdi->bhsdi_flags & BHSDI_FLAGS_F) { 394 /* 395 * Firmware will set F on the final PDU in the 396 * burst. 397 */ 398 flags |= CXGBE_ISO_F; 399 bhsdi->bhsdi_flags &= ~BHSDI_FLAGS_F; 400 } 401 set_mbuf_iscsi_iso(m, true); 402 set_mbuf_iscsi_iso_flags(m, flags); 403 set_mbuf_iscsi_iso_mss(m, data_len); 404 } 405 406 bhs->bhs_data_segment_len[2] = data_len; 407 bhs->bhs_data_segment_len[1] = data_len >> 8; 408 bhs->bhs_data_segment_len[0] = data_len >> 16; 409 410 /* 411 * Extract mbuf chain from PDU. 412 */ 413 m->m_pkthdr.len += ip->ip_data_len + padding; 414 m->m_next = ip->ip_data_mbuf; 415 set_mbuf_ulp_submode(m, ulp_submode); 416 ip->ip_bhs_mbuf = NULL; 417 ip->ip_data_mbuf = NULL; 418 ip->ip_bhs = NULL; 419 420 /* 421 * Drop PDU reference on icp. Additional references might 422 * still be held by zero-copy PDU buffers (ICL_NOCOPY). 423 */ 424 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1) 425 icl_cxgbei_pdu_call_cb(ip); 426 427 return (m); 428 } 429 430 static void 431 icl_cxgbei_tx_main(void *arg) 432 { 433 struct epoch_tracker et; 434 struct icl_cxgbei_conn *icc = arg; 435 struct icl_conn *ic = &icc->ic; 436 struct toepcb *toep = icc->toep; 437 struct socket *so = ic->ic_socket; 438 struct inpcb *inp = sotoinpcb(so); 439 struct icl_pdu *ip; 440 struct mbuf *m; 441 struct mbufq mq; 442 STAILQ_HEAD(, icl_pdu) tx_pdus = STAILQ_HEAD_INITIALIZER(tx_pdus); 443 444 mbufq_init(&mq, INT_MAX); 445 446 ICL_CONN_LOCK(ic); 447 while (__predict_true(!ic->ic_disconnecting)) { 448 while (STAILQ_EMPTY(&icc->sent_pdus)) { 449 icc->tx_active = false; 450 mtx_sleep(&icc->tx_active, ic->ic_lock, 0, "-", 0); 451 if (__predict_false(ic->ic_disconnecting)) 452 goto out; 453 MPASS(icc->tx_active); 454 } 455 456 STAILQ_SWAP(&icc->sent_pdus, &tx_pdus, icl_pdu); 457 ICL_CONN_UNLOCK(ic); 458 459 while ((ip = STAILQ_FIRST(&tx_pdus)) != NULL) { 460 STAILQ_REMOVE_HEAD(&tx_pdus, ip_next); 461 462 m = finalize_pdu(icc, ip_to_icp(ip)); 463 M_ASSERTPKTHDR(m); 464 MPASS((m->m_pkthdr.len & 3) == 0); 465 466 mbufq_enqueue(&mq, m); 467 } 468 469 ICL_CONN_LOCK(ic); 470 if (__predict_false(ic->ic_disconnecting) || 471 __predict_false(ic->ic_socket == NULL)) { 472 mbufq_drain(&mq); 473 break; 474 } 475 476 CURVNET_SET(toep->vnet); 477 NET_EPOCH_ENTER(et); 478 INP_WLOCK(inp); 479 480 ICL_CONN_UNLOCK(ic); 481 if (__predict_false(inp->inp_flags & (INP_DROPPED | 482 INP_TIMEWAIT)) || 483 __predict_false((toep->flags & TPF_ATTACHED) == 0)) { 484 mbufq_drain(&mq); 485 } else { 486 mbufq_concat(&toep->ulp_pduq, &mq); 487 t4_push_pdus(icc->sc, toep, 0); 488 } 489 INP_WUNLOCK(inp); 490 NET_EPOCH_EXIT(et); 491 CURVNET_RESTORE(); 492 493 ICL_CONN_LOCK(ic); 494 } 495 out: 496 ICL_CONN_UNLOCK(ic); 497 498 kthread_exit(); 499 } 500 501 static void 502 icl_cxgbei_rx_main(void *arg) 503 { 504 struct icl_cxgbei_conn *icc = arg; 505 struct icl_conn *ic = &icc->ic; 506 struct icl_pdu *ip; 507 struct sockbuf *sb; 508 STAILQ_HEAD(, icl_pdu) rx_pdus = STAILQ_HEAD_INITIALIZER(rx_pdus); 509 bool cantrcvmore; 510 511 sb = &ic->ic_socket->so_rcv; 512 SOCKBUF_LOCK(sb); 513 while (__predict_true(!ic->ic_disconnecting)) { 514 while (STAILQ_EMPTY(&icc->rcvd_pdus)) { 515 icc->rx_active = false; 516 mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0); 517 if (__predict_false(ic->ic_disconnecting)) 518 goto out; 519 MPASS(icc->rx_active); 520 } 521 522 if (__predict_false(sbused(sb)) != 0) { 523 /* 524 * PDUs were received before the tid 525 * transitioned to ULP mode. Convert 526 * them to icl_cxgbei_pdus and insert 527 * them into the head of rcvd_pdus. 528 */ 529 parse_pdus(icc, sb); 530 } 531 cantrcvmore = (sb->sb_state & SBS_CANTRCVMORE) != 0; 532 MPASS(STAILQ_EMPTY(&rx_pdus)); 533 STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu); 534 SOCKBUF_UNLOCK(sb); 535 536 /* Hand over PDUs to ICL. */ 537 while ((ip = STAILQ_FIRST(&rx_pdus)) != NULL) { 538 STAILQ_REMOVE_HEAD(&rx_pdus, ip_next); 539 if (cantrcvmore) 540 icl_cxgbei_pdu_done(ip, ENOTCONN); 541 else 542 ic->ic_receive(ip); 543 } 544 545 SOCKBUF_LOCK(sb); 546 } 547 out: 548 /* 549 * Since ic_disconnecting is set before the SOCKBUF_MTX is 550 * locked in icl_cxgbei_conn_close, the loop above can exit 551 * before icl_cxgbei_conn_close can lock SOCKBUF_MTX and block 552 * waiting for the thread exit. 553 */ 554 while (!icc->rx_exiting) 555 mtx_sleep(&icc->rx_active, SOCKBUF_MTX(sb), 0, "-", 0); 556 SOCKBUF_UNLOCK(sb); 557 558 kthread_exit(); 559 } 560 561 static void 562 cxgbei_free_mext_pg(struct mbuf *m) 563 { 564 struct icl_cxgbei_pdu *icp; 565 566 M_ASSERTEXTPG(m); 567 568 /* 569 * Nothing to do for the pages; they are owned by the PDU / 570 * I/O request. 571 */ 572 573 /* Drop reference on the PDU. */ 574 icp = m->m_ext.ext_arg1; 575 if (atomic_fetchadd_int(&icp->ref_cnt, -1) == 1) 576 icl_cxgbei_pdu_call_cb(&icp->ip); 577 } 578 579 static struct mbuf * 580 cxgbei_getm(size_t len, int flags) 581 { 582 struct mbuf *m, *m0, *m_tail; 583 584 m_tail = m0 = NULL; 585 586 /* Allocate as jumbo mbufs of size MJUM16BYTES. */ 587 while (len >= MJUM16BYTES) { 588 m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES); 589 if (__predict_false(m == NULL)) { 590 if ((flags & M_WAITOK) != 0) { 591 /* Fall back to non-jumbo mbufs. */ 592 break; 593 } 594 return (NULL); 595 } 596 if (m0 == NULL) { 597 m0 = m_tail = m; 598 } else { 599 m_tail->m_next = m; 600 m_tail = m; 601 } 602 len -= MJUM16BYTES; 603 } 604 605 /* Allocate mbuf chain for the remaining data. */ 606 if (len != 0) { 607 m = m_getm2(NULL, len, flags, MT_DATA, 0); 608 if (__predict_false(m == NULL)) { 609 m_freem(m0); 610 return (NULL); 611 } 612 if (m0 == NULL) 613 m0 = m; 614 else 615 m_tail->m_next = m; 616 } 617 618 return (m0); 619 } 620 621 int 622 icl_cxgbei_conn_pdu_append_bio(struct icl_conn *ic, struct icl_pdu *ip, 623 struct bio *bp, size_t offset, size_t len, int flags) 624 { 625 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 626 struct mbuf *m, *m_tail; 627 vm_offset_t vaddr; 628 size_t page_offset, todo, mtodo; 629 boolean_t mapped; 630 int i; 631 632 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 633 MPASS(ic == ip->ip_conn); 634 KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len)); 635 636 m_tail = ip->ip_data_mbuf; 637 if (m_tail != NULL) 638 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next) 639 ; 640 641 MPASS(bp->bio_flags & BIO_UNMAPPED); 642 if (offset < PAGE_SIZE - bp->bio_ma_offset) { 643 page_offset = bp->bio_ma_offset + offset; 644 i = 0; 645 } else { 646 offset -= PAGE_SIZE - bp->bio_ma_offset; 647 for (i = 1; offset >= PAGE_SIZE; i++) 648 offset -= PAGE_SIZE; 649 page_offset = offset; 650 } 651 652 if (flags & ICL_NOCOPY) { 653 m = NULL; 654 while (len > 0) { 655 if (m == NULL) { 656 m = mb_alloc_ext_pgs(flags & ~ICL_NOCOPY, 657 cxgbei_free_mext_pg); 658 if (__predict_false(m == NULL)) 659 return (ENOMEM); 660 atomic_add_int(&icp->ref_cnt, 1); 661 m->m_ext.ext_arg1 = icp; 662 m->m_epg_1st_off = page_offset; 663 } 664 665 todo = MIN(len, PAGE_SIZE - page_offset); 666 667 m->m_epg_pa[m->m_epg_npgs] = 668 VM_PAGE_TO_PHYS(bp->bio_ma[i]); 669 m->m_epg_npgs++; 670 m->m_epg_last_len = todo; 671 m->m_len += todo; 672 m->m_ext.ext_size += PAGE_SIZE; 673 MBUF_EXT_PGS_ASSERT_SANITY(m); 674 675 if (m->m_epg_npgs == MBUF_PEXT_MAX_PGS) { 676 if (m_tail != NULL) 677 m_tail->m_next = m; 678 else 679 ip->ip_data_mbuf = m; 680 m_tail = m; 681 ip->ip_data_len += m->m_len; 682 m = NULL; 683 } 684 685 page_offset = 0; 686 len -= todo; 687 i++; 688 } 689 690 if (m != NULL) { 691 if (m_tail != NULL) 692 m_tail->m_next = m; 693 else 694 ip->ip_data_mbuf = m; 695 ip->ip_data_len += m->m_len; 696 } 697 return (0); 698 } 699 700 m = cxgbei_getm(len, flags); 701 if (__predict_false(m == NULL)) 702 return (ENOMEM); 703 704 if (ip->ip_data_mbuf == NULL) { 705 ip->ip_data_mbuf = m; 706 ip->ip_data_len = len; 707 } else { 708 m_tail->m_next = m; 709 ip->ip_data_len += len; 710 } 711 712 while (len > 0) { 713 todo = MIN(len, PAGE_SIZE - page_offset); 714 715 mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1, 716 FALSE); 717 718 do { 719 mtodo = min(todo, M_SIZE(m) - m->m_len); 720 memcpy(mtod(m, char *) + m->m_len, (char *)vaddr + 721 page_offset, mtodo); 722 m->m_len += mtodo; 723 if (m->m_len == M_SIZE(m)) 724 m = m->m_next; 725 page_offset += mtodo; 726 todo -= mtodo; 727 } while (todo > 0); 728 729 if (__predict_false(mapped)) 730 pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1, 731 FALSE); 732 733 page_offset = 0; 734 len -= todo; 735 i++; 736 } 737 738 MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length, 739 ic->ic_hw_isomax)); 740 741 return (0); 742 } 743 744 int 745 icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip, 746 const void *addr, size_t len, int flags) 747 { 748 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 749 struct mbuf *m, *m_tail; 750 const char *src; 751 752 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 753 MPASS(ic == ip->ip_conn); 754 KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len)); 755 756 m_tail = ip->ip_data_mbuf; 757 if (m_tail != NULL) 758 for (; m_tail->m_next != NULL; m_tail = m_tail->m_next) 759 ; 760 761 if (flags & ICL_NOCOPY) { 762 m = m_get(flags & ~ICL_NOCOPY, MT_DATA); 763 if (m == NULL) { 764 ICL_WARN("failed to allocate mbuf"); 765 return (ENOMEM); 766 } 767 768 m->m_flags |= M_RDONLY; 769 m_extaddref(m, __DECONST(char *, addr), len, &icp->ref_cnt, 770 icl_cxgbei_mbuf_done, icp, NULL); 771 m->m_len = len; 772 if (ip->ip_data_mbuf == NULL) { 773 ip->ip_data_mbuf = m; 774 ip->ip_data_len = len; 775 } else { 776 m_tail->m_next = m; 777 m_tail = m_tail->m_next; 778 ip->ip_data_len += len; 779 } 780 781 return (0); 782 } 783 784 m = cxgbei_getm(len, flags); 785 if (__predict_false(m == NULL)) 786 return (ENOMEM); 787 788 if (ip->ip_data_mbuf == NULL) { 789 ip->ip_data_mbuf = m; 790 ip->ip_data_len = len; 791 } else { 792 m_tail->m_next = m; 793 ip->ip_data_len += len; 794 } 795 src = (const char *)addr; 796 for (; m != NULL; m = m->m_next) { 797 m->m_len = min(len, M_SIZE(m)); 798 memcpy(mtod(m, void *), src, m->m_len); 799 src += m->m_len; 800 len -= m->m_len; 801 } 802 MPASS(len == 0); 803 804 MPASS(ip->ip_data_len <= max(ic->ic_max_send_data_segment_length, 805 ic->ic_hw_isomax)); 806 807 return (0); 808 } 809 810 void 811 icl_cxgbei_conn_pdu_get_bio(struct icl_conn *ic, struct icl_pdu *ip, 812 size_t pdu_off, struct bio *bp, size_t bio_off, size_t len) 813 { 814 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 815 vm_offset_t vaddr; 816 size_t page_offset, todo; 817 boolean_t mapped; 818 int i; 819 820 if (icp->icp_flags & ICPF_RX_DDP) 821 return; /* data is DDP'ed, no need to copy */ 822 823 MPASS(bp->bio_flags & BIO_UNMAPPED); 824 if (bio_off < PAGE_SIZE - bp->bio_ma_offset) { 825 page_offset = bp->bio_ma_offset + bio_off; 826 i = 0; 827 } else { 828 bio_off -= PAGE_SIZE - bp->bio_ma_offset; 829 for (i = 1; bio_off >= PAGE_SIZE; i++) 830 bio_off -= PAGE_SIZE; 831 page_offset = bio_off; 832 } 833 834 while (len > 0) { 835 todo = MIN(len, PAGE_SIZE - page_offset); 836 837 mapped = pmap_map_io_transient(bp->bio_ma + i, &vaddr, 1, 838 FALSE); 839 m_copydata(ip->ip_data_mbuf, pdu_off, todo, (char *)vaddr + 840 page_offset); 841 if (__predict_false(mapped)) 842 pmap_unmap_io_transient(bp->bio_ma + 1, &vaddr, 1, 843 FALSE); 844 845 page_offset = 0; 846 pdu_off += todo; 847 len -= todo; 848 i++; 849 } 850 } 851 852 void 853 icl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip, 854 size_t off, void *addr, size_t len) 855 { 856 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 857 858 if (icp->icp_flags & ICPF_RX_DDP) 859 return; /* data is DDP'ed, no need to copy */ 860 m_copydata(ip->ip_data_mbuf, off, len, addr); 861 } 862 863 void 864 icl_cxgbei_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip) 865 { 866 icl_cxgbei_conn_pdu_queue_cb(ic, ip, NULL); 867 } 868 869 void 870 icl_cxgbei_conn_pdu_queue_cb(struct icl_conn *ic, struct icl_pdu *ip, 871 icl_pdu_cb cb) 872 { 873 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 874 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 875 struct socket *so = ic->ic_socket; 876 877 MPASS(ic == ip->ip_conn); 878 MPASS(ip->ip_bhs_mbuf != NULL); 879 /* The kernel doesn't generate PDUs with AHS. */ 880 MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0); 881 882 ICL_CONN_LOCK_ASSERT(ic); 883 884 icp->cb = cb; 885 886 /* NOTE: sowriteable without so_snd lock is a mostly harmless race. */ 887 if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) { 888 icl_cxgbei_pdu_done(ip, ENOTCONN); 889 return; 890 } 891 892 STAILQ_INSERT_TAIL(&icc->sent_pdus, ip, ip_next); 893 if (!icc->tx_active) { 894 icc->tx_active = true; 895 wakeup(&icc->tx_active); 896 } 897 } 898 899 static struct icl_conn * 900 icl_cxgbei_new_conn(const char *name, struct mtx *lock) 901 { 902 struct icl_cxgbei_conn *icc; 903 struct icl_conn *ic; 904 905 refcount_acquire(&icl_cxgbei_ncons); 906 907 icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE, 908 M_WAITOK | M_ZERO); 909 icc->icc_signature = CXGBEI_CONN_SIGNATURE; 910 STAILQ_INIT(&icc->rcvd_pdus); 911 STAILQ_INIT(&icc->sent_pdus); 912 913 icc->cmp_table = hashinit(64, M_CXGBEI, &icc->cmp_hash_mask); 914 mtx_init(&icc->cmp_lock, "cxgbei_cmp", NULL, MTX_DEF); 915 916 ic = &icc->ic; 917 ic->ic_lock = lock; 918 919 #ifdef DIAGNOSTIC 920 refcount_init(&ic->ic_outstanding_pdus, 0); 921 #endif 922 ic->ic_name = name; 923 ic->ic_offload = "cxgbei"; 924 ic->ic_unmapped = true; 925 926 CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); 927 928 return (ic); 929 } 930 931 void 932 icl_cxgbei_conn_free(struct icl_conn *ic) 933 { 934 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 935 936 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 937 938 CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc); 939 940 mtx_destroy(&icc->cmp_lock); 941 hashdestroy(icc->cmp_table, M_CXGBEI, icc->cmp_hash_mask); 942 kobj_delete((struct kobj *)icc, M_CXGBE); 943 refcount_release(&icl_cxgbei_ncons); 944 } 945 946 static int 947 icl_cxgbei_setsockopt(struct icl_conn *ic, struct socket *so, int sspace, 948 int rspace) 949 { 950 struct sockopt opt; 951 int error, one = 1, ss, rs; 952 953 ss = max(sendspace, sspace); 954 rs = max(recvspace, rspace); 955 956 error = soreserve(so, ss, rs); 957 if (error != 0) 958 return (error); 959 SOCKBUF_LOCK(&so->so_snd); 960 so->so_snd.sb_flags |= SB_AUTOSIZE; 961 SOCKBUF_UNLOCK(&so->so_snd); 962 SOCKBUF_LOCK(&so->so_rcv); 963 so->so_rcv.sb_flags |= SB_AUTOSIZE; 964 SOCKBUF_UNLOCK(&so->so_rcv); 965 966 /* 967 * Disable Nagle. 968 */ 969 bzero(&opt, sizeof(opt)); 970 opt.sopt_dir = SOPT_SET; 971 opt.sopt_level = IPPROTO_TCP; 972 opt.sopt_name = TCP_NODELAY; 973 opt.sopt_val = &one; 974 opt.sopt_valsize = sizeof(one); 975 error = sosetopt(so, &opt); 976 if (error != 0) 977 return (error); 978 979 return (0); 980 } 981 982 /* 983 * Request/response structure used to find out the adapter offloading a socket. 984 */ 985 struct find_ofld_adapter_rr { 986 struct socket *so; 987 struct adapter *sc; /* result */ 988 }; 989 990 static void 991 find_offload_adapter(struct adapter *sc, void *arg) 992 { 993 struct find_ofld_adapter_rr *fa = arg; 994 struct socket *so = fa->so; 995 struct tom_data *td = sc->tom_softc; 996 struct tcpcb *tp; 997 struct inpcb *inp; 998 999 /* Non-TCP were filtered out earlier. */ 1000 MPASS(so->so_proto->pr_protocol == IPPROTO_TCP); 1001 1002 if (fa->sc != NULL) 1003 return; /* Found already. */ 1004 1005 if (td == NULL) 1006 return; /* TOE not enabled on this adapter. */ 1007 1008 inp = sotoinpcb(so); 1009 INP_WLOCK(inp); 1010 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) { 1011 tp = intotcpcb(inp); 1012 if (tp->t_flags & TF_TOE && tp->tod == &td->tod) 1013 fa->sc = sc; /* Found. */ 1014 } 1015 INP_WUNLOCK(inp); 1016 } 1017 1018 static bool 1019 is_memfree(struct adapter *sc) 1020 { 1021 uint32_t em; 1022 1023 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1024 if ((em & F_EXT_MEM_ENABLE) != 0) 1025 return (false); 1026 if (is_t5(sc) && (em & F_EXT_MEM1_ENABLE) != 0) 1027 return (false); 1028 return (true); 1029 } 1030 1031 /* XXXNP: move this to t4_tom. */ 1032 static void 1033 send_iscsi_flowc_wr(struct adapter *sc, struct toepcb *toep, int maxlen) 1034 { 1035 struct wrqe *wr; 1036 struct fw_flowc_wr *flowc; 1037 const u_int nparams = 1; 1038 u_int flowclen; 1039 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; 1040 1041 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval); 1042 1043 wr = alloc_wrqe(roundup2(flowclen, 16), &toep->ofld_txq->wrq); 1044 if (wr == NULL) { 1045 /* XXX */ 1046 panic("%s: allocation failure.", __func__); 1047 } 1048 flowc = wrtod(wr); 1049 memset(flowc, 0, wr->wr_len); 1050 1051 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) | 1052 V_FW_FLOWC_WR_NPARAMS(nparams)); 1053 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) | 1054 V_FW_WR_FLOWID(toep->tid)); 1055 1056 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; 1057 flowc->mnemval[0].val = htobe32(maxlen); 1058 1059 txsd->tx_credits = howmany(flowclen, 16); 1060 txsd->plen = 0; 1061 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0, 1062 ("%s: not enough credits (%d)", __func__, toep->tx_credits)); 1063 toep->tx_credits -= txsd->tx_credits; 1064 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) 1065 toep->txsd_pidx = 0; 1066 toep->txsd_avail--; 1067 1068 t4_wrq_tx(sc, wr); 1069 } 1070 1071 static void 1072 set_ulp_mode_iscsi(struct adapter *sc, struct toepcb *toep, u_int ulp_submode) 1073 { 1074 uint64_t val; 1075 1076 CTR3(KTR_CXGBE, "%s: tid %u, ULP_MODE_ISCSI, submode=%#x", 1077 __func__, toep->tid, ulp_submode); 1078 1079 val = V_TCB_ULP_TYPE(ULP_MODE_ISCSI) | V_TCB_ULP_RAW(ulp_submode); 1080 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_ULP_TYPE, 1081 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE) | V_TCB_ULP_RAW(M_TCB_ULP_RAW), val, 1082 0, 0); 1083 1084 val = V_TF_RX_FLOW_CONTROL_DISABLE(1ULL); 1085 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_T_FLAGS, val, val, 0, 0); 1086 } 1087 1088 /* 1089 * XXXNP: Who is responsible for cleaning up the socket if this returns with an 1090 * error? Review all error paths. 1091 * 1092 * XXXNP: What happens to the socket's fd reference if the operation is 1093 * successful, and how does that affect the socket's life cycle? 1094 */ 1095 int 1096 icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd) 1097 { 1098 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 1099 struct find_ofld_adapter_rr fa; 1100 struct file *fp; 1101 struct socket *so; 1102 struct inpcb *inp; 1103 struct tcpcb *tp; 1104 struct toepcb *toep; 1105 cap_rights_t rights; 1106 u_int max_iso_payload, max_rx_pdu_len, max_tx_pdu_len; 1107 int error, max_iso_pdus; 1108 1109 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 1110 ICL_CONN_LOCK_ASSERT_NOT(ic); 1111 1112 /* 1113 * Steal the socket from userland. 1114 */ 1115 error = fget(curthread, fd, 1116 cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp); 1117 if (error != 0) 1118 return (error); 1119 if (fp->f_type != DTYPE_SOCKET) { 1120 fdrop(fp, curthread); 1121 return (EINVAL); 1122 } 1123 so = fp->f_data; 1124 if (so->so_type != SOCK_STREAM || 1125 so->so_proto->pr_protocol != IPPROTO_TCP) { 1126 fdrop(fp, curthread); 1127 return (EINVAL); 1128 } 1129 1130 ICL_CONN_LOCK(ic); 1131 if (ic->ic_socket != NULL) { 1132 ICL_CONN_UNLOCK(ic); 1133 fdrop(fp, curthread); 1134 return (EBUSY); 1135 } 1136 ic->ic_disconnecting = false; 1137 ic->ic_socket = so; 1138 fp->f_ops = &badfileops; 1139 fp->f_data = NULL; 1140 fdrop(fp, curthread); 1141 ICL_CONN_UNLOCK(ic); 1142 1143 /* Find the adapter offloading this socket. */ 1144 fa.sc = NULL; 1145 fa.so = so; 1146 t4_iterate(find_offload_adapter, &fa); 1147 if (fa.sc == NULL) { 1148 error = EINVAL; 1149 goto out; 1150 } 1151 icc->sc = fa.sc; 1152 1153 max_rx_pdu_len = ISCSI_BHS_SIZE + ic->ic_max_recv_data_segment_length; 1154 max_tx_pdu_len = ISCSI_BHS_SIZE + ic->ic_max_send_data_segment_length; 1155 if (ic->ic_header_crc32c) { 1156 max_rx_pdu_len += ISCSI_HEADER_DIGEST_SIZE; 1157 max_tx_pdu_len += ISCSI_HEADER_DIGEST_SIZE; 1158 } 1159 if (ic->ic_data_crc32c) { 1160 max_rx_pdu_len += ISCSI_DATA_DIGEST_SIZE; 1161 max_tx_pdu_len += ISCSI_DATA_DIGEST_SIZE; 1162 } 1163 1164 inp = sotoinpcb(so); 1165 INP_WLOCK(inp); 1166 tp = intotcpcb(inp); 1167 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) { 1168 INP_WUNLOCK(inp); 1169 error = ENOTCONN; 1170 goto out; 1171 } 1172 1173 /* 1174 * socket could not have been "unoffloaded" if here. 1175 */ 1176 MPASS(tp->t_flags & TF_TOE); 1177 MPASS(tp->tod != NULL); 1178 MPASS(tp->t_toe != NULL); 1179 toep = tp->t_toe; 1180 MPASS(toep->vi->adapter == icc->sc); 1181 1182 if (ulp_mode(toep) != ULP_MODE_NONE) { 1183 INP_WUNLOCK(inp); 1184 error = EINVAL; 1185 goto out; 1186 } 1187 1188 icc->toep = toep; 1189 1190 icc->ulp_submode = 0; 1191 if (ic->ic_header_crc32c) 1192 icc->ulp_submode |= ULP_CRC_HEADER; 1193 if (ic->ic_data_crc32c) 1194 icc->ulp_submode |= ULP_CRC_DATA; 1195 1196 if (icc->sc->tt.iso && chip_id(icc->sc) >= CHELSIO_T5 && 1197 !is_memfree(icc->sc)) { 1198 max_iso_payload = rounddown(CXGBEI_MAX_ISO_PAYLOAD, 1199 tp->t_maxseg); 1200 max_iso_pdus = max_iso_payload / max_tx_pdu_len; 1201 ic->ic_hw_isomax = max_iso_pdus * 1202 ic->ic_max_send_data_segment_length; 1203 } else 1204 max_iso_pdus = 1; 1205 1206 toep->params.ulp_mode = ULP_MODE_ISCSI; 1207 toep->ulpcb = icc; 1208 1209 send_iscsi_flowc_wr(icc->sc, toep, 1210 roundup(max_iso_pdus * max_tx_pdu_len, tp->t_maxseg)); 1211 set_ulp_mode_iscsi(icc->sc, toep, icc->ulp_submode); 1212 INP_WUNLOCK(inp); 1213 1214 error = kthread_add(icl_cxgbei_tx_main, icc, NULL, &icc->tx_thread, 0, 1215 0, "%stx (cxgbei)", ic->ic_name); 1216 if (error != 0) 1217 goto out; 1218 1219 error = kthread_add(icl_cxgbei_rx_main, icc, NULL, &icc->rx_thread, 0, 1220 0, "%srx (cxgbei)", ic->ic_name); 1221 if (error != 0) 1222 goto out; 1223 1224 error = icl_cxgbei_setsockopt(ic, so, max_tx_pdu_len, max_rx_pdu_len); 1225 out: 1226 if (error != 0) 1227 icl_cxgbei_conn_close(ic); 1228 return (error); 1229 } 1230 1231 void 1232 icl_cxgbei_conn_close(struct icl_conn *ic) 1233 { 1234 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 1235 struct icl_pdu *ip; 1236 struct socket *so; 1237 struct sockbuf *sb; 1238 struct inpcb *inp; 1239 struct toepcb *toep = icc->toep; 1240 1241 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); 1242 ICL_CONN_LOCK_ASSERT_NOT(ic); 1243 1244 ICL_CONN_LOCK(ic); 1245 so = ic->ic_socket; 1246 if (ic->ic_disconnecting || so == NULL) { 1247 CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p", 1248 __func__, icc, ic->ic_disconnecting, so); 1249 ICL_CONN_UNLOCK(ic); 1250 return; 1251 } 1252 ic->ic_disconnecting = true; 1253 1254 #ifdef DIAGNOSTIC 1255 KASSERT(ic->ic_outstanding_pdus == 0, 1256 ("destroying session with %d outstanding PDUs", 1257 ic->ic_outstanding_pdus)); 1258 #endif 1259 1260 CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1, 1261 icc); 1262 1263 /* 1264 * Wait for the transmit thread to stop processing 1265 * this connection. 1266 */ 1267 if (icc->tx_thread != NULL) { 1268 wakeup(&icc->tx_active); 1269 mtx_sleep(icc->tx_thread, ic->ic_lock, 0, "conclo", 0); 1270 } 1271 1272 /* Discard PDUs queued for TX. */ 1273 while (!STAILQ_EMPTY(&icc->sent_pdus)) { 1274 ip = STAILQ_FIRST(&icc->sent_pdus); 1275 STAILQ_REMOVE_HEAD(&icc->sent_pdus, ip_next); 1276 icl_cxgbei_pdu_done(ip, ENOTCONN); 1277 } 1278 ICL_CONN_UNLOCK(ic); 1279 1280 inp = sotoinpcb(so); 1281 sb = &so->so_rcv; 1282 1283 /* 1284 * Wait for the receive thread to stop processing this 1285 * connection. 1286 */ 1287 SOCKBUF_LOCK(sb); 1288 if (icc->rx_thread != NULL) { 1289 icc->rx_exiting = true; 1290 wakeup(&icc->rx_active); 1291 mtx_sleep(icc->rx_thread, SOCKBUF_MTX(sb), 0, "conclo", 0); 1292 } 1293 1294 /* 1295 * Discard received PDUs not passed to the iSCSI layer. 1296 */ 1297 while (!STAILQ_EMPTY(&icc->rcvd_pdus)) { 1298 ip = STAILQ_FIRST(&icc->rcvd_pdus); 1299 STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next); 1300 icl_cxgbei_pdu_done(ip, ENOTCONN); 1301 } 1302 SOCKBUF_UNLOCK(sb); 1303 1304 INP_WLOCK(inp); 1305 if (toep != NULL) { /* NULL if connection was never offloaded. */ 1306 toep->ulpcb = NULL; 1307 1308 /* Discard mbufs queued for TX. */ 1309 mbufq_drain(&toep->ulp_pduq); 1310 1311 /* 1312 * Grab a reference to use when waiting for the final 1313 * CPL to be received. If toep->inp is NULL, then 1314 * final_cpl_received() has already been called (e.g. 1315 * due to the peer sending a RST). 1316 */ 1317 if (toep->inp != NULL) { 1318 toep = hold_toepcb(toep); 1319 toep->flags |= TPF_WAITING_FOR_FINAL; 1320 } else 1321 toep = NULL; 1322 } 1323 INP_WUNLOCK(inp); 1324 1325 ICL_CONN_LOCK(ic); 1326 ic->ic_socket = NULL; 1327 ICL_CONN_UNLOCK(ic); 1328 1329 /* 1330 * XXXNP: we should send RST instead of FIN when PDUs held in various 1331 * queues were purged instead of delivered reliably but soabort isn't 1332 * really general purpose and wouldn't do the right thing here. 1333 */ 1334 soclose(so); 1335 1336 /* 1337 * Wait for the socket to fully close. This ensures any 1338 * pending received data has been received (and in particular, 1339 * any data that would be received by DDP has been handled). 1340 * Callers assume that it is safe to free buffers for tasks 1341 * and transfers after this function returns. 1342 */ 1343 if (toep != NULL) { 1344 struct mtx *lock = mtx_pool_find(mtxpool_sleep, toep); 1345 1346 mtx_lock(lock); 1347 while ((toep->flags & TPF_WAITING_FOR_FINAL) != 0) 1348 mtx_sleep(toep, lock, PSOCK, "conclo2", 0); 1349 mtx_unlock(lock); 1350 free_toepcb(toep); 1351 } 1352 } 1353 1354 static void 1355 cxgbei_insert_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp, 1356 uint32_t tt) 1357 { 1358 #ifdef INVARIANTS 1359 struct cxgbei_cmp *cmp2; 1360 #endif 1361 1362 cmp->tt = tt; 1363 1364 mtx_lock(&icc->cmp_lock); 1365 #ifdef INVARIANTS 1366 LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, tt)], link) { 1367 KASSERT(cmp2->tt != tt, ("%s: duplicate cmp", __func__)); 1368 } 1369 #endif 1370 LIST_INSERT_HEAD(&icc->cmp_table[TT_HASH(icc, tt)], cmp, link); 1371 mtx_unlock(&icc->cmp_lock); 1372 } 1373 1374 struct cxgbei_cmp * 1375 cxgbei_find_cmp(struct icl_cxgbei_conn *icc, uint32_t tt) 1376 { 1377 struct cxgbei_cmp *cmp; 1378 1379 mtx_lock(&icc->cmp_lock); 1380 LIST_FOREACH(cmp, &icc->cmp_table[TT_HASH(icc, tt)], link) { 1381 if (cmp->tt == tt) 1382 break; 1383 } 1384 mtx_unlock(&icc->cmp_lock); 1385 return (cmp); 1386 } 1387 1388 static void 1389 cxgbei_rm_cmp(struct icl_cxgbei_conn *icc, struct cxgbei_cmp *cmp) 1390 { 1391 #ifdef INVARIANTS 1392 struct cxgbei_cmp *cmp2; 1393 #endif 1394 1395 mtx_lock(&icc->cmp_lock); 1396 1397 #ifdef INVARIANTS 1398 LIST_FOREACH(cmp2, &icc->cmp_table[TT_HASH(icc, cmp->tt)], link) { 1399 if (cmp2 == cmp) 1400 goto found; 1401 } 1402 panic("%s: could not find cmp", __func__); 1403 found: 1404 #endif 1405 LIST_REMOVE(cmp, link); 1406 mtx_unlock(&icc->cmp_lock); 1407 } 1408 1409 int 1410 icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, 1411 struct ccb_scsiio *csio, uint32_t *ittp, void **arg) 1412 { 1413 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 1414 struct toepcb *toep = icc->toep; 1415 struct adapter *sc = icc->sc; 1416 struct cxgbei_data *ci = sc->iscsi_ulp_softc; 1417 struct ppod_region *pr = &ci->pr; 1418 struct cxgbei_ddp_state *ddp; 1419 struct ppod_reservation *prsv; 1420 struct inpcb *inp; 1421 struct mbufq mq; 1422 uint32_t itt; 1423 int rc = 0; 1424 1425 ICL_CONN_LOCK_ASSERT(ic); 1426 1427 /* This is for the offload driver's state. Must not be set already. */ 1428 MPASS(arg != NULL); 1429 MPASS(*arg == NULL); 1430 1431 if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN || 1432 csio->dxfer_len < ci->ddp_threshold || ic->ic_disconnecting || 1433 ic->ic_socket == NULL) { 1434 no_ddp: 1435 /* 1436 * No DDP for this I/O. Allocate an ITT (based on the one 1437 * passed in) that cannot be a valid hardware DDP tag in the 1438 * iSCSI region. 1439 */ 1440 itt = *ittp & M_PPOD_TAG; 1441 itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit; 1442 *ittp = htobe32(itt); 1443 MPASS(*arg == NULL); /* State is maintained for DDP only. */ 1444 if (rc != 0) 1445 counter_u64_add( 1446 toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1); 1447 return (0); 1448 } 1449 1450 /* 1451 * Reserve resources for DDP, update the itt that should be used in the 1452 * PDU, and save DDP specific state for this I/O in *arg. 1453 */ 1454 ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO); 1455 if (ddp == NULL) { 1456 rc = ENOMEM; 1457 goto no_ddp; 1458 } 1459 prsv = &ddp->prsv; 1460 1461 mbufq_init(&mq, INT_MAX); 1462 switch (csio->ccb_h.flags & CAM_DATA_MASK) { 1463 case CAM_DATA_BIO: 1464 rc = t4_alloc_page_pods_for_bio(pr, 1465 (struct bio *)csio->data_ptr, prsv); 1466 if (rc != 0) { 1467 free(ddp, M_CXGBEI); 1468 goto no_ddp; 1469 } 1470 1471 rc = t4_write_page_pods_for_bio(sc, toep, prsv, 1472 (struct bio *)csio->data_ptr, &mq); 1473 if (__predict_false(rc != 0)) { 1474 mbufq_drain(&mq); 1475 t4_free_page_pods(prsv); 1476 free(ddp, M_CXGBEI); 1477 goto no_ddp; 1478 } 1479 break; 1480 case CAM_DATA_VADDR: 1481 rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr, 1482 csio->dxfer_len, prsv); 1483 if (rc != 0) { 1484 free(ddp, M_CXGBEI); 1485 goto no_ddp; 1486 } 1487 1488 rc = t4_write_page_pods_for_buf(sc, toep, prsv, 1489 (vm_offset_t)csio->data_ptr, csio->dxfer_len, &mq); 1490 if (__predict_false(rc != 0)) { 1491 mbufq_drain(&mq); 1492 t4_free_page_pods(prsv); 1493 free(ddp, M_CXGBEI); 1494 goto no_ddp; 1495 } 1496 break; 1497 default: 1498 free(ddp, M_CXGBEI); 1499 rc = EINVAL; 1500 goto no_ddp; 1501 } 1502 1503 /* 1504 * Do not get inp from toep->inp as the toepcb might have 1505 * detached already. 1506 */ 1507 inp = sotoinpcb(ic->ic_socket); 1508 INP_WLOCK(inp); 1509 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) { 1510 INP_WUNLOCK(inp); 1511 mbufq_drain(&mq); 1512 t4_free_page_pods(prsv); 1513 free(ddp, M_CXGBEI); 1514 goto no_ddp; 1515 } 1516 mbufq_concat(&toep->ulp_pduq, &mq); 1517 INP_WUNLOCK(inp); 1518 1519 ddp->cmp.last_datasn = -1; 1520 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); 1521 *ittp = htobe32(prsv->prsv_tag); 1522 *arg = prsv; 1523 counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1); 1524 return (0); 1525 } 1526 1527 void 1528 icl_cxgbei_conn_task_done(struct icl_conn *ic, void *arg) 1529 { 1530 1531 if (arg != NULL) { 1532 struct cxgbei_ddp_state *ddp = arg; 1533 1534 cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp); 1535 t4_free_page_pods(&ddp->prsv); 1536 free(ddp, M_CXGBEI); 1537 } 1538 } 1539 1540 static inline bool 1541 ddp_sgl_check(struct ctl_sg_entry *sg, int entries, int xferlen) 1542 { 1543 #ifdef INVARIANTS 1544 int total_len = 0; 1545 #endif 1546 1547 MPASS(entries > 0); 1548 if (((vm_offset_t)sg[--entries].addr & 3U) != 0) 1549 return (false); 1550 1551 #ifdef INVARIANTS 1552 total_len += sg[entries].len; 1553 #endif 1554 1555 while (--entries >= 0) { 1556 if (((vm_offset_t)sg[entries].addr & PAGE_MASK) != 0 || 1557 (sg[entries].len % PAGE_SIZE) != 0) 1558 return (false); 1559 #ifdef INVARIANTS 1560 total_len += sg[entries].len; 1561 #endif 1562 } 1563 1564 MPASS(total_len == xferlen); 1565 return (true); 1566 } 1567 1568 #define io_to_ddp_state(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr) 1569 1570 int 1571 icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, struct icl_pdu *ip, 1572 union ctl_io *io, uint32_t *tttp, void **arg) 1573 { 1574 struct icl_cxgbei_conn *icc = ic_to_icc(ic); 1575 struct toepcb *toep = icc->toep; 1576 struct ctl_scsiio *ctsio = &io->scsiio; 1577 struct adapter *sc = icc->sc; 1578 struct cxgbei_data *ci = sc->iscsi_ulp_softc; 1579 struct ppod_region *pr = &ci->pr; 1580 struct cxgbei_ddp_state *ddp; 1581 struct ppod_reservation *prsv; 1582 struct ctl_sg_entry *sgl, sg_entry; 1583 struct inpcb *inp; 1584 struct mbufq mq; 1585 int sg_entries = ctsio->kern_sg_entries; 1586 uint32_t ttt; 1587 int xferlen, rc = 0, alias; 1588 1589 /* This is for the offload driver's state. Must not be set already. */ 1590 MPASS(arg != NULL); 1591 MPASS(*arg == NULL); 1592 1593 if (ctsio->ext_data_filled == 0) { 1594 int first_burst; 1595 #ifdef INVARIANTS 1596 struct icl_cxgbei_pdu *icp = ip_to_icp(ip); 1597 1598 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); 1599 MPASS(ic == ip->ip_conn); 1600 MPASS(ip->ip_bhs_mbuf != NULL); 1601 #endif 1602 first_burst = icl_pdu_data_segment_length(ip); 1603 1604 /* 1605 * Note that ICL calls conn_transfer_setup even if the first 1606 * burst had everything and there's nothing left to transfer. 1607 * 1608 * NB: The CTL frontend might have provided a buffer 1609 * whose length (kern_data_len) is smaller than the 1610 * FirstBurstLength of unsolicited data. Treat those 1611 * as an empty transfer. 1612 */ 1613 xferlen = ctsio->kern_data_len; 1614 if (xferlen < first_burst || 1615 xferlen - first_burst < ci->ddp_threshold) { 1616 no_ddp: 1617 /* 1618 * No DDP for this transfer. Allocate a TTT (based on 1619 * the one passed in) that cannot be a valid hardware 1620 * DDP tag in the iSCSI region. 1621 */ 1622 ttt = *tttp & M_PPOD_TAG; 1623 ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit; 1624 *tttp = htobe32(ttt); 1625 MPASS(io_to_ddp_state(io) == NULL); 1626 if (rc != 0) 1627 counter_u64_add( 1628 toep->ofld_rxq->rx_iscsi_ddp_setup_error, 1); 1629 return (0); 1630 } 1631 1632 if (sg_entries == 0) { 1633 sgl = &sg_entry; 1634 sgl->len = xferlen; 1635 sgl->addr = (void *)ctsio->kern_data_ptr; 1636 sg_entries = 1; 1637 } else 1638 sgl = (void *)ctsio->kern_data_ptr; 1639 1640 if (!ddp_sgl_check(sgl, sg_entries, xferlen)) 1641 goto no_ddp; 1642 1643 /* 1644 * Reserve resources for DDP, update the ttt that should be used 1645 * in the PDU, and save DDP specific state for this I/O. 1646 */ 1647 MPASS(io_to_ddp_state(io) == NULL); 1648 ddp = malloc(sizeof(*ddp), M_CXGBEI, M_NOWAIT | M_ZERO); 1649 if (ddp == NULL) { 1650 rc = ENOMEM; 1651 goto no_ddp; 1652 } 1653 prsv = &ddp->prsv; 1654 1655 rc = t4_alloc_page_pods_for_sgl(pr, sgl, sg_entries, prsv); 1656 if (rc != 0) { 1657 free(ddp, M_CXGBEI); 1658 goto no_ddp; 1659 } 1660 1661 mbufq_init(&mq, INT_MAX); 1662 rc = t4_write_page_pods_for_sgl(sc, toep, prsv, sgl, sg_entries, 1663 xferlen, &mq); 1664 if (__predict_false(rc != 0)) { 1665 mbufq_drain(&mq); 1666 t4_free_page_pods(prsv); 1667 free(ddp, M_CXGBEI); 1668 goto no_ddp; 1669 } 1670 1671 /* 1672 * Do not get inp from toep->inp as the toepcb might 1673 * have detached already. 1674 */ 1675 ICL_CONN_LOCK(ic); 1676 if (ic->ic_disconnecting || ic->ic_socket == NULL) { 1677 ICL_CONN_UNLOCK(ic); 1678 mbufq_drain(&mq); 1679 t4_free_page_pods(prsv); 1680 free(ddp, M_CXGBEI); 1681 return (ECONNRESET); 1682 } 1683 inp = sotoinpcb(ic->ic_socket); 1684 INP_WLOCK(inp); 1685 ICL_CONN_UNLOCK(ic); 1686 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) != 0) { 1687 INP_WUNLOCK(inp); 1688 mbufq_drain(&mq); 1689 t4_free_page_pods(prsv); 1690 free(ddp, M_CXGBEI); 1691 return (ECONNRESET); 1692 } 1693 mbufq_concat(&toep->ulp_pduq, &mq); 1694 INP_WUNLOCK(inp); 1695 1696 ddp->cmp.next_buffer_offset = ctsio->kern_rel_offset + 1697 first_burst; 1698 ddp->cmp.last_datasn = -1; 1699 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); 1700 *tttp = htobe32(prsv->prsv_tag); 1701 io_to_ddp_state(io) = ddp; 1702 *arg = ctsio; 1703 counter_u64_add(toep->ofld_rxq->rx_iscsi_ddp_setup_ok, 1); 1704 return (0); 1705 } 1706 1707 /* 1708 * In the middle of an I/O. A non-NULL page pod reservation indicates 1709 * that a DDP buffer is being used for the I/O. 1710 */ 1711 ddp = io_to_ddp_state(ctsio); 1712 if (ddp == NULL) 1713 goto no_ddp; 1714 prsv = &ddp->prsv; 1715 1716 alias = (prsv->prsv_tag & pr->pr_alias_mask) >> pr->pr_alias_shift; 1717 alias++; 1718 prsv->prsv_tag &= ~pr->pr_alias_mask; 1719 prsv->prsv_tag |= alias << pr->pr_alias_shift & pr->pr_alias_mask; 1720 1721 ddp->cmp.last_datasn = -1; 1722 cxgbei_insert_cmp(icc, &ddp->cmp, prsv->prsv_tag); 1723 *tttp = htobe32(prsv->prsv_tag); 1724 *arg = ctsio; 1725 1726 return (0); 1727 } 1728 1729 void 1730 icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *arg) 1731 { 1732 struct ctl_scsiio *ctsio = arg; 1733 1734 if (ctsio != NULL) { 1735 struct cxgbei_ddp_state *ddp; 1736 1737 ddp = io_to_ddp_state(ctsio); 1738 MPASS(ddp != NULL); 1739 1740 cxgbei_rm_cmp(ic_to_icc(ic), &ddp->cmp); 1741 if (ctsio->kern_data_len == ctsio->ext_data_filled || 1742 ic->ic_disconnecting) { 1743 t4_free_page_pods(&ddp->prsv); 1744 free(ddp, M_CXGBEI); 1745 io_to_ddp_state(ctsio) = NULL; 1746 } 1747 } 1748 } 1749 1750 #ifdef COMPAT_FREEBSD13 1751 static void 1752 cxgbei_limits(struct adapter *sc, void *arg) 1753 { 1754 struct icl_drv_limits *idl = arg; 1755 struct cxgbei_data *ci; 1756 int max_dsl; 1757 1758 if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims") != 0) 1759 return; 1760 1761 if (uld_active(sc, ULD_ISCSI)) { 1762 ci = sc->iscsi_ulp_softc; 1763 MPASS(ci != NULL); 1764 1765 1766 max_dsl = ci->max_rx_data_len; 1767 if (idl->idl_max_recv_data_segment_length > max_dsl) 1768 idl->idl_max_recv_data_segment_length = max_dsl; 1769 1770 max_dsl = ci->max_tx_data_len; 1771 if (idl->idl_max_send_data_segment_length > max_dsl) 1772 idl->idl_max_send_data_segment_length = max_dsl; 1773 } 1774 1775 end_synchronized_op(sc, LOCK_HELD); 1776 } 1777 #endif 1778 1779 static int 1780 cxgbei_limits_fd(struct icl_drv_limits *idl, int fd) 1781 { 1782 struct find_ofld_adapter_rr fa; 1783 struct file *fp; 1784 struct socket *so; 1785 struct adapter *sc; 1786 struct cxgbei_data *ci; 1787 cap_rights_t rights; 1788 int error; 1789 1790 error = fget(curthread, fd, 1791 cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp); 1792 if (error != 0) 1793 return (error); 1794 if (fp->f_type != DTYPE_SOCKET) { 1795 fdrop(fp, curthread); 1796 return (EINVAL); 1797 } 1798 so = fp->f_data; 1799 if (so->so_type != SOCK_STREAM || 1800 so->so_proto->pr_protocol != IPPROTO_TCP) { 1801 fdrop(fp, curthread); 1802 return (EINVAL); 1803 } 1804 1805 /* Find the adapter offloading this socket. */ 1806 fa.sc = NULL; 1807 fa.so = so; 1808 t4_iterate(find_offload_adapter, &fa); 1809 if (fa.sc == NULL) { 1810 fdrop(fp, curthread); 1811 return (ENXIO); 1812 } 1813 fdrop(fp, curthread); 1814 1815 sc = fa.sc; 1816 error = begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims"); 1817 if (error != 0) 1818 return (error); 1819 1820 if (uld_active(sc, ULD_ISCSI)) { 1821 ci = sc->iscsi_ulp_softc; 1822 MPASS(ci != NULL); 1823 1824 idl->idl_max_recv_data_segment_length = ci->max_rx_data_len; 1825 idl->idl_max_send_data_segment_length = ci->max_tx_data_len; 1826 } else 1827 error = ENXIO; 1828 1829 end_synchronized_op(sc, LOCK_HELD); 1830 1831 return (error); 1832 } 1833 1834 static int 1835 icl_cxgbei_limits(struct icl_drv_limits *idl, int socket) 1836 { 1837 1838 /* Maximum allowed by the RFC. cxgbei_limits will clip them. */ 1839 idl->idl_max_recv_data_segment_length = (1 << 24) - 1; 1840 idl->idl_max_send_data_segment_length = (1 << 24) - 1; 1841 1842 /* These are somewhat arbitrary. */ 1843 idl->idl_max_burst_length = max_burst_length; 1844 idl->idl_first_burst_length = first_burst_length; 1845 1846 #ifdef COMPAT_FREEBSD13 1847 if (socket == 0) { 1848 t4_iterate(cxgbei_limits, idl); 1849 return (0); 1850 } 1851 #endif 1852 1853 return (cxgbei_limits_fd(idl, socket)); 1854 } 1855 1856 int 1857 icl_cxgbei_mod_load(void) 1858 { 1859 int rc; 1860 1861 refcount_init(&icl_cxgbei_ncons, 0); 1862 1863 rc = icl_register("cxgbei", false, -100, icl_cxgbei_limits, 1864 icl_cxgbei_new_conn); 1865 1866 return (rc); 1867 } 1868 1869 int 1870 icl_cxgbei_mod_unload(void) 1871 { 1872 1873 if (icl_cxgbei_ncons != 0) 1874 return (EBUSY); 1875 1876 icl_unregister("cxgbei", false); 1877 1878 return (0); 1879 } 1880 #endif 1881