1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/ddi.h> 28 #include <sys/types.h> 29 #include <sys/socket.h> 30 #include <netinet/in.h> 31 #include <sys/sunddi.h> 32 #include <sys/ib/ibtl/ibti.h> 33 #include <sys/ib/ibtl/ibtl_types.h> 34 35 #include <sys/ib/clients/iser/iser.h> 36 37 extern idm_transport_ops_t iser_transport_ops; 38 39 /* 40 * iser_cm.c 41 * InfiniBand Communication Manager routines for iSER 42 */ 43 static ibt_cm_status_t iser_ib_handle_cm_req(idm_svc_t *svc_hdl, 44 ibt_cm_event_t *evp, ibt_cm_return_args_t *rargsp, void *rcmp, 45 ibt_priv_data_len_t rcmp_len); 46 47 static ibt_cm_status_t iser_ib_handle_cm_rep(iser_state_t *statep, 48 ibt_cm_event_t *evp, ibt_cm_return_args_t *rargsp, void *rcmp, 49 ibt_priv_data_len_t rcmp_len); 50 51 static ibt_cm_status_t iser_handle_cm_conn_est(ibt_cm_event_t *evp); 52 static ibt_cm_status_t iser_handle_cm_conn_closed(ibt_cm_event_t *evp); 53 static ibt_cm_status_t iser_handle_cm_event_failure(ibt_cm_event_t *evp); 54 55 /* 56 * iser_ib_cm_handler() 57 */ 58 ibt_cm_status_t 59 iser_ib_cm_handler(void *cm_private, ibt_cm_event_t *eventp, 60 ibt_cm_return_args_t *ret_args, void *ret_priv_data, 61 ibt_priv_data_len_t ret_len_max) 62 { 63 ibt_cm_status_t ret = IBT_CM_REJECT; 64 65 switch (eventp->cm_type) { 66 67 case IBT_CM_EVENT_REQ_RCV: 68 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_REQ_RCV"); 69 ret = iser_ib_handle_cm_req((idm_svc_t *)cm_private, eventp, 70 ret_args, ret_priv_data, ret_len_max); 71 break; 72 73 case IBT_CM_EVENT_REP_RCV: 74 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_REP_RCV"); 75 ret = iser_ib_handle_cm_rep((iser_state_t *)cm_private, 76 eventp, ret_args, ret_priv_data, ret_len_max); 77 break; 78 79 case IBT_CM_EVENT_CONN_EST: 80 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: IBT_CM_EVENT_CONN_EST"); 81 ret = iser_handle_cm_conn_est(eventp); 82 break; 83 84 case IBT_CM_EVENT_CONN_CLOSED: 85 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: " 86 "IBT_CM_EVENT_CONN_CLOSED"); 87 ret = iser_handle_cm_conn_closed(eventp); 88 break; 89 90 case IBT_CM_EVENT_FAILURE: 91 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: Event failure"); 92 ret = iser_handle_cm_event_failure(eventp); 93 break; 94 95 case IBT_CM_EVENT_MRA_RCV: 96 /* Not supported */ 97 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: MRA message received"); 98 break; 99 100 case IBT_CM_EVENT_LAP_RCV: 101 /* Not supported */ 102 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: LAP message received"); 103 break; 104 105 case IBT_CM_EVENT_APR_RCV: 106 /* Not supported */ 107 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: APR message received"); 108 break; 109 110 default: 111 ISER_LOG(CE_NOTE, "iser_ib_cm_handler: unknown event (0x%x)", 112 eventp->cm_type); 113 break; 114 } 115 116 return (ret); 117 } 118 119 /* ARGSUSED */ 120 static ibt_cm_status_t 121 iser_ib_handle_cm_req(idm_svc_t *svc_hdl, ibt_cm_event_t *evp, 122 ibt_cm_return_args_t *rargsp, void *rcmp, ibt_priv_data_len_t rcmp_len) 123 { 124 125 iser_private_data_t iser_priv_data; 126 ibt_ip_cm_info_t ipcm_info; 127 iser_chan_t *chan; 128 iser_conn_t *iser_conn; 129 int status; 130 131 /* 132 * CM private data brings IP information 133 * Private data received is a stream of bytes and may not be properly 134 * aligned. So, bcopy the data onto the stack before accessing it. 135 */ 136 bcopy((uint8_t *)evp->cm_priv_data, &iser_priv_data, 137 sizeof (iser_private_data_t)); 138 139 /* extract the CM IP info */ 140 status = ibt_get_ip_data(evp->cm_priv_data_len, evp->cm_priv_data, 141 &ipcm_info); 142 if (status != IBT_SUCCESS) { 143 return (IBT_CM_REJECT); 144 } 145 146 ISER_LOG(CE_NOTE, "iser_ib_handle_cm_req: ipcm_info (0x%p): src IP " 147 "(0x%08x) src port (0x%04x) dst IP: (0x%08x)", (void *)&ipcm_info, 148 ipcm_info.src_addr.un.ip4addr, ipcm_info.src_port, 149 ipcm_info.dst_addr.un.ip4addr); 150 151 /* Allocate a channel to establish the new connection */ 152 chan = iser_ib_alloc_rc_channel(&ipcm_info.dst_addr, 153 &ipcm_info.src_addr); 154 if (chan == NULL) { 155 return (IBT_CM_REJECT); 156 } 157 158 /* Set the local and remote port numbers on the channel handle */ 159 chan->ic_lport = svc_hdl->is_svc_req.sr_port; 160 chan->ic_rport = ipcm_info.src_port; 161 162 /* Allocate the iser_conn_t for the IDM svc binding */ 163 iser_conn = kmem_zalloc(sizeof (iser_conn_t), KM_SLEEP); 164 165 /* Set up the iser_conn attributes */ 166 mutex_init(&iser_conn->ic_lock, NULL, MUTEX_DRIVER, NULL); 167 cv_init(&iser_conn->ic_stage_cv, NULL, CV_DEFAULT, NULL); 168 iser_conn->ic_type = ISER_CONN_TYPE_TGT; 169 iser_conn->ic_chan = chan; 170 iser_conn->ic_stage = ISER_CONN_STAGE_ALLOCATED; 171 172 /* Hold a reference to the iSER service handle */ 173 iser_tgt_svc_hold((iser_svc_t *)svc_hdl->is_iser_svc); 174 175 iser_conn->ic_idms = svc_hdl; 176 177 /* 178 * Now set a pointer to the iser_conn in the iser_chan for 179 * access during CM event handling 180 */ 181 chan->ic_conn = iser_conn; 182 183 rargsp->cm_ret.rep.cm_channel = chan->ic_chanhdl; 184 185 return (IBT_CM_ACCEPT); 186 } 187 188 /* ARGSUSED */ 189 static ibt_cm_status_t 190 iser_ib_handle_cm_rep(iser_state_t *statep, ibt_cm_event_t *evp, 191 ibt_cm_return_args_t *rargsp, void *rcmp, ibt_priv_data_len_t rcmp_len) 192 { 193 /* pre-post work requests into the receive queue */ 194 iser_ib_post_recv(evp->cm_channel); 195 196 /* It looks like the RTU need not be send specifically */ 197 return (IBT_CM_ACCEPT); 198 } 199 200 static ibt_cm_status_t 201 iser_handle_cm_conn_est(ibt_cm_event_t *evp) 202 { 203 iser_chan_t *iser_chan; 204 iser_conn_t *iser_conn; 205 iser_svc_t *iser_svc; 206 idm_status_t status; 207 idm_conn_t *ic; 208 209 iser_chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel); 210 211 /* 212 * An ibt_open_rc_channel() comes in as a IBT_CM_EVENT_REQ_RCV on the 213 * iSER-IB target, upon which the target sends a Response, accepting 214 * the request. This comes in as a IBT_CM_EVENT_REP_RCV on the iSER-IB 215 * initiator, which then sends an RTU. Upon getting this RTU from the 216 * iSER-IB initiator, the IBT_CM_EVENT_CONN_EST event is generated on 217 * the target. Then subsequently an IBT_CM_EVENT_CONN_EST event is 218 * generated on the initiator. 219 * 220 * Our new connection has been established on the target. If we are 221 * receiving this event on the target side, the iser_channel can be 222 * used as it is already populated. On the target side, an IDM 223 * connection is then allocated and the IDM layer is notified. 224 * If we are on the initiator we needn't do anything, since we 225 * already have the IDM linkage in place for this connection. 226 */ 227 if (iser_chan->ic_conn->ic_type == ISER_CONN_TYPE_TGT) { 228 229 iser_conn = iser_chan->ic_conn; 230 iser_svc = (iser_svc_t *)iser_conn->ic_idms->is_iser_svc; 231 232 mutex_enter(&iser_conn->ic_lock); 233 234 status = idm_svc_conn_create(iser_conn->ic_idms, 235 IDM_TRANSPORT_TYPE_ISER, &ic); 236 if (status != IDM_STATUS_SUCCESS) { 237 /* 238 * No IDM rsrcs or something equally Bad. 239 * Return non-SUCCESS to IBCM. He'll give 240 * us a CONN_CLOSED, which we'll handle 241 * below. 242 */ 243 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_est: " 244 "idm_svc_conn_create_failed"); 245 mutex_exit(&iser_conn->ic_lock); 246 return (IBT_CM_NO_RESOURCE); 247 } 248 249 /* We no longer need the hold on the iSER service handle */ 250 iser_tgt_svc_rele(iser_svc); 251 252 /* Hold a reference on the IDM connection handle */ 253 idm_conn_hold(ic); 254 255 /* Set the transport ops and conn on the idm_conn handle */ 256 ic->ic_transport_ops = &iser_transport_ops; 257 ic->ic_transport_private = (void *)iser_conn; 258 ic->ic_transport_hdrlen = ISER_HEADER_LENGTH; 259 iser_conn->ic_idmc = ic; 260 261 /* 262 * Set the local and remote addresses in the idm conn handle. 263 */ 264 iser_ib_conv_ibtaddr2sockaddr(&ic->ic_laddr, 265 &iser_conn->ic_chan->ic_localip, iser_chan->ic_lport); 266 iser_ib_conv_ibtaddr2sockaddr(&ic->ic_raddr, 267 &iser_conn->ic_chan->ic_remoteip, iser_chan->ic_rport); 268 269 /* 270 * Kick the state machine. At CS_S3_XPT_UP the state machine 271 * will notify the client (target) about the new connection. 272 */ 273 idm_conn_event(ic, CE_CONNECT_ACCEPT, NULL); 274 iser_conn->ic_stage = ISER_CONN_STAGE_IC_CONNECTED; 275 mutex_exit(&iser_conn->ic_lock); 276 277 /* 278 * Post work requests on the receive queue 279 */ 280 iser_ib_post_recv(iser_chan->ic_chanhdl); 281 282 } 283 284 return (IBT_CM_ACCEPT); 285 } 286 287 static ibt_cm_status_t 288 iser_handle_cm_conn_closed(ibt_cm_event_t *evp) 289 { 290 291 iser_chan_t *chan; 292 293 chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel); 294 295 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_closed: chan (0x%p) " 296 "reason (0x%x)", (void *)chan, evp->cm_event.closed); 297 298 switch (evp->cm_event.closed) { 299 case IBT_CM_CLOSED_DREP_RCVD: /* we requested a disconnect */ 300 case IBT_CM_CLOSED_ALREADY: /* duplicate close */ 301 /* ignore these */ 302 return (IBT_CM_ACCEPT); 303 304 case IBT_CM_CLOSED_DREQ_RCVD: /* request to close the channel */ 305 case IBT_CM_CLOSED_REJ_RCVD: /* reject after conn establishment */ 306 case IBT_CM_CLOSED_DREQ_TIMEOUT: /* our close request timed out */ 307 case IBT_CM_CLOSED_DUP: /* duplicate close request */ 308 case IBT_CM_CLOSED_ABORT: /* aborted connection establishment */ 309 case IBT_CM_CLOSED_STALE: /* stale / unref connection */ 310 /* handle these depending upon our connection state */ 311 mutex_enter(&chan->ic_conn->ic_lock); 312 switch (chan->ic_conn->ic_stage) { 313 case ISER_CONN_STAGE_UNDEFINED: 314 case ISER_CONN_STAGE_CLOSED: 315 /* do nothing, just drop the lock */ 316 mutex_exit(&chan->ic_conn->ic_lock); 317 break; 318 319 case ISER_CONN_STAGE_ALLOCATED: 320 /* 321 * We blew up or were offlined during connection 322 * establishment. Teardown the iSER conn and chan 323 * handles. 324 */ 325 mutex_exit(&chan->ic_conn->ic_lock); 326 iser_internal_conn_destroy(chan->ic_conn); 327 break; 328 329 case ISER_CONN_STAGE_IC_DISCONNECTED: 330 case ISER_CONN_STAGE_IC_FREED: 331 case ISER_CONN_STAGE_CLOSING: 332 /* we're down, set CLOSED */ 333 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSED; 334 mutex_exit(&chan->ic_conn->ic_lock); 335 break; 336 337 case ISER_CONN_STAGE_IC_CONNECTED: 338 case ISER_CONN_STAGE_HELLO_SENT: 339 case ISER_CONN_STAGE_HELLO_SENT_FAIL: 340 case ISER_CONN_STAGE_HELLO_WAIT: 341 case ISER_CONN_STAGE_HELLO_RCV: 342 case ISER_CONN_STAGE_HELLO_RCV_FAIL: 343 case ISER_CONN_STAGE_HELLOREPLY_SENT: 344 case ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL: 345 case ISER_CONN_STAGE_HELLOREPLY_RCV: 346 case ISER_CONN_STAGE_HELLOREPLY_RCV_FAIL: 347 case ISER_CONN_STAGE_LOGGED_IN: 348 /* for all other stages, fail the transport */ 349 idm_conn_event(chan->ic_conn->ic_idmc, 350 CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 351 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSING; 352 mutex_exit(&chan->ic_conn->ic_lock); 353 break; 354 355 default: 356 mutex_exit(&chan->ic_conn->ic_lock); 357 ASSERT(0); 358 359 } 360 361 /* accept the event */ 362 return (IBT_CM_ACCEPT); 363 364 default: 365 /* unknown event */ 366 ISER_LOG(CE_NOTE, "iser_handle_cm_conn_closed: unknown closed " 367 "event: (0x%x)", evp->cm_event.closed); 368 return (IBT_CM_REJECT); 369 } 370 } 371 372 /* 373 * Handle EVENT FAILURE 374 */ 375 static ibt_cm_status_t 376 iser_handle_cm_event_failure(ibt_cm_event_t *evp) 377 { 378 iser_chan_t *chan; 379 380 chan = (iser_chan_t *)ibt_get_chan_private(evp->cm_channel); 381 382 ISER_LOG(CE_NOTE, "iser_handle_cm_event_failure: chan (0x%p): " 383 "code: %d msg: %d reason: %d", (void *)chan, 384 evp->cm_event.failed.cf_code, evp->cm_event.failed.cf_msg, 385 evp->cm_event.failed.cf_reason); 386 387 if ((evp->cm_channel == NULL) || (chan == NULL)) { 388 /* channel not established yet */ 389 return (IBT_CM_ACCEPT); 390 } 391 392 if ((evp->cm_event.failed.cf_code != IBT_CM_FAILURE_STALE) && 393 (evp->cm_event.failed.cf_msg == IBT_CM_FAILURE_REQ)) { 394 /* 395 * This end is active, just ignore, ibt_open_rc_channel() 396 * caller will take care of cleanup. 397 */ 398 return (IBT_CM_ACCEPT); 399 } 400 401 /* handle depending upon our connection state */ 402 mutex_enter(&chan->ic_conn->ic_lock); 403 switch (chan->ic_conn->ic_stage) { 404 case ISER_CONN_STAGE_UNDEFINED: 405 case ISER_CONN_STAGE_CLOSED: 406 /* do nothing, just drop the lock */ 407 mutex_exit(&chan->ic_conn->ic_lock); 408 break; 409 410 case ISER_CONN_STAGE_ALLOCATED: 411 /* 412 * We blew up or were offlined during connection 413 * establishment. Teardown the iSER conn and chan 414 * handles. 415 */ 416 mutex_exit(&chan->ic_conn->ic_lock); 417 iser_internal_conn_destroy(chan->ic_conn); 418 break; 419 420 case ISER_CONN_STAGE_IC_DISCONNECTED: 421 case ISER_CONN_STAGE_IC_FREED: 422 case ISER_CONN_STAGE_CLOSING: 423 /* update to CLOSED, then drop the lock */ 424 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSED; 425 mutex_exit(&chan->ic_conn->ic_lock); 426 break; 427 428 case ISER_CONN_STAGE_IC_CONNECTED: 429 case ISER_CONN_STAGE_HELLO_SENT: 430 case ISER_CONN_STAGE_HELLO_SENT_FAIL: 431 case ISER_CONN_STAGE_HELLO_WAIT: 432 case ISER_CONN_STAGE_HELLO_RCV: 433 case ISER_CONN_STAGE_HELLO_RCV_FAIL: 434 case ISER_CONN_STAGE_HELLOREPLY_SENT: 435 case ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL: 436 case ISER_CONN_STAGE_HELLOREPLY_RCV: 437 case ISER_CONN_STAGE_HELLOREPLY_RCV_FAIL: 438 case ISER_CONN_STAGE_LOGGED_IN: 439 /* fail the transport and move the conn to CLOSING */ 440 idm_conn_event(chan->ic_conn->ic_idmc, CE_TRANSPORT_FAIL, 441 IDM_STATUS_FAIL); 442 chan->ic_conn->ic_stage = ISER_CONN_STAGE_CLOSING; 443 mutex_exit(&chan->ic_conn->ic_lock); 444 break; 445 446 default: 447 mutex_exit(&chan->ic_conn->ic_lock); 448 ASSERT(0); 449 } 450 451 /* accept the event */ 452 return (IBT_CM_ACCEPT); 453 } 454