1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2017 Joyent, Inc. 25 */ 26 27 /* 28 * The sol_ucma driver provides the API for librdmacm library for RDMACM 29 * functionality. 30 * 31 * sol_uverbs will create a minor node with prefix ":ucma", 32 * which can be opened only by the kernel (cred == kcred). 33 * 34 * sol_cma driver will open and close the sol_uverb minor 35 * device using the Layered Driver Interfaces (See PSARC 36 * 2001/769). 37 */ 38 39 /* Standard driver includes */ 40 #include <sys/types.h> 41 #include <sys/modctl.h> 42 #include <sys/ddi.h> 43 #include <sys/sunddi.h> 44 #include <sys/file.h> 45 #include <sys/errno.h> 46 #include <sys/open.h> 47 #include <sys/cred.h> 48 #include <sys/stat.h> 49 #include <sys/ddi.h> 50 #include <sys/sunddi.h> 51 #include <sys/conf.h> 52 #include <sys/uio.h> 53 #include <sys/sunldi.h> 54 #include <sys/modctl.h> 55 56 /* Common header files */ 57 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h> 58 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs2ucma.h> 59 #include <sys/ib/clients/of/ofed_kernel.h> 60 61 /* Kernel Headers for User rdma_cm API */ 62 #include <sys/ib/clients/of/rdma/ib_addr.h> 63 #include <sys/ib/clients/of/rdma/rdma_user_cm.h> 64 65 /* Kernel rdma_cm API */ 66 #include <sys/ib/clients/of/rdma/rdma_cm.h> 67 68 /* sol_ucma internal Header files */ 69 #include <sys/ib/clients/of/sol_ucma/sol_ucma.h> 70 71 /* entry point function prototype declarations */ 72 static int sol_ucma_attach(dev_info_t *, ddi_attach_cmd_t); 73 static int sol_ucma_detach(dev_info_t *, ddi_detach_cmd_t); 74 static int sol_ucma_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 75 static int sol_ucma_open(dev_t *, int, int, cred_t *); 76 static int sol_ucma_close(dev_t, int, int, cred_t *); 77 static int sol_ucma_write(dev_t, struct uio *, cred_t *); 78 static int sol_ucma_poll(dev_t, short, int, short *, struct pollhead **); 79 80 /* Driver entry points */ 81 static struct cb_ops sol_ucma_cb_ops = { 82 sol_ucma_open, /* open */ 83 sol_ucma_close, /* close */ 84 nodev, /* strategy (block) */ 85 nodev, /* print (block) */ 86 nodev, /* dump (block) */ 87 nodev, /* read */ 88 sol_ucma_write, /* write */ 89 nodev, /* ioctl */ 90 nodev, /* devmap */ 91 nodev, /* mmap */ 92 nodev, /* segmap */ 93 sol_ucma_poll, /* chpoll */ 94 ddi_prop_op, /* prop_op */ 95 NULL, /* streams */ 96 D_NEW | D_MP | D_64BIT, /* flags */ 97 CB_REV /* rev */ 98 }; 99 100 /* Driver operations */ 101 static struct dev_ops sol_ucma_dev_ops = { 102 DEVO_REV, /* struct rev */ 103 0, /* refcnt */ 104 sol_ucma_getinfo, /* getinfo */ 105 nulldev, /* identify */ 106 nulldev, /* probe */ 107 sol_ucma_attach, /* attach */ 108 sol_ucma_detach, /* detach */ 109 nodev, /* reset */ 110 &sol_ucma_cb_ops, /* cb_ops */ 111 NULL, /* bus_ops */ 112 nodev, /* power */ 113 ddi_quiesce_not_needed /* quiesce */ 114 }; 115 116 /* Module Driver Info */ 117 static struct modldrv sol_ucma_modldrv = { 118 &mod_driverops, 119 "Solaris User RDMACM driver", 120 &sol_ucma_dev_ops 121 }; 122 123 /* Module Linkage */ 124 static struct modlinkage sol_ucma_modlinkage = { 125 MODREV_1, 126 &sol_ucma_modldrv, 127 NULL, 128 }; 129 130 static char *sol_ucma_dbg_str = "sol_ucma"; 131 sol_ofs_uobj_table_t ucma_file_uo_tbl; 132 sol_ofs_uobj_table_t ucma_ctx_uo_tbl; 133 sol_ofs_uobj_table_t ucma_mcast_uo_tbl; 134 135 /* Function pointers for uverbs functions */ 136 static uverbs_get_clnt_hdl_t uverbs_get_hdl_fp = NULL; 137 static uverbs_qpnum2qphdl_t uverbs_qpnum2qphdl_fp = NULL; 138 static uverbs_disable_uqpn_mod_t uverbs_disable_uqpn_modify_fp = NULL; 139 static uverbs_uqpn_cq_ctrl_t uverbs_uqpn_cq_ctrl_fp = NULL; 140 static uverbs_set_qp_free_state_t uverbs_set_qp_free_state_fp = NULL; 141 static uverbs_flush_qp_t uverbs_flush_qp_fp = NULL; 142 143 /* Global Variables */ 144 sol_ucma_t sol_ucma; 145 146 /* RDMACM Functions */ 147 static int sol_ucma_create_id(dev_t, void *, struct uio *); 148 static int sol_ucma_destroy_id(dev_t, void *, struct uio *); 149 static int sol_ucma_bind_addr(dev_t, void *, struct uio *); 150 static int sol_ucma_resolve_addr(dev_t, void *, struct uio *); 151 static int sol_ucma_resolve_route(dev_t, void *, struct uio *); 152 static int sol_ucma_query_route(dev_t, void *, struct uio *); 153 static int sol_ucma_connect(dev_t, void *, struct uio *); 154 static int sol_ucma_listen(dev_t, void *, struct uio *); 155 static int sol_ucma_accept(dev_t, void *, struct uio *); 156 static int sol_ucma_reject(dev_t, void *, struct uio *); 157 static int sol_ucma_disconnect(dev_t, void *, struct uio *); 158 static int sol_ucma_init_qp_attr(dev_t, void *, struct uio *); 159 static int sol_ucma_get_event(dev_t, void *, struct uio *); 160 static int sol_ucma_set_option(dev_t, void *, struct uio *); 161 static int sol_ucma_notify(dev_t, void *, struct uio *); 162 static int sol_ucma_join_mcast(dev_t, void *, struct uio *); 163 static int sol_ucma_leave_mcast(dev_t, void *, struct uio *); 164 165 /* 166 * Event callback from sol_cma 167 */ 168 int sol_ucma_evt_hdlr(struct rdma_cm_id *, struct rdma_cm_event *); 169 170 /* 171 * Internal functions. 172 */ 173 static sol_ucma_file_t * 174 ucma_alloc_file(minor_t *); 175 176 static sol_ucma_chan_t * 177 ucma_alloc_chan(sol_ucma_file_t *, sol_ucma_create_id_t *); 178 179 static void 180 ucma_free_chan(sol_ucma_chan_t *, int); 181 182 static int 183 get_file_chan(uint32_t, sol_ucma_file_t **, sol_ucma_chan_t **, char *, int); 184 185 static void 186 rdma2usr_route(struct rdma_cm_id *, sol_ucma_query_route_resp_t *); 187 188 static void 189 usr2rdma_conn_param(struct rdma_ucm_conn_param *, struct rdma_conn_param *); 190 191 static void 192 rdma2usr_conn_param(struct rdma_conn_param *, struct rdma_ucm_conn_param *); 193 194 static void 195 rdma2usr_ud_param(struct rdma_ud_param *, sol_ucma_ud_param_t *); 196 197 static void sol_ucma_user_objs_init(); 198 static void sol_ucma_user_objs_fini(); 199 200 int 201 _init(void) 202 { 203 int error; 204 205 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_init()"); 206 sol_ucma_user_objs_init(); 207 mutex_init(&sol_ucma.ucma_mutex, NULL, MUTEX_DRIVER, NULL); 208 cv_init(&sol_ucma.ucma_open_cv, NULL, CV_DRIVER, NULL); 209 210 if ((error = ldi_ident_from_mod(&sol_ucma_modlinkage, 211 &sol_ucma.ucma_ldi_ident)) != 0) { 212 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 213 "ldi_ident_from_mod() failed"); 214 mutex_destroy(&sol_ucma.ucma_mutex); 215 cv_destroy(&sol_ucma.ucma_open_cv); 216 sol_ucma_user_objs_fini(); 217 return (error); 218 } 219 sol_ucma.ucma_clnt_hdl_flag = SOL_UCMA_CLNT_HDL_UNINITIALIZED; 220 error = mod_install(&sol_ucma_modlinkage); 221 if (error) { 222 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "mod_install() failed"); 223 ldi_ident_release(sol_ucma.ucma_ldi_ident); 224 mutex_destroy(&sol_ucma.ucma_mutex); 225 cv_destroy(&sol_ucma.ucma_open_cv); 226 sol_ucma_user_objs_fini(); 227 return (error); 228 } 229 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_init(): ret"); 230 return (error); 231 } 232 233 int 234 _info(struct modinfo *modinfop) 235 { 236 return (mod_info(&sol_ucma_modlinkage, modinfop)); 237 } 238 239 int 240 _fini(void) 241 { 242 int ret; 243 244 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_fini()"); 245 if ((ret = mod_remove(&sol_ucma_modlinkage)) != 0) { 246 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, 247 "sol_ucma, _fini : mod_remove failed"); 248 return (ret); 249 } 250 ldi_ident_release(sol_ucma.ucma_ldi_ident); 251 mutex_destroy(&sol_ucma.ucma_mutex); 252 cv_destroy(&sol_ucma.ucma_open_cv); 253 sol_ucma_user_objs_fini(); 254 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_fini(): ret"); 255 return (DDI_SUCCESS); 256 } 257 258 static int 259 sol_ucma_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 260 { 261 int rval; 262 263 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "attach(%p, %x)", dip, cmd); 264 265 switch (cmd) { 266 case DDI_ATTACH: 267 mutex_enter(&sol_ucma.ucma_mutex); 268 if (sol_ucma.ucma_dip != NULL) { 269 mutex_exit(&sol_ucma.ucma_mutex); 270 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 271 "attach: failed, > 1 instance"); 272 return (DDI_FAILURE); 273 } 274 sol_ucma.ucma_dip = dip; 275 mutex_exit(&sol_ucma.ucma_mutex); 276 277 rval = ddi_create_minor_node(dip, "sol_ucma", S_IFCHR, 278 0, DDI_PSEUDO, 0); 279 if (rval != DDI_SUCCESS) { 280 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 281 "attach: ddi_create_minor_node failed"); 282 mutex_enter(&sol_ucma.ucma_mutex); 283 sol_ucma.ucma_dip = NULL; 284 mutex_exit(&sol_ucma.ucma_mutex); 285 return (DDI_FAILURE); 286 } 287 288 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, 289 "attach : DDI_ATTACH success"); 290 return (DDI_SUCCESS); 291 case DDI_RESUME: 292 return (DDI_SUCCESS); 293 default: 294 return (DDI_FAILURE); 295 } 296 } 297 298 static int 299 sol_ucma_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 300 { 301 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "detach(%p, %x)", dip, cmd); 302 303 switch (cmd) { 304 case DDI_DETACH: 305 mutex_enter(&sol_ucma.ucma_mutex); 306 if (sol_ucma.ucma_num_file) { 307 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 308 "detach : %x files not closed", 309 sol_ucma.ucma_num_file); 310 mutex_exit(&sol_ucma.ucma_mutex); 311 return (DDI_FAILURE); 312 } 313 sol_ucma.ucma_dip = NULL; 314 mutex_exit(&sol_ucma.ucma_mutex); 315 316 ddi_remove_minor_node(dip, "sol_ucma"); 317 318 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, 319 "detach : DDI_DETACH success"); 320 return (DDI_SUCCESS); 321 case DDI_SUSPEND: 322 return (DDI_SUCCESS); 323 default: 324 return (DDI_FAILURE); 325 } 326 } 327 328 /*ARGSUSED*/ 329 static int 330 sol_ucma_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, 331 void **resultp) 332 { 333 switch (cmd) { 334 case DDI_INFO_DEVT2DEVINFO: 335 *resultp = (void *)sol_ucma.ucma_dip; 336 return (DDI_SUCCESS); 337 case DDI_INFO_DEVT2INSTANCE: 338 *resultp = (void *)0; 339 return (DDI_SUCCESS); 340 default : 341 return (DDI_FAILURE); 342 } 343 } 344 345 static int 346 sol_ucma_open(dev_t *devp, int flag, int otype, cred_t *credp) 347 { 348 sol_ucma_file_t *new_filep; 349 minor_t new_minor; 350 351 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "open(%p, %x, %x, %p)", 352 devp, flag, otype, credp); 353 354 new_filep = ucma_alloc_file(&new_minor); 355 if (new_filep == NULL) 356 return (EAGAIN); 357 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, "sol_ucma new minor %x", 358 new_minor); 359 360 /* 361 * For the first open, ensure that the sol_uverbs driver is attached. 362 * Also get the function pointers for uverbs API functions using 363 * ddi_modopen() and ddi_modsym() for the sol_uverbs driver. 364 * 365 * ldi_open() is done to ensure that sol_uverbs driver is attached, 366 * even though ddi_modopen is sufficient to get the function pointers 367 * for the uverbs APIs 368 */ 369 mutex_enter(&sol_ucma.ucma_mutex); 370 if (sol_ucma.ucma_clnt_hdl_flag == SOL_UCMA_CLNT_HDL_UNINITIALIZED) { 371 int rval, ret_errno; 372 373 sol_ucma.ucma_clnt_hdl_flag = 374 SOL_UCMA_CLNT_HDL_INITIALIZING; 375 if ((rval = ldi_open_by_name(SOL_UCMA_UVERBS_PATH, 376 FREAD | FWRITE, kcred, &sol_ucma.ucma_ldi_hdl, 377 sol_ucma.ucma_ldi_ident)) != 0) { 378 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 379 "ldi_open_by_name(%s, ...) failed with rval %x", 380 SOL_UCMA_UVERBS_PATH, rval); 381 sol_ofs_uobj_free(&new_filep->file_uobj); 382 sol_ucma.ucma_clnt_hdl_flag = 383 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 384 mutex_exit(&sol_ucma.ucma_mutex); 385 return (ENODEV); 386 } 387 if ((sol_ucma.ucma_mod_hdl = ddi_modopen("drv/sol_uverbs", 388 KRTLD_MODE_FIRST, &ret_errno)) == NULL) { 389 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 390 "ddi_modopen(%s, ...) failed", "drv/sol_uverbs"); 391 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 392 FREAD | FWRITE, kcred); 393 sol_ofs_uobj_free(&new_filep->file_uobj); 394 sol_ucma.ucma_clnt_hdl_flag = 395 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 396 mutex_exit(&sol_ucma.ucma_mutex); 397 return (ret_errno); 398 } 399 if ((uverbs_get_hdl_fp = (uverbs_get_clnt_hdl_t)ddi_modsym( 400 sol_ucma.ucma_mod_hdl, SOL_UVERBS_GET_CLNT_HDL, &ret_errno)) 401 == NULL) { 402 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 403 "ddi_modsym(%s, ...) failed", 404 SOL_UVERBS_GET_CLNT_HDL); 405 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 406 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 407 FREAD | FWRITE, kcred); 408 sol_ofs_uobj_free(&new_filep->file_uobj); 409 sol_ucma.ucma_clnt_hdl_flag = 410 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 411 mutex_exit(&sol_ucma.ucma_mutex); 412 return (ret_errno); 413 } 414 if ((uverbs_qpnum2qphdl_fp = (uverbs_qpnum2qphdl_t)ddi_modsym( 415 sol_ucma.ucma_mod_hdl, SOL_UVERBS_QPNUM2QPHDL, &ret_errno)) 416 == NULL) { 417 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 418 "ddi_modsym(%s, ...) failed", 419 SOL_UVERBS_QPNUM2QPHDL); 420 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 421 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 422 FREAD | FWRITE, kcred); 423 sol_ofs_uobj_free(&new_filep->file_uobj); 424 sol_ucma.ucma_clnt_hdl_flag = 425 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 426 mutex_exit(&sol_ucma.ucma_mutex); 427 return (ret_errno); 428 } 429 if ((uverbs_disable_uqpn_modify_fp = 430 (uverbs_disable_uqpn_mod_t)ddi_modsym( 431 sol_ucma.ucma_mod_hdl, SOL_UVERBS_DISABLE_UQPN_MODIFY, 432 &ret_errno)) == NULL) { 433 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 434 "ddi_modsym(%s, ...) failed", 435 SOL_UVERBS_DISABLE_UQPN_MODIFY); 436 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 437 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 438 FREAD | FWRITE, kcred); 439 sol_ofs_uobj_free(&new_filep->file_uobj); 440 sol_ucma.ucma_clnt_hdl_flag = 441 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 442 mutex_exit(&sol_ucma.ucma_mutex); 443 return (ret_errno); 444 } 445 if ((uverbs_uqpn_cq_ctrl_fp = 446 (uverbs_uqpn_cq_ctrl_t)ddi_modsym( 447 sol_ucma.ucma_mod_hdl, SOL_UVERBS_UQPN_CQ_CTRL, 448 &ret_errno)) == NULL) { 449 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 450 "ddi_modsym(%s, ...) failed", 451 SOL_UVERBS_UQPN_CQ_CTRL); 452 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 453 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 454 FREAD | FWRITE, kcred); 455 sol_ofs_uobj_free(&new_filep->file_uobj); 456 sol_ucma.ucma_clnt_hdl_flag = 457 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 458 mutex_exit(&sol_ucma.ucma_mutex); 459 return (ret_errno); 460 } 461 if ((uverbs_set_qp_free_state_fp = 462 (uverbs_set_qp_free_state_t)ddi_modsym( 463 sol_ucma.ucma_mod_hdl, SOL_UVERBS_SET_QPFREE_STATE, 464 &ret_errno)) == NULL) { 465 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 466 "ddi_modsym(%s, ...) failed", 467 SOL_UVERBS_SET_QPFREE_STATE); 468 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 469 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 470 FREAD | FWRITE, kcred); 471 sol_ofs_uobj_free(&new_filep->file_uobj); 472 sol_ucma.ucma_clnt_hdl_flag = 473 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 474 mutex_exit(&sol_ucma.ucma_mutex); 475 return (ret_errno); 476 } 477 if ((uverbs_flush_qp_fp = 478 (uverbs_flush_qp_t)ddi_modsym( 479 sol_ucma.ucma_mod_hdl, SOL_UVERBS_FLUSH_QP, 480 &ret_errno)) == NULL) { 481 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 482 "ddi_modsym(%s, ...) failed", 483 SOL_UVERBS_FLUSH_QP); 484 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 485 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 486 FREAD | FWRITE, kcred); 487 sol_ofs_uobj_free(&new_filep->file_uobj); 488 sol_ucma.ucma_clnt_hdl_flag = 489 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 490 mutex_exit(&sol_ucma.ucma_mutex); 491 return (ret_errno); 492 } 493 494 (*uverbs_get_hdl_fp) (&sol_ucma.ucma_ib_clnt_hdl, 495 &sol_ucma.ucma_iw_clnt_hdl); 496 if (sol_ucma.ucma_ib_clnt_hdl == NULL && 497 sol_ucma.ucma_iw_clnt_hdl == NULL) { 498 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 499 "uverbs_get_clnt_hdl failed"); 500 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 501 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 502 FREAD | FWRITE, kcred); 503 sol_ofs_uobj_free(&new_filep->file_uobj); 504 sol_ucma.ucma_clnt_hdl_flag = 505 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 506 mutex_exit(&sol_ucma.ucma_mutex); 507 return (ENODEV); 508 } 509 sol_ucma.ucma_clnt_hdl_flag = 510 SOL_UCMA_CLNT_HDL_INITIALIZED; 511 cv_broadcast(&sol_ucma.ucma_open_cv); 512 } else if (sol_ucma.ucma_clnt_hdl_flag == 513 SOL_UCMA_CLNT_HDL_INITIALIZING) { 514 cv_wait(&sol_ucma.ucma_open_cv, &sol_ucma.ucma_mutex); 515 } 516 mutex_exit(&sol_ucma.ucma_mutex); 517 *devp = makedevice(getmajor(*devp), new_minor); 518 519 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "open Success"); 520 return (0); 521 } 522 523 static int 524 sol_ucma_close(dev_t dev, int flag, int otype, cred_t *credp) 525 { 526 minor_t minor; 527 sol_ucma_file_t *filep; 528 genlist_entry_t *entry; 529 530 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "close(%x, %x, %x, %p)", 531 dev, flag, otype, credp); 532 533 minor = getminor(dev); 534 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read( 535 &ucma_file_uo_tbl, minor); 536 if (!filep) { 537 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, "close, no dev_t %x", 538 dev); 539 return (0); 540 } 541 542 /* Disable further event handling for this CM event channel */ 543 mutex_enter(&filep->file_mutex); 544 if (filep->file_evt_close_flag == SOL_UCMA_EVT_PROGRESS) { 545 cv_wait(&filep->file_evt_close_cv, &filep->file_mutex); 546 } 547 filep->file_evt_close_flag = SOL_UCMA_EVT_DISABLED; 548 mutex_exit(&filep->file_mutex); 549 550 /* 551 * Destroy CM IDs which have not been destroyed. 552 * For CMIDs which have been connected, call 553 * uverbs_set_qp_free_state(SOL_UVERBS2UCMA_ENABLE_QP_FREE) 554 * so that QP free will be done when appropriate, 555 */ 556 entry = remove_genlist_head(&filep->file_id_list); 557 while (entry) { 558 sol_ucma_chan_t *chanp; 559 void *qphdl; 560 561 chanp = (sol_ucma_chan_t *)entry->data; 562 mutex_enter(&chanp->chan_mutex); 563 if (chanp->chan_rdma_id) 564 (chanp->chan_rdma_id)->context = NULL; 565 mutex_exit(&chanp->chan_mutex); 566 rdma_destroy_id(chanp->chan_rdma_id); 567 568 mutex_enter(&chanp->chan_mutex); 569 qphdl = chanp->chan_qp_hdl; 570 chanp->chan_qp_hdl = NULL; 571 mutex_exit(&chanp->chan_mutex); 572 if (qphdl) 573 (*uverbs_set_qp_free_state_fp) ( 574 SOL_UVERBS2UCMA_ENABLE_QP_FREE, 0, qphdl); 575 ucma_free_chan(chanp, 1); 576 577 entry = remove_genlist_head(&filep->file_id_list); 578 } 579 580 /* Flush out any events that have not been acknowledged. */ 581 mutex_enter(&filep->file_mutex); 582 if (filep->file_pending_evt_cnt) { 583 sol_ucma_event_t *evtp; 584 585 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, 586 "close : %d Events not reported to userland", 587 filep->file_pending_evt_cnt); 588 entry = remove_genlist_head(&filep->file_evt_list); 589 while (entry) { 590 evtp = (sol_ucma_event_t *)entry->data; 591 kmem_free(evtp, sizeof (sol_ucma_event_t)); 592 kmem_free(entry, sizeof (genlist_entry_t)); 593 entry = remove_genlist_head(&filep->file_evt_list); 594 }; 595 mutex_exit(&filep->file_mutex); 596 } 597 598 /* 599 * Module close for sol_uverbs when the last file is closed. 600 * Set the function pointers to sol_uverbs API to NULL 601 * ddi_modclose() and ldi_close() - sol_uverbs driver 602 */ 603 mutex_enter(&sol_ucma.ucma_mutex); 604 if (sol_ucma.ucma_num_file == 1) { 605 sol_ucma.ucma_clnt_hdl_flag = 606 SOL_UCMA_CLNT_HDL_UNINITIALIZED; 607 uverbs_get_hdl_fp = NULL; 608 uverbs_qpnum2qphdl_fp = NULL; 609 uverbs_disable_uqpn_modify_fp = NULL; 610 uverbs_uqpn_cq_ctrl_fp = NULL; 611 uverbs_uqpn_cq_ctrl_fp = NULL; 612 uverbs_set_qp_free_state_fp = NULL; 613 uverbs_flush_qp_fp = NULL; 614 sol_ucma.ucma_ib_clnt_hdl = NULL; 615 sol_ucma.ucma_iw_clnt_hdl = NULL; 616 (void) ddi_modclose(sol_ucma.ucma_mod_hdl); 617 (void) ldi_close(sol_ucma.ucma_ldi_hdl, 618 FREAD | FWRITE, kcred); 619 } 620 sol_ucma.ucma_num_file--; 621 mutex_exit(&sol_ucma.ucma_mutex); 622 623 kmem_free(filep->file_pollhead, sizeof (struct pollhead)); 624 sol_ofs_uobj_put(&filep->file_uobj); 625 mutex_destroy(&filep->file_mutex); 626 cv_destroy(&filep->file_evt_cv); 627 cv_destroy(&filep->file_evt_close_cv); 628 rw_enter(&(filep->file_uobj.uo_lock), RW_WRITER); 629 (void) sol_ofs_uobj_remove(&ucma_file_uo_tbl, &(filep->file_uobj)); 630 rw_exit(&(filep->file_uobj.uo_lock)); 631 sol_ofs_uobj_free(&(filep->file_uobj)); 632 return (0); 633 } 634 635 typedef struct sol_ucma_cmd_table_s { 636 int (*sol_ucma_cmd_fnc) (dev_t, void *, struct uio *); 637 uint16_t sol_ucma_in_len; 638 uint16_t sol_ucma_out_len; 639 } sol_ucma_cmd_table_t; 640 641 static sol_ucma_cmd_table_t sol_ucma_cmd_table[] = { 642 [RDMA_USER_CM_CMD_CREATE_ID] = sol_ucma_create_id, 643 sizeof (sol_ucma_create_id_t), 644 sizeof (sol_ucma_create_id_resp_t), 645 [RDMA_USER_CM_CMD_DESTROY_ID] = sol_ucma_destroy_id, 646 sizeof (sol_ucma_destroy_id_t), 647 sizeof (sol_ucma_destroy_id_resp_t), 648 [RDMA_USER_CM_CMD_BIND_ADDR] = sol_ucma_bind_addr, 649 sizeof (sol_ucma_bind_addr_t), 650 0, 651 [RDMA_USER_CM_CMD_RESOLVE_ADDR] = sol_ucma_resolve_addr, 652 sizeof (sol_ucma_resolve_addr_t), 653 0, 654 [RDMA_USER_CM_CMD_RESOLVE_ROUTE] = sol_ucma_resolve_route, 655 sizeof (sol_ucma_resolve_route_t), 656 0, 657 [RDMA_USER_CM_CMD_QUERY_ROUTE] = sol_ucma_query_route, 658 sizeof (sol_ucma_query_route_t), 659 sizeof (sol_ucma_query_route_resp_t), 660 [RDMA_USER_CM_CMD_CONNECT] = sol_ucma_connect, 661 sizeof (sol_ucma_connect_t), 662 0, 663 [RDMA_USER_CM_CMD_LISTEN] = sol_ucma_listen, 664 sizeof (sol_ucma_listen_t), 665 0, 666 [RDMA_USER_CM_CMD_ACCEPT] = sol_ucma_accept, 667 sizeof (sol_ucma_accept_t), 668 0, 669 [RDMA_USER_CM_CMD_REJECT] = sol_ucma_reject, 670 sizeof (sol_ucma_reject_t), 671 0, 672 [RDMA_USER_CM_CMD_DISCONNECT] = sol_ucma_disconnect, 673 sizeof (sol_ucma_disconnect_t), 674 0, 675 [RDMA_USER_CM_CMD_INIT_QP_ATTR] = sol_ucma_init_qp_attr, 676 sizeof (sol_ucma_init_qp_attr_t), 677 sizeof (struct ib_uverbs_qp_attr), 678 [RDMA_USER_CM_CMD_GET_EVENT] = sol_ucma_get_event, 679 sizeof (sol_ucma_get_event_t), 680 sizeof (sol_ucma_event_resp_t), 681 [RDMA_USER_CM_CMD_GET_OPTION] = NULL, 682 0, 683 0, 684 [RDMA_USER_CM_CMD_SET_OPTION] = sol_ucma_set_option, 685 sizeof (sol_ucma_set_option_t), 686 0, 687 [RDMA_USER_CM_CMD_NOTIFY] = sol_ucma_notify, 688 sizeof (sol_ucma_notify_t), 689 0, 690 [RDMA_USER_CM_CMD_JOIN_MCAST] = sol_ucma_join_mcast, 691 sizeof (sol_ucma_join_mcast_t), 692 sizeof (sol_ucma_create_id_resp_t), 693 [RDMA_USER_CM_CMD_LEAVE_MCAST] = sol_ucma_leave_mcast, 694 sizeof (sol_ucma_destroy_id_t), 695 sizeof (sol_ucma_destroy_id_resp_t) 696 }; 697 698 #define SOL_UCMA_MAX_CMD_DATA 512 699 static int 700 sol_ucma_write(dev_t dev, struct uio *uio, cred_t *credp) 701 { 702 sol_ucma_cmd_hdr_t *user_hdrp; 703 int ret; 704 void *data_buf = NULL; 705 char uio_data[SOL_UCMA_MAX_CMD_DATA]; 706 size_t uio_data_len = uio->uio_resid; 707 708 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "write(%x, %p, %p)", 709 dev, uio, credp); 710 711 ret = uiomove((caddr_t)&uio_data, uio_data_len, UIO_WRITE, uio); 712 user_hdrp = (sol_ucma_cmd_hdr_t *)uio_data; 713 714 if (ret != 0) { 715 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "write: uiomove failed"); 716 return (ret); 717 } 718 719 if (user_hdrp->cmd >= 720 sizeof (sol_ucma_cmd_table) / sizeof (sol_ucma_cmd_table_t)) { 721 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 722 "open : cmd out of bound 0x%x", user_hdrp->cmd); 723 return (EINVAL); 724 } 725 if (!(sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_cmd_fnc)) { 726 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 727 "open : Unsupported cmd 0x%x", user_hdrp->cmd); 728 return (EINVAL); 729 } 730 731 /* 732 * Check the user passed IN-OUT buffer length, with expected lengths 733 */ 734 if (sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_in_len != 735 (user_hdrp->in)) { 736 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 737 "write : Invalid Input length cmd %x, in %x expected %x", 738 user_hdrp->cmd, user_hdrp->in, 739 sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_in_len); 740 return (EINVAL); 741 } 742 743 if (sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_out_len != 744 (user_hdrp->out)) { 745 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 746 "write : Invalid Output length cmd %x, in %x expected %x", 747 user_hdrp->cmd, user_hdrp->out, 748 sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_out_len); 749 return (EINVAL); 750 } 751 752 753 if (user_hdrp->in) { 754 data_buf = (void *)((char *)uio_data + 755 sizeof (sol_ucma_cmd_hdr_t)); 756 } 757 758 ret = (sol_ucma_cmd_table[user_hdrp->cmd].sol_ucma_cmd_fnc) 759 (dev, data_buf, uio); 760 761 /* If the command fails, set back the uio_resid */ 762 if (ret) 763 uio->uio_resid += uio_data_len; 764 765 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "write : ret %x", ret); 766 return (ret); 767 } 768 769 static int 770 sol_ucma_poll(dev_t dev, short events, int anyyet, short *reventsp, 771 struct pollhead **phpp) 772 { 773 minor_t minor = getminor(dev); 774 sol_ucma_file_t *filep; 775 776 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "poll(%x, %x)", 777 dev, events); 778 if (!(events & (POLLIN | POLLRDNORM))) 779 return (EINVAL); 780 781 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read( 782 &ucma_file_uo_tbl, minor); 783 ASSERT(filep); 784 785 if (filep->file_pending_evt_cnt) { 786 *reventsp = POLLIN | POLLRDNORM; 787 } else { 788 *reventsp = 0; 789 } 790 if ((*reventsp == 0 && !anyyet) || (events && POLLET)) { 791 *phpp = filep->file_pollhead; 792 } 793 sol_ofs_uobj_put(&filep->file_uobj); 794 795 return (0); 796 } 797 798 /* 799 * RDMACM functions. 800 */ 801 /*ARGSUSED*/ 802 static int 803 sol_ucma_create_id(dev_t dev, void *io_buf, struct uio *uio) 804 { 805 minor_t minor = getminor(dev); 806 sol_ucma_file_t *filep; 807 sol_ucma_chan_t *chanp; 808 sol_ucma_create_id_t *ucma_id_inp; 809 sol_ucma_create_id_resp_t ucma_id_resp; 810 811 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "create_id(%x, %p), minor %x", 812 dev, io_buf, minor); 813 814 ucma_id_inp = (sol_ucma_create_id_t *)io_buf; 815 ASSERT(ucma_id_inp); 816 ASSERT(ucma_id_inp->response.r_laddr); 817 818 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read(&ucma_file_uo_tbl, 819 minor); 820 ASSERT(filep); 821 822 chanp = ucma_alloc_chan(filep, ucma_id_inp); 823 if (chanp == NULL) { 824 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 825 "create_id: No free Channel"); 826 sol_ofs_uobj_put(&filep->file_uobj); 827 return (ENODEV); 828 } 829 ucma_id_resp.id = chanp->chan_id; 830 831 #ifdef _LP64 832 if (copyout(&ucma_id_resp, (void *)(ucma_id_inp->response.r_laddr), 833 sizeof (sol_ucma_create_id_resp_t))) { 834 #else 835 if (copyout(&ucma_id_resp, (void *)(ucma_id_inp->response.r_addr), 836 sizeof (sol_ucma_create_id_resp_t))) { 837 #endif 838 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 839 "create_id: copyout fault"); 840 ucma_free_chan(chanp, 1); 841 sol_ofs_uobj_put(&filep->file_uobj); 842 return (EFAULT); 843 } 844 /* */ 845 846 chanp->chan_rdma_id = rdma_create_id(sol_ucma_evt_hdlr, 847 chanp, ucma_id_inp->ps); 848 if (chanp->chan_rdma_id == NULL) { 849 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 850 "create_id: rdma_create_id failed"); 851 ucma_free_chan(chanp, 1); 852 sol_ofs_uobj_put(&filep->file_uobj); 853 return (EINVAL); 854 } 855 mutex_enter(&chanp->chan_mutex); 856 (chanp->chan_rdma_id)->context = chanp; 857 mutex_exit(&chanp->chan_mutex); 858 rdma_map_id2clnthdl(chanp->chan_rdma_id, sol_ucma.ucma_ib_clnt_hdl, 859 sol_ucma.ucma_iw_clnt_hdl); 860 861 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "create_id: Return SUCCESS"); 862 sol_ofs_uobj_put(&filep->file_uobj); 863 return (0); 864 } 865 866 /*ARGSUSED*/ 867 static int 868 sol_ucma_destroy_id(dev_t dev, void *io_buf, struct uio *uio) 869 { 870 sol_ucma_chan_t *chanp; 871 uint32_t ucma_id; 872 sol_ucma_file_t *filep; 873 sol_ucma_destroy_id_t *id_inp; 874 minor_t minor; 875 genlist_entry_t *entry; 876 sol_ucma_destroy_id_resp_t id_resp; 877 878 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "destroy_id(%x, %p)", 879 dev, io_buf); 880 881 id_inp = (sol_ucma_destroy_id_t *)io_buf; 882 ucma_id = id_inp->id; 883 if (!get_file_chan(ucma_id, &filep, &chanp, "destroy_id", 0)) { 884 minor = getminor(dev); 885 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read( 886 &ucma_file_uo_tbl, minor); 887 if (!filep) { 888 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 889 "destroy_id : filep NULL"); 890 return (EINVAL); 891 } 892 } else { 893 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "destroy_id : " 894 "ucma_id %x invalid", ucma_id); 895 return (0); 896 } 897 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "destroy_id: chanp %p", chanp); 898 899 /* 900 * Event handling, Flush out events pending 901 * return the number of events that were acked. Free events not acked. 902 */ 903 ASSERT(filep); 904 mutex_enter(&filep->file_mutex); 905 if (filep->file_pending_evt_cnt != 0) { 906 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, 907 "destroy_id: pending events"); 908 entry = remove_genlist_head(&filep->file_evt_list); 909 while (entry) { 910 kmem_free((void *) (entry->data), 911 sizeof (sol_ucma_event_t)); 912 kmem_free(entry, sizeof (genlist_entry_t)); 913 entry = remove_genlist_head(&filep->file_evt_list); 914 }; 915 filep->file_pending_evt_cnt = 0; 916 } 917 if (chanp) { 918 mutex_enter(&chanp->chan_mutex); 919 id_resp.events_reported = chanp->chan_evt_cnt; 920 mutex_exit(&chanp->chan_mutex); 921 } else { 922 id_resp.events_reported = 0; 923 } 924 mutex_exit(&filep->file_mutex); 925 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "destroy_id : chanp %p, " 926 "evts %x", chanp, id_resp.events_reported); 927 928 #ifdef _LP64 929 if (copyout(&id_resp, (void *) (id_inp->response.r_laddr), 930 sizeof (sol_ucma_destroy_id_resp_t))) { 931 #else 932 if (copyout(&id_resp, (void *) (id_inp->response.r_addr), 933 sizeof (sol_ucma_destroy_id_resp_t))) { 934 #endif 935 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 936 "destroy_id: copyout fault"); 937 sol_ofs_uobj_put(&filep->file_uobj); 938 return (EFAULT); 939 } 940 /* */ 941 942 if (chanp) { 943 mutex_enter(&chanp->chan_mutex); 944 if (chanp->chan_rdma_id) 945 (chanp->chan_rdma_id)->context = NULL; 946 mutex_exit(&chanp->chan_mutex); 947 rdma_destroy_id(chanp->chan_rdma_id); 948 ucma_free_chan(chanp, 1); 949 } 950 951 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "destroy_id: Success"); 952 sol_ofs_uobj_put(&filep->file_uobj); 953 return (0); 954 } 955 956 /*ARGSUSED*/ 957 static int 958 sol_ucma_bind_addr(dev_t dev, void *io_buf, struct uio *uio) 959 { 960 int ret; 961 sol_ucma_chan_t *chanp; 962 uint32_t ucma_id; 963 sol_ucma_bind_addr_t *bind_addrp; 964 965 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "bind_addr(%x, %p)", 966 dev, io_buf); 967 968 bind_addrp = (sol_ucma_bind_addr_t *)io_buf; 969 ucma_id = bind_addrp->id; 970 if (get_file_chan(ucma_id, NULL, &chanp, "bind_addr", 1)) 971 return (EINVAL); 972 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "bind_addr - chanp %p", chanp); 973 974 ret = rdma_bind_addr(chanp->chan_rdma_id, 975 (struct sockaddr *)&bind_addrp->addr); 976 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "bind_addr: ret %x", ret); 977 return (ret); 978 } 979 980 /*ARGSUSED*/ 981 static int 982 sol_ucma_resolve_addr(dev_t dev, void *io_buf, struct uio *uio) 983 { 984 sol_ucma_chan_t *chanp; 985 uint32_t ucma_id; 986 int ret; 987 sol_ucma_resolve_addr_t *resolve_addrp; 988 989 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_addr(%x, %p)", 990 dev, io_buf); 991 992 resolve_addrp = (sol_ucma_resolve_addr_t *)io_buf; 993 ucma_id = resolve_addrp->id; 994 if (get_file_chan(ucma_id, NULL, &chanp, "resolve_addr", 1)) { 995 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 996 "resolve_addr: ucma_id %x invalid", ucma_id); 997 return (EINVAL); 998 } 999 ASSERT(chanp); 1000 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_addr - chanp %p", chanp); 1001 1002 ret = rdma_resolve_addr(chanp->chan_rdma_id, 1003 (struct sockaddr *)&resolve_addrp->src_addr, 1004 (struct sockaddr *)&resolve_addrp->dst_addr, 1005 resolve_addrp->timeout_ms); 1006 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_addr: ret %x", ret); 1007 return (ret); 1008 } 1009 1010 /*ARGSUSED*/ 1011 static int 1012 sol_ucma_resolve_route(dev_t dev, void *io_buf, struct uio *uio) 1013 { 1014 sol_ucma_chan_t *chanp; 1015 uint32_t ucma_id; 1016 int ret; 1017 sol_ucma_resolve_route_t *resolve_routep; 1018 1019 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, 1020 "resolve_route(%x, %p)", dev, io_buf); 1021 1022 resolve_routep = (sol_ucma_resolve_route_t *)io_buf; 1023 ucma_id = resolve_routep->id; 1024 if (get_file_chan(ucma_id, NULL, &chanp, "resolve_route", 1)) 1025 return (EINVAL); 1026 ASSERT(chanp); 1027 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_route - chanp %p", 1028 chanp); 1029 1030 ret = rdma_resolve_route(chanp->chan_rdma_id, 1031 resolve_routep->timeout_ms); 1032 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "resolve_route: ret %x", ret); 1033 return (ret); 1034 } 1035 1036 /*ARGSUSED*/ 1037 static int 1038 sol_ucma_query_route(dev_t dev, void *io_buf, struct uio *uio) 1039 { 1040 sol_ucma_chan_t *chanp; 1041 uint32_t ucma_id; 1042 struct rdma_cm_id *idp; 1043 sol_ucma_query_route_t *query_routep; 1044 sol_ucma_query_route_resp_t route_info; 1045 1046 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "query_route(%x, %p)", 1047 dev, io_buf); 1048 1049 query_routep = (sol_ucma_query_route_t *)io_buf; 1050 ucma_id = query_routep->id; 1051 if (get_file_chan(ucma_id, NULL, &chanp, "query_route", 1)) 1052 return (EINVAL); 1053 ASSERT(chanp); 1054 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "query_route - chanp %p", chanp); 1055 idp = chanp->chan_rdma_id; 1056 1057 bzero(&route_info, sizeof (route_info)); 1058 rdma2usr_route(idp, &route_info); 1059 1060 #ifdef _LP64 1061 if (copyout(&route_info, (void *) (query_routep->response.r_laddr), 1062 sizeof (sol_ucma_query_route_resp_t))) { 1063 #else 1064 if (copyout(&route_info, (void *) (query_routep->response.r_addr), 1065 sizeof (sol_ucma_query_route_resp_t))) { 1066 #endif 1067 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1068 "query_route: copyout fault"); 1069 return (EFAULT); 1070 } 1071 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "query_route: Succcess"); 1072 return (0); 1073 } 1074 1075 /*ARGSUSED*/ 1076 static int 1077 sol_ucma_connect(dev_t dev, void *io_buf, struct uio *uio) 1078 { 1079 sol_ucma_chan_t *chanp; 1080 uint32_t ucma_id; 1081 int ret; 1082 void *qphdl; 1083 sol_ucma_connect_t *connectp; 1084 struct rdma_conn_param conn_param; 1085 struct rdma_cm_id *idp; 1086 1087 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "connect(%x, %p)", 1088 dev, io_buf); 1089 1090 connectp = (sol_ucma_connect_t *)io_buf; 1091 ucma_id = connectp->id; 1092 if (get_file_chan(ucma_id, NULL, &chanp, "connect", 1)) 1093 return (EINVAL); 1094 ASSERT(chanp); 1095 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "connect - chanp %p", chanp); 1096 1097 usr2rdma_conn_param(&(connectp->conn_param), &conn_param); 1098 ASSERT(uverbs_qpnum2qphdl_fp); 1099 ASSERT(uverbs_disable_uqpn_modify_fp); 1100 ASSERT(uverbs_uqpn_cq_ctrl_fp); 1101 qphdl = (*uverbs_qpnum2qphdl_fp) (conn_param.qp_num); 1102 if (qphdl == NULL) { 1103 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "connect: " 1104 "invalid QPNum %x", conn_param.qp_num); 1105 return (EINVAL); 1106 } 1107 (*uverbs_disable_uqpn_modify_fp) (conn_param.qp_num); 1108 rdma_map_id2qphdl(chanp->chan_rdma_id, qphdl); 1109 idp = chanp->chan_rdma_id; 1110 if (idp->ps == RDMA_PS_TCP) 1111 (void) (*uverbs_uqpn_cq_ctrl_fp) (conn_param.qp_num, 1112 SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE); 1113 chanp->chan_qp_num = conn_param.qp_num; 1114 ret = rdma_connect(chanp->chan_rdma_id, &conn_param); 1115 1116 /* 1117 * rdma_connect() initiated for this CMID, disable sol_uverbs to 1118 * free the QP assosiated with this CM ID. 1119 */ 1120 if (ret == 0 && idp->ps == RDMA_PS_TCP) { 1121 mutex_enter(&chanp->chan_mutex); 1122 chanp->chan_qp_hdl = qphdl; 1123 chanp->chan_flags |= SOL_UCMA_CHAN_CONNECT_FLAG; 1124 mutex_exit(&chanp->chan_mutex); 1125 (*uverbs_set_qp_free_state_fp) ( 1126 SOL_UVERBS2UCMA_DISABLE_QP_FREE, conn_param.qp_num, 1127 NULL); 1128 } 1129 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "connect: ret %x", ret); 1130 return (ret); 1131 } 1132 1133 /*ARGSUSED*/ 1134 static int 1135 sol_ucma_listen(dev_t dev, void *io_buf, struct uio *uio) 1136 { 1137 sol_ucma_chan_t *chanp; 1138 uint32_t ucma_id; 1139 int ret; 1140 sol_ucma_listen_t *listenp; 1141 1142 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "listen(%x, %p)", 1143 dev, io_buf); 1144 1145 listenp = (sol_ucma_listen_t *)io_buf; 1146 ucma_id = listenp->id; 1147 if (get_file_chan(ucma_id, NULL, &chanp, "listen", 1)) 1148 return (EINVAL); 1149 ASSERT(chanp); 1150 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "listen - chanp %p", chanp); 1151 1152 listenp->backlog = (listenp->backlog == 0 || 1153 listenp->backlog > SOL_UCMA_MAX_LISTEN) ? 1154 SOL_UCMA_MAX_LISTEN : listenp->backlog; 1155 chanp->chan_backlog = listenp->backlog; 1156 1157 ret = rdma_listen(chanp->chan_rdma_id, listenp->backlog); 1158 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "listen: ret %x", ret); 1159 return (ret); 1160 } 1161 1162 /*ARGSUSED*/ 1163 static int 1164 sol_ucma_accept(dev_t dev, void *io_buf, struct uio *uio) 1165 { 1166 int ret; 1167 uint32_t ucma_id; 1168 sol_ucma_chan_t *chanp; 1169 void *qphdl; 1170 sol_ucma_accept_t *acpt; 1171 struct rdma_conn_param conn_param; 1172 1173 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "accept(%x, %p)", 1174 dev, io_buf); 1175 1176 acpt = (sol_ucma_accept_t *)io_buf; 1177 ucma_id = acpt->id; 1178 if (get_file_chan(ucma_id, NULL, &chanp, "accept", 1)) 1179 return (EINVAL); 1180 ASSERT(chanp); 1181 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "accept - chanp %p", chanp); 1182 1183 if ((acpt->conn_param).valid) { 1184 struct rdma_cm_id *idp; 1185 1186 chanp->chan_user_id = acpt->uid; 1187 usr2rdma_conn_param(&acpt->conn_param, &conn_param); 1188 1189 ASSERT(uverbs_qpnum2qphdl_fp); 1190 qphdl = (*uverbs_qpnum2qphdl_fp) (conn_param.qp_num); 1191 if (qphdl == NULL) { 1192 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "accept: " 1193 "invalid QPNum %x", conn_param.qp_num); 1194 return (EINVAL); 1195 } 1196 (*uverbs_disable_uqpn_modify_fp) (conn_param.qp_num); 1197 rdma_map_id2qphdl(chanp->chan_rdma_id, qphdl); 1198 idp = chanp->chan_rdma_id; 1199 if (idp->ps == RDMA_PS_TCP) 1200 (void) (*uverbs_uqpn_cq_ctrl_fp) (conn_param.qp_num, 1201 SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE); 1202 chanp->chan_qp_num = conn_param.qp_num; 1203 ret = rdma_accept(chanp->chan_rdma_id, &conn_param); 1204 } else 1205 ret = rdma_accept(chanp->chan_rdma_id, NULL); 1206 1207 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "accept: ret %x", ret); 1208 return (ret); 1209 } 1210 1211 /*ARGSUSED*/ 1212 static int 1213 sol_ucma_reject(dev_t dev, void *io_buf, struct uio *uio) 1214 { 1215 int ret; 1216 uint32_t ucma_id; 1217 sol_ucma_chan_t *chanp; 1218 sol_ucma_reject_t *rjct; 1219 1220 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "reject(%x, %p)", dev, io_buf); 1221 1222 rjct = (sol_ucma_reject_t *)io_buf; 1223 ucma_id = rjct->id; 1224 if (get_file_chan(ucma_id, NULL, &chanp, "reject", 1)) 1225 return (EINVAL); 1226 ASSERT(chanp); 1227 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "reject - chanp %p", chanp); 1228 1229 ret = rdma_reject(chanp->chan_rdma_id, rjct->private_data, 1230 rjct->private_data_len); 1231 1232 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "reject: ret %x", ret); 1233 return (ret); 1234 } 1235 1236 /*ARGSUSED*/ 1237 static int 1238 sol_ucma_init_qp_attr(dev_t dev, void *io_buf, struct uio *uio) 1239 { 1240 int ret; 1241 uint32_t ucma_id; 1242 uint32_t qp_attr_mask; 1243 sol_ucma_chan_t *chanp; 1244 sol_ucma_init_qp_attr_t *qp_attr_inp; 1245 struct ib_uverbs_qp_attr uverbs_qp_attr; 1246 struct ib_qp_attr qp_attr; 1247 1248 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "init_qp_attr(%x, %p)", 1249 dev, io_buf); 1250 1251 qp_attr_inp = (sol_ucma_init_qp_attr_t *)io_buf; 1252 ucma_id = qp_attr_inp->id; 1253 if (get_file_chan(ucma_id, NULL, &chanp, "init_qp_attr", 1)) 1254 return (EINVAL); 1255 ASSERT(chanp); 1256 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "init_qp_attr - chanp %p", chanp); 1257 1258 qp_attr.qp_state = qp_attr_inp->qp_state; 1259 if ((ret = rdma_init_qp_attr(chanp->chan_rdma_id, &qp_attr, 1260 (int *)&qp_attr_mask)) != 0) { 1261 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "init_qp_attr: ret %x, " 1262 "mask %x", ret, qp_attr_mask); 1263 return (EINVAL); 1264 } 1265 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "init_qp_attr: ret %x, mask %x", 1266 ret, qp_attr_mask); 1267 1268 bzero(&uverbs_qp_attr, sizeof (uverbs_qp_attr)); 1269 uverbs_qp_attr.qp_attr_mask = qp_attr_mask; 1270 uverbs_qp_attr.qp_state = qp_attr.qp_state; 1271 uverbs_qp_attr.pkey_index = qp_attr.pkey_index; 1272 uverbs_qp_attr.port_num = qp_attr.port_num; 1273 uverbs_qp_attr.qp_access_flags = qp_attr.qp_access_flags; 1274 uverbs_qp_attr.qkey = qp_attr.qkey; 1275 uverbs_qp_attr.path_mtu = qp_attr.path_mtu; 1276 uverbs_qp_attr.dest_qp_num = qp_attr.dest_qp_num; 1277 uverbs_qp_attr.rq_psn = qp_attr.rq_psn; 1278 uverbs_qp_attr.max_dest_rd_atomic = qp_attr.max_dest_rd_atomic; 1279 uverbs_qp_attr.min_rnr_timer = qp_attr.min_rnr_timer; 1280 uverbs_qp_attr.ah_attr.dlid = qp_attr.ah_attr.dlid; 1281 if (qp_attr.ah_attr.ah_flags) { 1282 uverbs_qp_attr.ah_attr.is_global = 1; 1283 bcopy(&(qp_attr.ah_attr.grh.dgid), 1284 &(uverbs_qp_attr.ah_attr.grh.dgid), 16); 1285 uverbs_qp_attr.ah_attr.grh.flow_label = 1286 qp_attr.ah_attr.grh.flow_label; 1287 uverbs_qp_attr.ah_attr.grh.sgid_index = 1288 qp_attr.ah_attr.grh.sgid_index; 1289 uverbs_qp_attr.ah_attr.grh.hop_limit = 1290 qp_attr.ah_attr.grh.hop_limit; 1291 uverbs_qp_attr.ah_attr.grh.traffic_class = 1292 qp_attr.ah_attr.grh.traffic_class; 1293 } 1294 uverbs_qp_attr.ah_attr.sl = qp_attr.ah_attr.sl; 1295 uverbs_qp_attr.ah_attr.src_path_bits = qp_attr.ah_attr.src_path_bits; 1296 uverbs_qp_attr.ah_attr.static_rate = qp_attr.ah_attr.static_rate; 1297 uverbs_qp_attr.ah_attr.port_num = qp_attr.ah_attr.port_num; 1298 1299 #ifdef _LP64 1300 if (copyout(&uverbs_qp_attr, (void *) (qp_attr_inp->response.r_laddr), 1301 sizeof (uverbs_qp_attr))) { 1302 #else 1303 if (copyout(&uverbs_qp_attr, (void *) (qp_attr_inp->response.r_addr), 1304 sizeof (uverbs_qp_attr))) { 1305 #endif 1306 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "init_qp_attr : copyout " 1307 "failed"); 1308 return (EFAULT); 1309 } 1310 return (0); 1311 } 1312 1313 static int 1314 sol_ucma_get_event(dev_t dev, void *io_buf, struct uio *uio) 1315 { 1316 minor_t minor; 1317 sol_ucma_file_t *filep; 1318 sol_ucma_chan_t *evt_chanp; 1319 genlist_entry_t *entry; 1320 struct rdma_ucm_get_event *user_evt_inp; 1321 sol_ucma_event_t *queued_evt; 1322 struct rdma_ucm_event_resp *user_evt_resp; 1323 1324 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event(%x, %p)", dev, io_buf); 1325 user_evt_inp = (struct rdma_ucm_get_event *)io_buf; 1326 1327 minor = getminor(dev); 1328 filep = (sol_ucma_file_t *)sol_ofs_uobj_get_read(&ucma_file_uo_tbl, 1329 minor); 1330 ASSERT(filep); 1331 1332 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event fmode %x", 1333 uio->uio_fmode); 1334 1335 mutex_enter(&filep->file_mutex); 1336 while (filep->file_pending_evt_cnt == 0) { 1337 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, "get_event: No events"); 1338 if (uio->uio_fmode & (FNONBLOCK | FNDELAY)) { 1339 mutex_exit(&filep->file_mutex); 1340 sol_ofs_uobj_put(&filep->file_uobj); 1341 SOL_OFS_DPRINTF_L4(sol_ucma_dbg_str, 1342 "get_event: No events, nonblocking"); 1343 return (EAGAIN); 1344 } 1345 if (!cv_wait_sig(&filep->file_evt_cv, &filep->file_mutex)) { 1346 mutex_exit(&filep->file_mutex); 1347 sol_ofs_uobj_put(&filep->file_uobj); 1348 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, 1349 "get_event: Got Sig"); 1350 return (EINTR); 1351 } 1352 } 1353 1354 entry = remove_genlist_head(&filep->file_evt_list); 1355 mutex_exit(&filep->file_mutex); 1356 ASSERT(entry); 1357 queued_evt = (sol_ucma_event_t *)entry->data; 1358 ASSERT(queued_evt); 1359 user_evt_resp = &queued_evt->event_resp; 1360 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "event2usr " 1361 "uid %llx, id %x, event %x, status %x", user_evt_resp->uid, 1362 user_evt_resp->id, user_evt_resp->event, user_evt_resp->status); 1363 #ifdef _LP64 1364 if (copyout((void *)user_evt_resp, 1365 (void *)(user_evt_inp->response.r_laddr), 1366 sizeof (sol_ucma_event_resp_t))) { 1367 #else 1368 if (copyout((void *)user_evt_resp, 1369 (void *)(user_evt_inp->response.r_addr), 1370 sizeof (sol_ucma_event_resp_t))) { 1371 #endif 1372 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event: copyout " 1373 "failed"); 1374 sol_ofs_uobj_put(&filep->file_uobj); 1375 kmem_free(entry, sizeof (genlist_entry_t)); 1376 return (EFAULT); 1377 } 1378 mutex_enter(&filep->file_mutex); 1379 filep->file_pending_evt_cnt--; 1380 if (queued_evt->event_mcast) 1381 (queued_evt->event_mcast)->mcast_events++; 1382 evt_chanp = queued_evt->event_chan; 1383 if (evt_chanp) { 1384 /* 1385 * If the event is RDMA_CM_EVENT_CONNECT_RESPONSE or 1386 * RDMA_CM_EVENT_ESTABLISHED and the CM ID is for RC, 1387 * enable completion notifications for the QP. 1388 */ 1389 if (user_evt_resp->event == RDMA_CM_EVENT_CONNECT_RESPONSE || 1390 user_evt_resp->event == RDMA_CM_EVENT_ESTABLISHED) { 1391 struct rdma_cm_id *idp; 1392 int rc; 1393 1394 idp = evt_chanp->chan_rdma_id; 1395 if (idp->ps == RDMA_PS_TCP) { 1396 ASSERT(uverbs_uqpn_cq_ctrl_fp); 1397 rc = (*uverbs_uqpn_cq_ctrl_fp)( 1398 evt_chanp->chan_qp_num, 1399 SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE); 1400 if (rc) { 1401 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1402 "uverbs_uqpn_cq_ctrl_fp(%X) " 1403 "failed!!", 1404 evt_chanp->chan_qp_num); 1405 mutex_exit(&filep->file_mutex); 1406 filep->file_pending_evt_cnt++; 1407 return (EIO); 1408 } 1409 } 1410 } 1411 1412 /* Bump up backlog for CONNECT_REQUEST events */ 1413 mutex_enter(&evt_chanp->chan_mutex); 1414 if (user_evt_resp->event == RDMA_CM_EVENT_CONNECT_REQUEST) 1415 evt_chanp->chan_backlog++; 1416 1417 evt_chanp->chan_evt_cnt++; 1418 mutex_exit(&evt_chanp->chan_mutex); 1419 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event : " 1420 "chan %p, cnt %x", evt_chanp, evt_chanp->chan_evt_cnt); 1421 } 1422 mutex_exit(&filep->file_mutex); 1423 kmem_free(entry, sizeof (genlist_entry_t)); 1424 kmem_free(queued_evt, sizeof (sol_ucma_event_t)); 1425 1426 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "get_event: Success"); 1427 sol_ofs_uobj_put(&filep->file_uobj); 1428 return (0); 1429 } 1430 1431 /* 1432 * This is used when ULP wants to set the QOS option. This is *not* 1433 * supported by Solaris IB stack, return failure. 1434 */ 1435 /*ARGSUSED*/ 1436 static int 1437 sol_ucma_set_option(dev_t dev, void *io_buf, struct uio *uio) 1438 { 1439 return (EINVAL); 1440 } 1441 1442 /* 1443 * This is used when ULP uses librdmacm but uses out of band connection for CM. 1444 */ 1445 /*ARGSUSED*/ 1446 static int 1447 sol_ucma_notify(dev_t dev, void *io_buf, struct uio *uio) 1448 { 1449 sol_ucma_notify_t *notifyp; 1450 uint32_t ucma_id; 1451 sol_ucma_chan_t *chan; 1452 int ret; 1453 1454 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "notify(%x, %p)", dev, io_buf); 1455 notifyp = (sol_ucma_notify_t *)io_buf; 1456 ucma_id = notifyp->id; 1457 if (get_file_chan(ucma_id, NULL, &chan, "notify", 1)) 1458 return (EINVAL); 1459 ASSERT(chan); 1460 1461 ret = rdma_notify(chan->chan_rdma_id, notifyp->event); 1462 if (ret) 1463 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "notify failed %x", ret); 1464 else 1465 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "notify Success"); 1466 return (ret); 1467 } 1468 1469 /*ARGSUSED*/ 1470 static int 1471 sol_ucma_join_mcast(dev_t dev, void *io_buf, struct uio *uio) 1472 { 1473 sol_ucma_join_mcast_t *join_buf; 1474 sol_ucma_create_id_resp_t join_resp; 1475 sol_ucma_chan_t *chanp; 1476 sol_ucma_mcast_t *mcastp; 1477 int rc; 1478 uint32_t ucma_id; 1479 1480 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "join_mcast(%x, %p)", 1481 dev, io_buf); 1482 join_buf = (sol_ucma_join_mcast_t *)io_buf; 1483 ucma_id = join_buf->id; 1484 if (get_file_chan(ucma_id, NULL, &chanp, "join_mcast", 1)) 1485 return (EINVAL); 1486 1487 mcastp = kmem_zalloc(sizeof (sol_ucma_mcast_t), KM_SLEEP); 1488 bcopy((void *)(&(join_buf->addr)), (void *)(&(mcastp->mcast_addr)), 1489 sizeof (struct sockaddr)); 1490 mcastp->mcast_chan = chanp; 1491 sol_ofs_uobj_init(&mcastp->mcast_uobj, 0, SOL_UCMA_MCAST_TYPE); 1492 if (sol_ofs_uobj_add(&ucma_mcast_uo_tbl, &mcastp->mcast_uobj) != 0) { 1493 sol_ofs_uobj_free(&mcastp->mcast_uobj); 1494 return (ENOMEM); 1495 } 1496 mcastp->mcast_uobj.uo_live = 1; 1497 mcastp->mcast_id = join_resp.id = mcastp->mcast_uobj.uo_id; 1498 mcastp->mcast_uid = join_buf->uid; 1499 1500 rc = rdma_join_multicast(chanp->chan_rdma_id, 1501 (struct sockaddr *)(&(join_buf->addr)), mcastp); 1502 if (rc) { 1503 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1504 "join_mcast: rdma_join_multicast ret %x", rc); 1505 rw_enter(&(mcastp->mcast_uobj.uo_lock), RW_WRITER); 1506 (void) sol_ofs_uobj_remove(&ucma_mcast_uo_tbl, 1507 &mcastp->mcast_uobj); 1508 rw_exit(&(mcastp->mcast_uobj.uo_lock)); 1509 sol_ofs_uobj_free(&mcastp->mcast_uobj); 1510 return (rc); 1511 } 1512 1513 #ifdef _LP64 1514 if (copyout(&join_resp, (void *) (join_buf->response.r_laddr), 1515 sizeof (sol_ucma_create_id_resp_t))) { 1516 #else 1517 if (copyout(&join_resp, (void *) (join_buf->response.r_addr), 1518 sizeof (sol_ucma_create_id_resp_t))) { 1519 #endif 1520 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "join_mcast: copyout " 1521 "failed"); 1522 rdma_leave_multicast(chanp->chan_rdma_id, 1523 (struct sockaddr *)(&(join_buf->addr))); 1524 rw_enter(&(mcastp->mcast_uobj.uo_lock), RW_WRITER); 1525 (void) sol_ofs_uobj_remove(&ucma_mcast_uo_tbl, 1526 &mcastp->mcast_uobj); 1527 rw_exit(&(mcastp->mcast_uobj.uo_lock)); 1528 sol_ofs_uobj_free(&mcastp->mcast_uobj); 1529 return (EFAULT); 1530 } 1531 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "join_mcast: Return Success"); 1532 return (0); 1533 } 1534 1535 /*ARGSUSED*/ 1536 static int 1537 sol_ucma_leave_mcast(dev_t dev, void *io_buf, struct uio *uio) 1538 { 1539 sol_ucma_destroy_id_t *id_inp; 1540 sol_ucma_destroy_id_resp_t id_resp; 1541 sol_ucma_mcast_t *mcastp; 1542 sol_ucma_chan_t *chanp; 1543 uint32_t ucma_id; 1544 1545 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "leave_mcast(%x, %p)", 1546 dev, io_buf); 1547 id_inp = (sol_ucma_destroy_id_t *)io_buf; 1548 ucma_id = id_inp->id; 1549 mcastp = (sol_ucma_mcast_t *)sol_ofs_uobj_get_read(&ucma_mcast_uo_tbl, 1550 ucma_id); 1551 if (mcastp == NULL) { 1552 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "leave_mcast: invalid " 1553 "ID %x", ucma_id); 1554 return (EINVAL); 1555 } 1556 chanp = mcastp->mcast_chan; 1557 1558 rdma_leave_multicast(chanp->chan_rdma_id, &mcastp->mcast_addr); 1559 id_resp.events_reported = mcastp->mcast_events; 1560 1561 #ifdef _LP64 1562 if (copyout(&id_resp, (void *) (id_inp->response.r_laddr), 1563 sizeof (sol_ucma_destroy_id_resp_t))) { 1564 #else 1565 if (copyout(&id_resp, (void *) (id_inp->response.r_addr), 1566 sizeof (sol_ucma_destroy_id_resp_t))) { 1567 #endif 1568 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "leave_mcast: copyout " 1569 "fault"); 1570 sol_ofs_uobj_put(&mcastp->mcast_uobj); 1571 return (EFAULT); 1572 } 1573 sol_ofs_uobj_put(&mcastp->mcast_uobj); 1574 rw_enter(&(mcastp->mcast_uobj.uo_lock), RW_WRITER); 1575 (void) sol_ofs_uobj_remove(&ucma_mcast_uo_tbl, &mcastp->mcast_uobj); 1576 rw_exit(&(mcastp->mcast_uobj.uo_lock)); 1577 sol_ofs_uobj_free(&mcastp->mcast_uobj); 1578 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "leave_mcast: ret 0"); 1579 return (0); 1580 } 1581 1582 /*ARGSUSED*/ 1583 static int 1584 sol_ucma_disconnect(dev_t dev, void *io_buf, struct uio *uio) 1585 { 1586 sol_ucma_disconnect_t *disconnectp; 1587 uint32_t ucma_id; 1588 sol_ucma_chan_t *chan; 1589 int ret; 1590 1591 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "disconnect(%x, %p)", 1592 dev, io_buf); 1593 disconnectp = (sol_ucma_disconnect_t *)io_buf; 1594 ucma_id = disconnectp->id; 1595 if (get_file_chan(ucma_id, NULL, &chan, "disconnect", 1)) 1596 return (EINVAL); 1597 ASSERT(chan); 1598 1599 /* 1600 * For a TCP CMID, which has got the DISCONNECT event, call 1601 * ibt_flush_qp(), to transition QP to error state. 1602 */ 1603 mutex_enter(&chan->chan_mutex); 1604 if (chan->chan_flush_qp_flag == SOL_UCMA_FLUSH_QP_PENDING) { 1605 chan->chan_flush_qp_flag = SOL_UCMA_FLUSH_QP_DONE; 1606 mutex_exit(&chan->chan_mutex); 1607 (*uverbs_flush_qp_fp)(chan->chan_qp_num); 1608 } else 1609 mutex_exit(&chan->chan_mutex); 1610 1611 ret = rdma_disconnect(chan->chan_rdma_id); 1612 mutex_enter(&chan->chan_mutex); 1613 chan->chan_flush_qp_flag = SOL_UCMA_FLUSH_QP_DONE; 1614 mutex_exit(&chan->chan_mutex); 1615 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "disconnect: ret %x", ret); 1616 return (ret); 1617 } 1618 1619 /* 1620 * RDMA ID Event handler 1621 */ 1622 int 1623 sol_ucma_evt_hdlr(struct rdma_cm_id *idp, struct rdma_cm_event *eventp) 1624 { 1625 sol_ucma_chan_t *chan, *req_chan; 1626 sol_ucma_file_t *file; 1627 sol_ucma_event_t *ucma_evt; 1628 sol_ucma_create_id_t ucma_create_id; 1629 1630 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "ucma_evt_hdlr(%p, %p), " 1631 "event %x, status %x", idp, eventp, eventp->event, 1632 eventp->status); 1633 chan = (sol_ucma_chan_t *)idp->context; 1634 if (!chan) { 1635 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, "ucma_evt_hdlr() - " 1636 "after destroy - %p", idp); 1637 return (0); 1638 } 1639 mutex_enter(&chan->chan_mutex); 1640 file = chan->chan_file; 1641 if (!file) { 1642 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, "ucma_evt_hdlr() - " 1643 "after file destroy - idp %p", idp); 1644 mutex_exit(&chan->chan_mutex); 1645 return (0); 1646 } 1647 mutex_exit(&chan->chan_mutex); 1648 1649 mutex_enter(&file->file_mutex); 1650 if (file->file_evt_close_flag == SOL_UCMA_EVT_DISABLED) { 1651 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, "ucma_evt_hdlr() - " 1652 "after file close - idp %p", idp); 1653 mutex_exit(&file->file_mutex); 1654 return (0); 1655 } 1656 file->file_evt_close_flag = SOL_UCMA_EVT_PROGRESS; 1657 mutex_exit(&file->file_mutex); 1658 1659 /* 1660 * If the event is RDMA_CM_EVENT_CONNECT_REQUEST, allocate a 1661 * new chan. The rdma_cm_id for this chan has already been 1662 * allocated by sol_ofs. 1663 */ 1664 ucma_evt = kmem_zalloc(sizeof (sol_ucma_event_t), KM_SLEEP); 1665 ucma_evt->event_chan = chan; 1666 if (eventp->event == RDMA_CM_EVENT_CONNECT_REQUEST) { 1667 mutex_enter(&chan->chan_mutex); 1668 if (!chan->chan_backlog) { 1669 SOL_OFS_DPRINTF_L3(sol_ucma_dbg_str, 1670 "backlog exceeded"); 1671 mutex_exit(&chan->chan_mutex); 1672 mutex_enter(&file->file_mutex); 1673 file->file_evt_close_flag = SOL_UCMA_EVT_NONE; 1674 cv_broadcast(&file->file_evt_close_cv); 1675 mutex_exit(&file->file_mutex); 1676 kmem_free(ucma_evt, sizeof (sol_ucma_event_t)); 1677 return (-1); 1678 } 1679 chan->chan_backlog--; 1680 mutex_exit(&chan->chan_mutex); 1681 ucma_create_id.uid = chan->chan_user_id; 1682 req_chan = ucma_alloc_chan(file, &ucma_create_id); 1683 if (req_chan == NULL) { 1684 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1685 "evt hdlr: No free Channel"); 1686 sol_ofs_uobj_put(&file->file_uobj); 1687 mutex_enter(&file->file_mutex); 1688 file->file_evt_close_flag = SOL_UCMA_EVT_NONE; 1689 cv_broadcast(&file->file_evt_close_cv); 1690 mutex_exit(&file->file_mutex); 1691 return (-1); 1692 } 1693 req_chan->chan_rdma_id = idp; 1694 mutex_enter(&req_chan->chan_mutex); 1695 idp->context = req_chan; 1696 mutex_exit(&req_chan->chan_mutex); 1697 chan = req_chan; 1698 } else if (eventp->event == RDMA_CM_EVENT_DISCONNECTED || 1699 eventp->event == RDMA_CM_EVENT_REJECTED) { 1700 void *qphdl; 1701 1702 /* 1703 * Connection has been rejected or disconnected, 1704 * Enable uverbs to free QP, if it had been disabled 1705 * before. sol_uverbs will free the QP appropriately. 1706 */ 1707 mutex_enter(&chan->chan_mutex); 1708 qphdl = chan->chan_qp_hdl; 1709 chan->chan_qp_hdl = NULL; 1710 if (idp->ps == RDMA_PS_TCP && 1711 chan->chan_flush_qp_flag != SOL_UCMA_FLUSH_QP_DONE && 1712 eventp->event == RDMA_CM_EVENT_DISCONNECTED) { 1713 chan->chan_flush_qp_flag = 1714 SOL_UCMA_FLUSH_QP_PENDING; 1715 } 1716 mutex_exit(&chan->chan_mutex); 1717 1718 if (idp->ps == RDMA_PS_TCP && qphdl) 1719 (*uverbs_set_qp_free_state_fp) ( 1720 SOL_UVERBS2UCMA_ENABLE_QP_FREE, 0, qphdl); 1721 } else if (eventp->event == RDMA_CM_EVENT_ESTABLISHED && 1722 chan->chan_flags & SOL_UCMA_CHAN_CONNECT_FLAG) 1723 eventp->event = RDMA_CM_EVENT_CONNECT_RESPONSE; 1724 1725 ucma_evt->event_resp.event = eventp->event; 1726 ucma_evt->event_resp.status = eventp->status; 1727 if (idp->ps == RDMA_PS_UDP || idp->ps == RDMA_PS_IPOIB) 1728 rdma2usr_ud_param(&(eventp->param.ud), 1729 &(ucma_evt->event_resp.param.ud)); 1730 else 1731 rdma2usr_conn_param(&(eventp->param.conn), 1732 &(ucma_evt->event_resp.param.conn)); 1733 1734 if (eventp->event == RDMA_CM_EVENT_MULTICAST_JOIN || eventp->event == 1735 RDMA_CM_EVENT_MULTICAST_ERROR) { 1736 ucma_evt->event_mcast = (sol_ucma_mcast_t *) 1737 eventp->param.ud.private_data; 1738 ucma_evt->event_resp.uid = (ucma_evt->event_mcast)->mcast_uid; 1739 ucma_evt->event_resp.id = (ucma_evt->event_mcast)->mcast_id; 1740 } else { 1741 ucma_evt->event_resp.uid = chan->chan_user_id; 1742 ucma_evt->event_resp.id = chan->chan_id; 1743 } 1744 1745 mutex_enter(&file->file_mutex); 1746 (void) add_genlist(&file->file_evt_list, (uintptr_t)ucma_evt, NULL); 1747 file->file_pending_evt_cnt++; 1748 mutex_exit(&file->file_mutex); 1749 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "ucma_evt_hdlr-pollwakeup"); 1750 pollwakeup(file->file_pollhead, POLLIN | POLLRDNORM); 1751 mutex_enter(&file->file_mutex); 1752 cv_broadcast(&file->file_evt_cv); 1753 mutex_exit(&file->file_mutex); 1754 1755 mutex_enter(&file->file_mutex); 1756 file->file_evt_close_flag = SOL_UCMA_EVT_NONE; 1757 cv_broadcast(&file->file_evt_close_cv); 1758 mutex_exit(&file->file_mutex); 1759 return (0); 1760 } 1761 1762 /* 1763 * Local Functions 1764 */ 1765 static sol_ucma_file_t * 1766 ucma_alloc_file(minor_t *new_minorp) 1767 { 1768 sol_ucma_file_t *new_file; 1769 1770 new_file = kmem_zalloc(sizeof (sol_ucma_file_t), KM_SLEEP); 1771 sol_ofs_uobj_init(&new_file->file_uobj, 0, SOL_UCMA_EVT_FILE_TYPE); 1772 if (sol_ofs_uobj_add(&ucma_file_uo_tbl, &new_file->file_uobj) != 0) { 1773 sol_ofs_uobj_free(&new_file->file_uobj); 1774 return (NULL); 1775 } 1776 new_file->file_uobj.uo_live = 1; 1777 init_genlist(&new_file->file_id_list); 1778 init_genlist(&new_file->file_evt_list); 1779 1780 mutex_enter(&sol_ucma.ucma_mutex); 1781 sol_ucma.ucma_num_file++; 1782 mutex_exit(&sol_ucma.ucma_mutex); 1783 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "new file num %x, %p", 1784 (new_file->file_uobj).uo_id, new_file); 1785 1786 mutex_init(&new_file->file_mutex, NULL, 1787 MUTEX_DRIVER, NULL); 1788 cv_init(&new_file->file_evt_cv, NULL, CV_DRIVER, 1789 NULL); 1790 cv_init(&new_file->file_evt_close_cv, NULL, CV_DRIVER, 1791 NULL); 1792 new_file->file_pollhead = kmem_zalloc(sizeof (struct pollhead), 1793 KM_SLEEP); 1794 1795 *new_minorp = (minor_t)((new_file->file_uobj).uo_id); 1796 return (new_file); 1797 } 1798 1799 static sol_ucma_chan_t * 1800 ucma_alloc_chan(sol_ucma_file_t *filep, sol_ucma_create_id_t *create_id_inp) 1801 { 1802 sol_ucma_chan_t *new_chanp; 1803 1804 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_alloc_chan(%p, %p)", 1805 filep, create_id_inp); 1806 1807 new_chanp = kmem_zalloc(sizeof (sol_ucma_chan_t), KM_SLEEP); 1808 sol_ofs_uobj_init(&new_chanp->chan_uobj, 0, SOL_UCMA_CM_ID_TYPE); 1809 if (sol_ofs_uobj_add(&ucma_ctx_uo_tbl, &new_chanp->chan_uobj) != 0) { 1810 sol_ofs_uobj_free(&new_chanp->chan_uobj); 1811 return (NULL); 1812 } 1813 mutex_init(&new_chanp->chan_mutex, NULL, MUTEX_DRIVER, NULL); 1814 1815 new_chanp->chan_uobj.uo_live = 1; 1816 mutex_enter(&filep->file_mutex); 1817 new_chanp->chan_list_ent = add_genlist(&filep->file_id_list, 1818 (uintptr_t)new_chanp, NULL); 1819 mutex_exit(&filep->file_mutex); 1820 1821 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, "_alloc_chan - filep %p, " 1822 "chan_num %x, new_chan %p", filep, (new_chanp->chan_uobj).uo_id, 1823 new_chanp); 1824 1825 new_chanp->chan_file = filep; 1826 new_chanp->chan_user_id = create_id_inp->uid; 1827 new_chanp->chan_id = (new_chanp->chan_uobj).uo_id; 1828 1829 return (new_chanp); 1830 } 1831 1832 static void 1833 ucma_free_chan(sol_ucma_chan_t *chanp, int delete_list) 1834 { 1835 sol_ucma_file_t *filep; 1836 1837 ASSERT(chanp); 1838 if (delete_list) { 1839 filep = chanp->chan_file; 1840 ASSERT(filep); 1841 mutex_enter(&filep->file_mutex); 1842 delete_genlist(&filep->file_id_list, chanp->chan_list_ent); 1843 mutex_exit(&filep->file_mutex); 1844 } 1845 1846 mutex_destroy(&chanp->chan_mutex); 1847 rw_enter(&(chanp->chan_uobj.uo_lock), RW_WRITER); 1848 (void) sol_ofs_uobj_remove(&ucma_ctx_uo_tbl, &(chanp->chan_uobj)); 1849 rw_exit(&(chanp->chan_uobj.uo_lock)); 1850 sol_ofs_uobj_free(&(chanp->chan_uobj)); 1851 } 1852 1853 static int 1854 get_file_chan(uint32_t ucma_id, sol_ucma_file_t **filep, 1855 sol_ucma_chan_t **chanp, char *caller, int flag_err) 1856 { 1857 sol_ucma_chan_t *chan; 1858 1859 if (filep) 1860 *filep = NULL; 1861 if (chanp) 1862 *chanp = NULL; 1863 1864 chan = (sol_ucma_chan_t *)sol_ofs_uobj_get_read(&ucma_ctx_uo_tbl, 1865 ucma_id); 1866 if (chan == NULL) { 1867 if (flag_err) 1868 SOL_OFS_DPRINTF_L2(sol_ucma_dbg_str, 1869 "%s, ucma_id %x invalid", caller, ucma_id); 1870 else 1871 SOL_OFS_DPRINTF_L5(sol_ucma_dbg_str, 1872 "%s, ucma_id %x invalid", caller, ucma_id); 1873 return (-1); 1874 } 1875 1876 if (filep) 1877 *filep = chan->chan_file; 1878 if (chanp) 1879 *chanp = chan; 1880 1881 sol_ofs_uobj_put(&chan->chan_uobj); 1882 return (0); 1883 } 1884 1885 static void 1886 rdma2usr_pathrec(struct ib_sa_path_rec *kern_path, 1887 struct ib_user_path_rec *usr_path) 1888 { 1889 bcopy(&kern_path->dgid, &usr_path->dgid, 16); 1890 bcopy(&kern_path->sgid, &usr_path->sgid, 16); 1891 usr_path->dlid = kern_path->dlid; 1892 usr_path->slid = kern_path->slid; 1893 usr_path->raw_traffic = kern_path->raw_traffic; 1894 usr_path->flow_label = kern_path->flow_label; 1895 usr_path->reversible = kern_path->reversible; 1896 usr_path->mtu = kern_path->mtu; 1897 usr_path->pkey = kern_path->pkey; 1898 usr_path->hop_limit = kern_path->hop_limit; 1899 usr_path->traffic_class = kern_path->traffic_class; 1900 usr_path->sl = kern_path->sl; 1901 usr_path->mtu_selector = kern_path->mtu_selector; 1902 usr_path->rate_selector = kern_path->rate_selector; 1903 usr_path->rate = kern_path->rate; 1904 usr_path->packet_life_time_selector = 1905 kern_path->packet_life_time_selector; 1906 usr_path->packet_life_time = kern_path->packet_life_time; 1907 usr_path->preference = kern_path->preference; 1908 usr_path->numb_path = kern_path->numb_path; 1909 } 1910 1911 static void 1912 rdma2usr_route(struct rdma_cm_id *idp, sol_ucma_query_route_resp_t *resp) 1913 { 1914 struct rdma_route *routep; 1915 int i; 1916 1917 routep = &(idp->route); 1918 if (idp->device) { 1919 resp->node_guid = idp->device->node_guid; 1920 resp->port_num = idp->port_num; 1921 } 1922 bcopy(&(routep->addr.src_addr), &resp->src_addr, 1923 sizeof (struct sockaddr_in6)); 1924 bcopy(&(routep->addr.dst_addr), &resp->dst_addr, 1925 sizeof (struct sockaddr_in6)); 1926 resp->num_paths = routep->num_paths; 1927 for (i = 0; i < resp->num_paths; i++) { 1928 rdma2usr_pathrec(&(routep->path_rec[i]), 1929 &(resp->ib_route[i])); 1930 } 1931 } 1932 1933 static void 1934 usr2rdma_conn_param(struct rdma_ucm_conn_param *usr_conn_paramp, 1935 struct rdma_conn_param *conn_paramp) 1936 { 1937 conn_paramp->private_data = usr_conn_paramp->private_data; 1938 conn_paramp->private_data_len = usr_conn_paramp->private_data_len; 1939 conn_paramp->responder_resources = usr_conn_paramp->responder_resources; 1940 conn_paramp->initiator_depth = usr_conn_paramp->initiator_depth; 1941 conn_paramp->flow_control = usr_conn_paramp->flow_control; 1942 conn_paramp->retry_count = usr_conn_paramp->retry_count; 1943 conn_paramp->rnr_retry_count = usr_conn_paramp->rnr_retry_count; 1944 conn_paramp->srq = usr_conn_paramp->srq; 1945 conn_paramp->qp_num = usr_conn_paramp->qp_num; 1946 } 1947 1948 static void 1949 rdma2usr_conn_param(struct rdma_conn_param *conn_paramp, 1950 struct rdma_ucm_conn_param *usr_conn_paramp) 1951 { 1952 usr_conn_paramp->private_data_len = conn_paramp->private_data_len; 1953 1954 bzero(usr_conn_paramp->private_data, RDMA_MAX_PRIVATE_DATA); 1955 if (conn_paramp->private_data) 1956 bcopy(conn_paramp->private_data, 1957 usr_conn_paramp->private_data, 1958 usr_conn_paramp->private_data_len); 1959 usr_conn_paramp->responder_resources = conn_paramp->responder_resources; 1960 usr_conn_paramp->initiator_depth = conn_paramp->initiator_depth; 1961 usr_conn_paramp->flow_control = conn_paramp->flow_control; 1962 usr_conn_paramp->retry_count = conn_paramp->retry_count; 1963 usr_conn_paramp->rnr_retry_count = conn_paramp->rnr_retry_count; 1964 usr_conn_paramp->srq = conn_paramp->srq; 1965 usr_conn_paramp->qp_num = conn_paramp->qp_num; 1966 } 1967 1968 static void 1969 rdma2usr_ud_param(struct rdma_ud_param *ud_paramp, 1970 sol_ucma_ud_param_t *usr_ud_paramp) 1971 { 1972 struct ib_ah_attr *ah_attrp; 1973 struct ib_uverbs_ah_attr *usr_ah_attrp; 1974 1975 usr_ud_paramp->private_data_len = ud_paramp->private_data_len; 1976 1977 bzero(usr_ud_paramp->private_data, RDMA_MAX_PRIVATE_DATA); 1978 if (ud_paramp->private_data) 1979 bcopy(ud_paramp->private_data, 1980 usr_ud_paramp->private_data, 1981 usr_ud_paramp->private_data_len); 1982 usr_ud_paramp->qp_num = ud_paramp->qp_num; 1983 usr_ud_paramp->qkey = ud_paramp->qkey; 1984 1985 ah_attrp = &(ud_paramp->ah_attr); 1986 usr_ah_attrp = &(usr_ud_paramp->ah_attr); 1987 bcopy(&(ah_attrp->grh.dgid), &(usr_ah_attrp->grh.dgid[0]), 16); 1988 usr_ah_attrp->grh.flow_label = ah_attrp->grh.flow_label; 1989 usr_ah_attrp->grh.sgid_index = ah_attrp->grh.sgid_index; 1990 usr_ah_attrp->grh.hop_limit = ah_attrp->grh.hop_limit; 1991 usr_ah_attrp->grh.traffic_class = ah_attrp->grh.traffic_class; 1992 usr_ah_attrp->dlid = ah_attrp->dlid; 1993 usr_ah_attrp->sl = ah_attrp->sl; 1994 usr_ah_attrp->src_path_bits = ah_attrp->src_path_bits; 1995 usr_ah_attrp->static_rate = ah_attrp->static_rate; 1996 usr_ah_attrp->is_global = ah_attrp->ah_flags; 1997 usr_ah_attrp->port_num = ah_attrp->port_num; 1998 } 1999 2000 static void 2001 sol_ucma_user_objs_init() 2002 { 2003 sol_ofs_uobj_tbl_init(&ucma_file_uo_tbl, sizeof (sol_ucma_file_t)); 2004 sol_ofs_uobj_tbl_init(&ucma_ctx_uo_tbl, sizeof (sol_ucma_chan_t)); 2005 sol_ofs_uobj_tbl_init(&ucma_mcast_uo_tbl, sizeof (sol_ucma_mcast_t)); 2006 } 2007 2008 static void 2009 sol_ucma_user_objs_fini() 2010 { 2011 sol_ofs_uobj_tbl_fini(&ucma_file_uo_tbl); 2012 sol_ofs_uobj_tbl_fini(&ucma_ctx_uo_tbl); 2013 sol_ofs_uobj_tbl_fini(&ucma_mcast_uo_tbl); 2014 } 2015