1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Solaris SCSI RDMA Protocol Target (SRP) transport port provider 29 * module for the COMSTAR framework. 30 */ 31 32 #include <sys/cpuvar.h> 33 #include <sys/types.h> 34 #include <sys/conf.h> 35 #include <sys/stat.h> 36 #include <sys/file.h> 37 #include <sys/ddi.h> 38 #include <sys/sunddi.h> 39 #include <sys/modctl.h> 40 #include <sys/sysmacros.h> 41 #include <sys/sdt.h> 42 #include <sys/taskq.h> 43 44 #include <stmf.h> 45 #include <stmf_ioctl.h> 46 #include <portif.h> 47 48 #include "srp.h" 49 #include "srpt_impl.h" 50 #include "srpt_ioc.h" 51 #include "srpt_stp.h" 52 #include "srpt_cm.h" 53 #include "srpt_ioctl.h" 54 55 #define SRPT_NAME_VERSION "COMSTAR SRP Target" 56 57 /* 58 * srpt_send_msg_depth - Tunable parameter that specifies the 59 * maximum messages that could be in flight for a channel. 60 */ 61 uint16_t srpt_send_msg_depth = SRPT_DEFAULT_SEND_MSG_DEPTH; 62 63 /* 64 * srpt_errlevel -- determine which error conditions are logged 65 */ 66 uint_t srpt_errlevel = SRPT_LOG_DEFAULT_LEVEL; 67 68 /* 69 * srpt_iu_size -- must be a multiple of 64 as it is registered 70 * as memory regions with IB. To support a scatter/gather table 71 * size of 32, the size must be at not less than 960. To support 72 * the maximum scatter/gather table size of 255, the IU must 73 * be at least 4160 bytes. 74 */ 75 uint32_t srpt_iu_size = SRPT_DEFAULT_SEND_MSG_SIZE; 76 77 srpt_ctxt_t *srpt_ctxt; 78 79 /* 80 * DDI entry points. 81 */ 82 static int srpt_drv_attach(dev_info_t *, ddi_attach_cmd_t); 83 static int srpt_drv_detach(dev_info_t *, ddi_detach_cmd_t); 84 static int srpt_drv_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 85 static int srpt_drv_open(dev_t *, int, int, cred_t *); 86 static int srpt_drv_close(dev_t, int, int, cred_t *); 87 static int srpt_drv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); 88 89 /* helper functions */ 90 static int srpt_disable_srp_services(void); 91 static int srpt_enable_srp_services(void); 92 static int srpt_ibdma_ops_load(srpt_ibdma_ops_t *); 93 static void srpt_ibdma_ops_unload(srpt_ibdma_ops_t *); 94 95 extern struct mod_ops mod_miscops; 96 97 static struct cb_ops srpt_cb_ops = { 98 srpt_drv_open, /* cb_open */ 99 srpt_drv_close, /* cb_close */ 100 nodev, /* cb_strategy */ 101 nodev, /* cb_print */ 102 nodev, /* cb_dump */ 103 nodev, /* cb_read */ 104 nodev, /* cb_write */ 105 srpt_drv_ioctl, /* cb_ioctl */ 106 nodev, /* cb_devmap */ 107 nodev, /* cb_mmap */ 108 nodev, /* cb_segmap */ 109 nochpoll, /* cb_chpoll */ 110 ddi_prop_op, /* cb_prop_op */ 111 NULL, /* cb_streamtab */ 112 D_MP, /* cb_flag */ 113 CB_REV, /* cb_rev */ 114 nodev, /* cb_aread */ 115 nodev, /* cb_awrite */ 116 }; 117 118 static struct dev_ops srpt_dev_ops = { 119 DEVO_REV, /* devo_rev */ 120 0, /* devo_refcnt */ 121 srpt_drv_getinfo, /* devo_getinfo */ 122 nulldev, /* devo_identify */ 123 nulldev, /* devo_probe */ 124 srpt_drv_attach, /* devo_attach */ 125 srpt_drv_detach, /* devo_detach */ 126 nodev, /* devo_reset */ 127 &srpt_cb_ops, /* devo_cb_ops */ 128 NULL, /* devo_bus_ops */ 129 NULL, /* devo_power */ 130 ddi_quiesce_not_needed, /* quiesce */ 131 }; 132 133 static struct modldrv modldrv = { 134 &mod_driverops, 135 SRPT_NAME_VERSION, 136 &srpt_dev_ops, 137 }; 138 139 static struct modlinkage srpt_modlinkage = { 140 MODREV_1, 141 &modldrv, 142 NULL, 143 }; 144 145 static char srpt_pp_name[] = "srpt"; 146 147 /* 148 * Prototypes 149 */ 150 static void srpt_pp_cb(stmf_port_provider_t *, int, void *, uint32_t); 151 152 /* 153 * _init() 154 */ 155 int 156 _init(void) 157 { 158 int status; 159 160 /* 161 * Global one time initialization. 162 */ 163 srpt_ctxt = kmem_zalloc(sizeof (srpt_ctxt_t), KM_SLEEP); 164 ASSERT(srpt_ctxt != NULL); 165 rw_init(&srpt_ctxt->sc_rwlock, NULL, RW_DRIVER, NULL); 166 167 /* Start-up state is DISABLED. SMF will tell us if we should enable. */ 168 srpt_ctxt->sc_svc_state = SRPT_SVC_DISABLED; 169 170 list_create(&srpt_ctxt->sc_ioc_list, sizeof (srpt_ioc_t), 171 offsetof(srpt_ioc_t, ioc_node)); 172 173 status = mod_install(&srpt_modlinkage); 174 if (status != DDI_SUCCESS) { 175 cmn_err(CE_CONT, "_init, failed mod_install %d", status); 176 rw_destroy(&srpt_ctxt->sc_rwlock); 177 kmem_free(srpt_ctxt, sizeof (srpt_ctxt_t)); 178 srpt_ctxt = NULL; 179 } 180 181 return (status); 182 } 183 184 /* 185 * _info() 186 */ 187 int 188 _info(struct modinfo *modinfop) 189 { 190 return (mod_info(&srpt_modlinkage, modinfop)); 191 } 192 193 /* 194 * _fini() 195 */ 196 int 197 _fini(void) 198 { 199 int status; 200 201 status = mod_remove(&srpt_modlinkage); 202 if (status != DDI_SUCCESS) { 203 return (status); 204 } 205 206 list_destroy(&srpt_ctxt->sc_ioc_list); 207 208 rw_destroy(&srpt_ctxt->sc_rwlock); 209 kmem_free(srpt_ctxt, sizeof (srpt_ctxt_t)); 210 srpt_ctxt = NULL; 211 212 return (status); 213 } 214 215 /* 216 * DDI entry points. 217 */ 218 219 /* 220 * srpt_getinfo() 221 */ 222 /* ARGSUSED */ 223 static int 224 srpt_drv_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 225 { 226 227 switch (cmd) { 228 case DDI_INFO_DEVT2DEVINFO: 229 *result = srpt_ctxt->sc_dip; 230 return (DDI_SUCCESS); 231 232 case DDI_INFO_DEVT2INSTANCE: 233 *result = NULL; 234 return (DDI_SUCCESS); 235 236 default: 237 break; 238 } 239 return (DDI_FAILURE); 240 } 241 242 /* 243 * srpt_drv_attach() 244 */ 245 static int 246 srpt_drv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 247 { 248 int status; 249 250 switch (cmd) { 251 case DDI_ATTACH: 252 break; 253 254 case DDI_RESUME: 255 return (DDI_SUCCESS); 256 257 default: 258 return (DDI_FAILURE); 259 } 260 261 /* 262 * We only allow a single instance. 263 */ 264 if (ddi_get_instance(dip) != 0) { 265 SRPT_DPRINTF_L1("drv_attach, error non-zero instance"); 266 return (DDI_FAILURE); 267 } 268 269 /* 270 * Create minor node that might ultimately be used to create 271 * targets outside of srpt. 272 */ 273 status = ddi_create_minor_node(dip, ddi_get_name(dip), 274 S_IFCHR, 0, DDI_PSEUDO, 0); 275 if (status != DDI_SUCCESS) { 276 SRPT_DPRINTF_L1("drv_attach, minor node creation error (%d)", 277 status); 278 return (DDI_FAILURE); 279 } 280 281 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 282 srpt_ctxt->sc_dip = dip; 283 rw_exit(&srpt_ctxt->sc_rwlock); 284 285 return (DDI_SUCCESS); 286 } 287 288 /* 289 * srpt_enable_srp_services() 290 * 291 * Caller must be holding the sc_rwlock as RW_WRITER. 292 */ 293 static int 294 srpt_enable_srp_services(void) 295 { 296 int status; 297 srpt_ioc_t *ioc; 298 299 ASSERT((rw_read_locked(&srpt_ctxt->sc_rwlock)) == 0); 300 301 SRPT_DPRINTF_L3("srpt_enable_srp_services"); 302 303 /* Register the port provider */ 304 srpt_ctxt->sc_pp = (stmf_port_provider_t *) 305 stmf_alloc(STMF_STRUCT_PORT_PROVIDER, 0, 0); 306 srpt_ctxt->sc_pp->pp_portif_rev = PORTIF_REV_1; 307 srpt_ctxt->sc_pp->pp_name = srpt_pp_name; 308 srpt_ctxt->sc_pp->pp_cb = srpt_pp_cb; 309 status = stmf_register_port_provider(srpt_ctxt->sc_pp); 310 if (status != STMF_SUCCESS) { 311 SRPT_DPRINTF_L1("enable_srp: SRP port_provider registration" 312 " failed(%d)", status); 313 goto err_exit_1; 314 } 315 316 /* 317 * Initialize IB resources, creating a list of SRP I/O Controllers. 318 */ 319 status = srpt_ioc_attach(); 320 if (status != DDI_SUCCESS) { 321 SRPT_DPRINTF_L1("enable_srp: error attach I/O" 322 " Controllers (%d)", status); 323 goto err_exit_2; 324 } 325 326 /* Not an error if no iocs yet; we will listen for ATTACH events */ 327 if (srpt_ctxt->sc_num_iocs == 0) { 328 SRPT_DPRINTF_L2("enable_srp: no IB I/O Controllers found"); 329 return (DDI_SUCCESS); 330 } 331 332 /* 333 * For each I/O Controller register the default SCSI Target Port 334 * with STMF, and prepare profile and services. SRP will not 335 * start until the associated LPORT is brought on-line. 336 */ 337 ioc = list_head(&srpt_ctxt->sc_ioc_list); 338 339 while (ioc != NULL) { 340 rw_enter(&ioc->ioc_rwlock, RW_WRITER); 341 ioc->ioc_tgt_port = srpt_stp_alloc_port(ioc, ioc->ioc_guid); 342 if (ioc->ioc_tgt_port == NULL) { 343 SRPT_DPRINTF_L1("enable_srp: alloc SCSI" 344 " Target Port error on GUID(%016llx)", 345 (u_longlong_t)ioc->ioc_guid); 346 } 347 348 rw_exit(&ioc->ioc_rwlock); 349 ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); 350 } 351 352 return (DDI_SUCCESS); 353 354 err_exit_2: 355 (void) stmf_deregister_port_provider(srpt_ctxt->sc_pp); 356 357 err_exit_1: 358 stmf_free(srpt_ctxt->sc_pp); 359 srpt_ctxt->sc_pp = NULL; 360 361 return (status); 362 } 363 364 /* 365 * srpt_drv_detach() 366 * 367 * Refuse the detach request if we have channels open on 368 * any IOC. Users should use 'svcadm disable' to shutdown 369 * active targets. 370 */ 371 /*ARGSUSED*/ 372 static int 373 srpt_drv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 374 { 375 switch (cmd) { 376 case DDI_DETACH: 377 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 378 if (srpt_ctxt->sc_svc_state != SRPT_SVC_DISABLED) { 379 rw_exit(&srpt_ctxt->sc_rwlock); 380 return (DDI_FAILURE); 381 } 382 383 ddi_remove_minor_node(dip, NULL); 384 srpt_ctxt->sc_dip = NULL; 385 386 rw_exit(&srpt_ctxt->sc_rwlock); 387 388 break; 389 390 case DDI_SUSPEND: 391 return (DDI_FAILURE); 392 393 default: 394 return (DDI_FAILURE); 395 } 396 397 return (DDI_SUCCESS); 398 } 399 400 /* 401 * srpt_disable_srp_services() 402 * 403 * Offlines all targets, deregisters all IOCs. Caller must hold 404 * the srpt_ctxt->sc_rwlock as RW_WRITER. 405 */ 406 static int 407 srpt_disable_srp_services(void) 408 { 409 stmf_status_t stmf_status; 410 stmf_change_status_t cstatus; 411 srpt_ioc_t *ioc; 412 srpt_target_port_t *tgt; 413 int ret_status = 0; 414 415 ASSERT((rw_read_locked(&srpt_ctxt->sc_rwlock)) == 0); 416 417 /* 418 * For each I/O Controller remove all SRP services and de-register 419 * with the associated I/O Unit's IB Device Management Agent. 420 */ 421 ioc = list_head(&srpt_ctxt->sc_ioc_list); 422 423 while (ioc != NULL) { 424 /* 425 * Notify STMF to take the I/O Controller SCSI Target Port(s) 426 * off-line after we mark them as disabled so that they will 427 * stay off-line. 428 */ 429 rw_enter(&ioc->ioc_rwlock, RW_WRITER); 430 431 tgt = ioc->ioc_tgt_port; 432 if (tgt != NULL) { 433 mutex_enter(&tgt->tp_lock); 434 tgt->tp_drv_disabled = 1; 435 mutex_exit(&tgt->tp_lock); 436 437 SRPT_DPRINTF_L2("disable_srp: unbind and de-register" 438 " services for GUID(%016llx)", 439 (u_longlong_t)ioc->ioc_guid); 440 441 cstatus.st_completion_status = STMF_SUCCESS; 442 cstatus.st_additional_info = NULL; 443 444 stmf_status = stmf_ctl(STMF_CMD_LPORT_OFFLINE, 445 tgt->tp_lport, &cstatus); 446 447 /* 448 * Wait for asynchronous target off-line operation 449 * to complete and then deregister the target 450 * port. 451 */ 452 mutex_enter(&tgt->tp_lock); 453 while (tgt->tp_state != SRPT_TGT_STATE_OFFLINE) { 454 cv_wait(&tgt->tp_offline_complete, 455 &tgt->tp_lock); 456 } 457 mutex_exit(&tgt->tp_lock); 458 ioc->ioc_tgt_port = NULL; 459 460 SRPT_DPRINTF_L3("disable_srp: IOC (0x%016llx) Target" 461 " SRP off-line complete", 462 (u_longlong_t)ioc->ioc_guid); 463 464 stmf_status = srpt_stp_deregister_port(tgt); 465 if (stmf_status != STMF_SUCCESS) { 466 /* Fails if I/O is pending */ 467 if (ret_status == 0) { 468 ret_status = EBUSY; 469 } 470 SRPT_DPRINTF_L1("disable_srp: could not" 471 " de-register LPORT, err(0x%llx)", 472 (u_longlong_t)stmf_status); 473 } else { 474 (void) srpt_stp_free_port(tgt); 475 } 476 } 477 478 rw_exit(&ioc->ioc_rwlock); 479 ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); 480 } 481 482 /* don't release IOCs until all ports are deregistered */ 483 if (ret_status != 0) { 484 return (ret_status); 485 } 486 487 /* 488 * Release I/O Controller(s) resources and detach. 489 */ 490 srpt_ioc_detach(); 491 492 /* De-register ourselves as an STMF port provider */ 493 (void) stmf_deregister_port_provider(srpt_ctxt->sc_pp); 494 stmf_free(srpt_ctxt->sc_pp); 495 srpt_ctxt->sc_pp = NULL; 496 497 return (0); 498 } 499 500 /* 501 * srpt_drv_open() 502 */ 503 /* ARGSUSED */ 504 static int 505 srpt_drv_open(dev_t *devp, int flag, int otyp, cred_t *credp) 506 { 507 SRPT_DPRINTF_L3("drv_open, invoked"); 508 return (0); 509 } 510 511 /* 512 * srpt_drv_close() 513 */ 514 /* ARGSUSED */ 515 static int 516 srpt_drv_close(dev_t dev, int flag, int otyp, cred_t *credp) 517 { 518 SRPT_DPRINTF_L3("drv_close, invoked"); 519 return (0); 520 } 521 522 /* 523 * srpt_drv_ioctl() 524 */ 525 /* ARGSUSED */ 526 static int 527 srpt_drv_ioctl(dev_t drv, int cmd, intptr_t argp, int flag, cred_t *cred, 528 int *retval) 529 { 530 int ret = 0; 531 532 SRPT_DPRINTF_L3("drv_ioctl, invoked, cmd = %d", cmd); 533 534 if (drv_priv(cred) != 0) { 535 return (EPERM); 536 } 537 538 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 539 540 switch (cmd) { 541 case SRPT_IOC_ENABLE_SVC: 542 if (srpt_ctxt->sc_svc_state != SRPT_SVC_DISABLED) { 543 break; 544 } 545 546 ret = srpt_ibdma_ops_load(&srpt_ctxt->sc_ibdma_ops); 547 if (ret != 0) { 548 break; 549 } 550 551 ret = srpt_enable_srp_services(); 552 if (ret == 0) { 553 srpt_ctxt->sc_svc_state = SRPT_SVC_ENABLED; 554 } 555 556 break; 557 558 case SRPT_IOC_DISABLE_SVC: 559 if (srpt_ctxt->sc_svc_state != SRPT_SVC_ENABLED) { 560 break; 561 } 562 563 ret = srpt_disable_srp_services(); 564 if (ret == 0) { 565 srpt_ctxt->sc_svc_state = SRPT_SVC_DISABLED; 566 } 567 568 srpt_ibdma_ops_unload(&srpt_ctxt->sc_ibdma_ops); 569 570 break; 571 572 default: 573 ret = EFAULT; 574 break; 575 } 576 577 rw_exit(&srpt_ctxt->sc_rwlock); 578 579 return (ret); 580 } 581 582 /* 583 * srpt_pp_cb() 584 */ 585 /* ARGSUSED */ 586 static void 587 srpt_pp_cb(stmf_port_provider_t *pp, int cmd, void *arg, uint32_t flags) 588 { 589 SRPT_DPRINTF_L3("srpt_pp_cb, invoked (%d)", cmd); 590 /* 591 * We don't currently utilize the port provider call-back, in the 592 * future we might use it to synchronize provider data via STMF. 593 */ 594 } 595 596 static int 597 srpt_ibdma_ops_load(srpt_ibdma_ops_t *ops) 598 { 599 int ibdma_err = 0; 600 601 ASSERT(ops != NULL); 602 603 ops->ibdmah = ddi_modopen("ibdma", KRTLD_MODE_FIRST, &ibdma_err); 604 if (ops->ibdmah == NULL) { 605 SRPT_DPRINTF_L0("failed to open ibdma driver, error = %d", 606 ibdma_err); 607 return (ibdma_err); 608 } 609 610 ops->ibdma_register = (ibdma_hdl_t (*)())ddi_modsym(ops->ibdmah, 611 "ibdma_ioc_register", &ibdma_err); 612 if (ops->ibdma_register == NULL) { 613 SRPT_DPRINTF_L0( 614 "failed to modsym ibdma_ioc_register, error = %d", 615 ibdma_err); 616 goto done; 617 } 618 619 ops->ibdma_unregister = (ibdma_status_t (*)())ddi_modsym(ops->ibdmah, 620 "ibdma_ioc_unregister", &ibdma_err); 621 if (ops->ibdma_unregister == NULL) { 622 SRPT_DPRINTF_L0( 623 "failed to modsym ibdma_ioc_unregister, error = %d", 624 ibdma_err); 625 goto done; 626 } 627 628 ops->ibdma_update = (ibdma_status_t (*)())ddi_modsym(ops->ibdmah, 629 "ibdma_ioc_update", &ibdma_err); 630 if (ops->ibdma_update == NULL) { 631 SRPT_DPRINTF_L0( 632 "failed to modsym ibdma_ioc_update, error = %d", 633 ibdma_err); 634 } 635 636 done: 637 if (ibdma_err != 0) { 638 srpt_ibdma_ops_unload(ops); 639 } 640 641 return (ibdma_err); 642 } 643 644 static void 645 srpt_ibdma_ops_unload(srpt_ibdma_ops_t *ops) 646 { 647 if (ops != NULL) { 648 if (ops->ibdmah != NULL) { 649 (void) ddi_modclose(ops->ibdmah); 650 } 651 bzero(ops, sizeof (srpt_ibdma_ops_t)); 652 } 653 } 654