1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Solaris SCSI RDMA Protocol Target (SRP) transport port provider 29 * module for the COMSTAR framework. 30 */ 31 32 #include <sys/cpuvar.h> 33 #include <sys/types.h> 34 #include <sys/conf.h> 35 #include <sys/stat.h> 36 #include <sys/file.h> 37 #include <sys/ddi.h> 38 #include <sys/sunddi.h> 39 #include <sys/modctl.h> 40 #include <sys/sysmacros.h> 41 #include <sys/sdt.h> 42 #include <sys/taskq.h> 43 44 #include <stmf.h> 45 #include <stmf_ioctl.h> 46 #include <portif.h> 47 48 #include "srp.h" 49 #include "srpt_impl.h" 50 #include "srpt_ioc.h" 51 #include "srpt_stp.h" 52 #include "srpt_cm.h" 53 #include "srpt_ioctl.h" 54 55 #define SRPT_NAME_VERSION "COMSTAR SRP Target" 56 57 /* 58 * srpt_send_msg_depth - Tunable parameter that specifies the 59 * maximum messages that could be in flight for a channel. 60 */ 61 uint16_t srpt_send_msg_depth = SRPT_DEFAULT_SEND_MSG_DEPTH; 62 63 /* 64 * srpt_errlevel -- determine which error conditions are logged 65 */ 66 uint_t srpt_errlevel = SRPT_LOG_DEFAULT_LEVEL; 67 68 srpt_ctxt_t *srpt_ctxt; 69 70 /* 71 * DDI entry points. 72 */ 73 static int srpt_drv_attach(dev_info_t *, ddi_attach_cmd_t); 74 static int srpt_drv_detach(dev_info_t *, ddi_detach_cmd_t); 75 static int srpt_drv_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 76 static int srpt_drv_open(dev_t *, int, int, cred_t *); 77 static int srpt_drv_close(dev_t, int, int, cred_t *); 78 static int srpt_drv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); 79 80 /* helper functions */ 81 static int srpt_disable_srp_services(void); 82 static int srpt_enable_srp_services(void); 83 static int srpt_ibdma_ops_load(srpt_ibdma_ops_t *); 84 static void srpt_ibdma_ops_unload(srpt_ibdma_ops_t *); 85 86 extern struct mod_ops mod_miscops; 87 88 static struct cb_ops srpt_cb_ops = { 89 srpt_drv_open, /* cb_open */ 90 srpt_drv_close, /* cb_close */ 91 nodev, /* cb_strategy */ 92 nodev, /* cb_print */ 93 nodev, /* cb_dump */ 94 nodev, /* cb_read */ 95 nodev, /* cb_write */ 96 srpt_drv_ioctl, /* cb_ioctl */ 97 nodev, /* cb_devmap */ 98 nodev, /* cb_mmap */ 99 nodev, /* cb_segmap */ 100 nochpoll, /* cb_chpoll */ 101 ddi_prop_op, /* cb_prop_op */ 102 NULL, /* cb_streamtab */ 103 D_MP, /* cb_flag */ 104 CB_REV, /* cb_rev */ 105 nodev, /* cb_aread */ 106 nodev, /* cb_awrite */ 107 }; 108 109 static struct dev_ops srpt_dev_ops = { 110 DEVO_REV, /* devo_rev */ 111 0, /* devo_refcnt */ 112 srpt_drv_getinfo, /* devo_getinfo */ 113 nulldev, /* devo_identify */ 114 nulldev, /* devo_probe */ 115 srpt_drv_attach, /* devo_attach */ 116 srpt_drv_detach, /* devo_detach */ 117 nodev, /* devo_reset */ 118 &srpt_cb_ops, /* devo_cb_ops */ 119 NULL, /* devo_bus_ops */ 120 NULL, /* devo_power */ 121 ddi_quiesce_not_needed, /* quiesce */ 122 }; 123 124 static struct modldrv modldrv = { 125 &mod_driverops, 126 SRPT_NAME_VERSION, 127 &srpt_dev_ops, 128 }; 129 130 static struct modlinkage srpt_modlinkage = { 131 MODREV_1, 132 &modldrv, 133 NULL, 134 }; 135 136 static char srpt_pp_name[] = "srpt"; 137 138 /* 139 * Prototypes 140 */ 141 static void srpt_pp_cb(stmf_port_provider_t *, int, void *, uint32_t); 142 143 /* 144 * _init() 145 */ 146 int 147 _init(void) 148 { 149 int status; 150 151 /* 152 * Global one time initialization. 153 */ 154 srpt_ctxt = kmem_zalloc(sizeof (srpt_ctxt_t), KM_SLEEP); 155 ASSERT(srpt_ctxt != NULL); 156 rw_init(&srpt_ctxt->sc_rwlock, NULL, RW_DRIVER, NULL); 157 158 /* Start-up state is DISABLED. SMF will tell us if we should enable. */ 159 srpt_ctxt->sc_svc_state = SRPT_SVC_DISABLED; 160 161 list_create(&srpt_ctxt->sc_ioc_list, sizeof (srpt_ioc_t), 162 offsetof(srpt_ioc_t, ioc_node)); 163 164 status = mod_install(&srpt_modlinkage); 165 if (status != DDI_SUCCESS) { 166 cmn_err(CE_CONT, "_init, failed mod_install %d", status); 167 rw_destroy(&srpt_ctxt->sc_rwlock); 168 kmem_free(srpt_ctxt, sizeof (srpt_ctxt_t)); 169 srpt_ctxt = NULL; 170 } 171 172 return (status); 173 } 174 175 /* 176 * _info() 177 */ 178 int 179 _info(struct modinfo *modinfop) 180 { 181 return (mod_info(&srpt_modlinkage, modinfop)); 182 } 183 184 /* 185 * _fini() 186 */ 187 int 188 _fini(void) 189 { 190 int status; 191 192 status = mod_remove(&srpt_modlinkage); 193 if (status != DDI_SUCCESS) { 194 return (status); 195 } 196 197 list_destroy(&srpt_ctxt->sc_ioc_list); 198 199 rw_destroy(&srpt_ctxt->sc_rwlock); 200 kmem_free(srpt_ctxt, sizeof (srpt_ctxt_t)); 201 srpt_ctxt = NULL; 202 203 return (status); 204 } 205 206 /* 207 * DDI entry points. 208 */ 209 210 /* 211 * srpt_getinfo() 212 */ 213 /* ARGSUSED */ 214 static int 215 srpt_drv_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 216 { 217 218 switch (cmd) { 219 case DDI_INFO_DEVT2DEVINFO: 220 *result = srpt_ctxt->sc_dip; 221 return (DDI_SUCCESS); 222 223 case DDI_INFO_DEVT2INSTANCE: 224 *result = NULL; 225 return (DDI_SUCCESS); 226 227 default: 228 break; 229 } 230 return (DDI_FAILURE); 231 } 232 233 /* 234 * srpt_drv_attach() 235 */ 236 static int 237 srpt_drv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 238 { 239 int status; 240 241 switch (cmd) { 242 case DDI_ATTACH: 243 break; 244 245 case DDI_RESUME: 246 return (DDI_SUCCESS); 247 248 default: 249 return (DDI_FAILURE); 250 } 251 252 /* 253 * We only allow a single instance. 254 */ 255 if (ddi_get_instance(dip) != 0) { 256 SRPT_DPRINTF_L1("drv_attach, error non-zero instance"); 257 return (DDI_FAILURE); 258 } 259 260 /* 261 * Create minor node that might ultimately be used to create 262 * targets outside of srpt. 263 */ 264 status = ddi_create_minor_node(dip, ddi_get_name(dip), 265 S_IFCHR, 0, DDI_PSEUDO, 0); 266 if (status != DDI_SUCCESS) { 267 SRPT_DPRINTF_L1("drv_attach, minor node creation error (%d)", 268 status); 269 return (DDI_FAILURE); 270 } 271 272 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 273 srpt_ctxt->sc_dip = dip; 274 rw_exit(&srpt_ctxt->sc_rwlock); 275 276 return (DDI_SUCCESS); 277 } 278 279 /* 280 * srpt_enable_srp_services() 281 * 282 * Caller must be holding the sc_rwlock as RW_WRITER. 283 */ 284 static int 285 srpt_enable_srp_services(void) 286 { 287 int status; 288 srpt_ioc_t *ioc; 289 290 ASSERT((rw_read_locked(&srpt_ctxt->sc_rwlock)) == 0); 291 292 SRPT_DPRINTF_L3("srpt_enable_srp_services"); 293 294 /* Register the port provider */ 295 srpt_ctxt->sc_pp = (stmf_port_provider_t *) 296 stmf_alloc(STMF_STRUCT_PORT_PROVIDER, 0, 0); 297 srpt_ctxt->sc_pp->pp_portif_rev = PORTIF_REV_1; 298 srpt_ctxt->sc_pp->pp_name = srpt_pp_name; 299 srpt_ctxt->sc_pp->pp_cb = srpt_pp_cb; 300 status = stmf_register_port_provider(srpt_ctxt->sc_pp); 301 if (status != STMF_SUCCESS) { 302 SRPT_DPRINTF_L1("enable_srp: SRP port_provider registration" 303 " failed(%d)", status); 304 goto err_exit_1; 305 } 306 307 /* 308 * Initialize IB resources, creating a list of SRP I/O Controllers. 309 */ 310 status = srpt_ioc_attach(); 311 if (status != DDI_SUCCESS) { 312 SRPT_DPRINTF_L1("enable_srp: error attach I/O" 313 " Controllers (%d)", status); 314 goto err_exit_2; 315 } 316 317 /* Not an error if no iocs yet; we will listen for ATTACH events */ 318 if (srpt_ctxt->sc_num_iocs == 0) { 319 SRPT_DPRINTF_L2("enable_srp: no IB I/O Controllers found"); 320 return (DDI_SUCCESS); 321 } 322 323 /* 324 * For each I/O Controller register the default SCSI Target Port 325 * with STMF, and prepare profile and services. SRP will not 326 * start until the associated LPORT is brought on-line. 327 */ 328 ioc = list_head(&srpt_ctxt->sc_ioc_list); 329 330 while (ioc != NULL) { 331 rw_enter(&ioc->ioc_rwlock, RW_WRITER); 332 ioc->ioc_tgt_port = srpt_stp_alloc_port(ioc, ioc->ioc_guid); 333 if (ioc->ioc_tgt_port == NULL) { 334 SRPT_DPRINTF_L1("enable_srp: alloc SCSI" 335 " Target Port error on GUID(%016llx)", 336 (u_longlong_t)ioc->ioc_guid); 337 } 338 339 rw_exit(&ioc->ioc_rwlock); 340 ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); 341 } 342 343 return (DDI_SUCCESS); 344 345 err_exit_2: 346 (void) stmf_deregister_port_provider(srpt_ctxt->sc_pp); 347 348 err_exit_1: 349 stmf_free(srpt_ctxt->sc_pp); 350 srpt_ctxt->sc_pp = NULL; 351 352 return (status); 353 } 354 355 /* 356 * srpt_drv_detach() 357 * 358 * Refuse the detach request if we have channels open on 359 * any IOC. Users should use 'svcadm disable' to shutdown 360 * active targets. 361 */ 362 /*ARGSUSED*/ 363 static int 364 srpt_drv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 365 { 366 switch (cmd) { 367 case DDI_DETACH: 368 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 369 if (srpt_ctxt->sc_svc_state != SRPT_SVC_DISABLED) { 370 rw_exit(&srpt_ctxt->sc_rwlock); 371 return (DDI_FAILURE); 372 } 373 374 ddi_remove_minor_node(dip, NULL); 375 srpt_ctxt->sc_dip = NULL; 376 377 rw_exit(&srpt_ctxt->sc_rwlock); 378 379 break; 380 381 case DDI_SUSPEND: 382 return (DDI_FAILURE); 383 384 default: 385 return (DDI_FAILURE); 386 } 387 388 return (DDI_SUCCESS); 389 } 390 391 /* 392 * srpt_disable_srp_services() 393 * 394 * Offlines all targets, deregisters all IOCs. Caller must hold 395 * the srpt_ctxt->sc_rwlock as RW_WRITER. 396 */ 397 static int 398 srpt_disable_srp_services(void) 399 { 400 stmf_status_t stmf_status; 401 stmf_change_status_t cstatus; 402 srpt_ioc_t *ioc; 403 srpt_target_port_t *tgt; 404 int ret_status = 0; 405 406 ASSERT((rw_read_locked(&srpt_ctxt->sc_rwlock)) == 0); 407 408 /* 409 * For each I/O Controller remove all SRP services and de-register 410 * with the associated I/O Unit's IB Device Management Agent. 411 */ 412 ioc = list_head(&srpt_ctxt->sc_ioc_list); 413 414 while (ioc != NULL) { 415 /* 416 * Notify STMF to take the I/O Controller SCSI Target Port(s) 417 * off-line after we mark them as disabled so that they will 418 * stay off-line. 419 */ 420 rw_enter(&ioc->ioc_rwlock, RW_WRITER); 421 422 tgt = ioc->ioc_tgt_port; 423 if (tgt != NULL) { 424 mutex_enter(&tgt->tp_lock); 425 tgt->tp_drv_disabled = 1; 426 mutex_exit(&tgt->tp_lock); 427 428 SRPT_DPRINTF_L2("disable_srp: unbind and de-register" 429 " services for GUID(%016llx)", 430 (u_longlong_t)ioc->ioc_guid); 431 432 cstatus.st_completion_status = STMF_SUCCESS; 433 cstatus.st_additional_info = NULL; 434 435 stmf_status = stmf_ctl(STMF_CMD_LPORT_OFFLINE, 436 tgt->tp_lport, &cstatus); 437 438 /* 439 * Wait for asynchronous target off-line operation 440 * to complete and then deregister the target 441 * port. 442 */ 443 mutex_enter(&tgt->tp_lock); 444 while (tgt->tp_state != SRPT_TGT_STATE_OFFLINE) { 445 cv_wait(&tgt->tp_offline_complete, 446 &tgt->tp_lock); 447 } 448 mutex_exit(&tgt->tp_lock); 449 ioc->ioc_tgt_port = NULL; 450 451 SRPT_DPRINTF_L3("disable_srp: IOC (0x%016llx) Target" 452 " SRP off-line complete", 453 (u_longlong_t)ioc->ioc_guid); 454 455 stmf_status = srpt_stp_deregister_port(tgt); 456 if (stmf_status != STMF_SUCCESS) { 457 /* Fails if I/O is pending */ 458 if (ret_status == 0) { 459 ret_status = EBUSY; 460 } 461 SRPT_DPRINTF_L1("disable_srp: could not" 462 " de-register LPORT, err(0x%llx)", 463 (u_longlong_t)stmf_status); 464 } else { 465 (void) srpt_stp_free_port(tgt); 466 } 467 } 468 469 rw_exit(&ioc->ioc_rwlock); 470 ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); 471 } 472 473 /* don't release IOCs until all ports are deregistered */ 474 if (ret_status != 0) { 475 return (ret_status); 476 } 477 478 /* 479 * Release I/O Controller(s) resources and detach. 480 */ 481 srpt_ioc_detach(); 482 483 /* De-register ourselves as an STMF port provider */ 484 (void) stmf_deregister_port_provider(srpt_ctxt->sc_pp); 485 stmf_free(srpt_ctxt->sc_pp); 486 srpt_ctxt->sc_pp = NULL; 487 488 return (0); 489 } 490 491 /* 492 * srpt_drv_open() 493 */ 494 /* ARGSUSED */ 495 static int 496 srpt_drv_open(dev_t *devp, int flag, int otyp, cred_t *credp) 497 { 498 SRPT_DPRINTF_L3("drv_open, invoked"); 499 return (0); 500 } 501 502 /* 503 * srpt_drv_close() 504 */ 505 /* ARGSUSED */ 506 static int 507 srpt_drv_close(dev_t dev, int flag, int otyp, cred_t *credp) 508 { 509 SRPT_DPRINTF_L3("drv_close, invoked"); 510 return (0); 511 } 512 513 /* 514 * srpt_drv_ioctl() 515 */ 516 /* ARGSUSED */ 517 static int 518 srpt_drv_ioctl(dev_t drv, int cmd, intptr_t argp, int flag, cred_t *cred, 519 int *retval) 520 { 521 int ret = 0; 522 523 SRPT_DPRINTF_L3("drv_ioctl, invoked, cmd = %d", cmd); 524 525 if (drv_priv(cred) != 0) { 526 return (EPERM); 527 } 528 529 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 530 531 switch (cmd) { 532 case SRPT_IOC_ENABLE_SVC: 533 if (srpt_ctxt->sc_svc_state != SRPT_SVC_DISABLED) { 534 break; 535 } 536 537 ret = srpt_ibdma_ops_load(&srpt_ctxt->sc_ibdma_ops); 538 if (ret != 0) { 539 break; 540 } 541 542 ret = srpt_enable_srp_services(); 543 if (ret == 0) { 544 srpt_ctxt->sc_svc_state = SRPT_SVC_ENABLED; 545 } 546 547 break; 548 549 case SRPT_IOC_DISABLE_SVC: 550 if (srpt_ctxt->sc_svc_state != SRPT_SVC_ENABLED) { 551 break; 552 } 553 554 ret = srpt_disable_srp_services(); 555 if (ret == 0) { 556 srpt_ctxt->sc_svc_state = SRPT_SVC_DISABLED; 557 } 558 559 srpt_ibdma_ops_unload(&srpt_ctxt->sc_ibdma_ops); 560 561 break; 562 563 default: 564 ret = EFAULT; 565 break; 566 } 567 568 rw_exit(&srpt_ctxt->sc_rwlock); 569 570 return (ret); 571 } 572 573 /* 574 * srpt_pp_cb() 575 */ 576 /* ARGSUSED */ 577 static void 578 srpt_pp_cb(stmf_port_provider_t *pp, int cmd, void *arg, uint32_t flags) 579 { 580 SRPT_DPRINTF_L3("srpt_pp_cb, invoked (%d)", cmd); 581 /* 582 * We don't currently utilize the port provider call-back, in the 583 * future we might use it to synchronize provider data via STMF. 584 */ 585 } 586 587 static int 588 srpt_ibdma_ops_load(srpt_ibdma_ops_t *ops) 589 { 590 int ibdma_err = 0; 591 592 ASSERT(ops != NULL); 593 594 ops->ibdmah = ddi_modopen("ibdma", KRTLD_MODE_FIRST, &ibdma_err); 595 if (ops->ibdmah == NULL) { 596 SRPT_DPRINTF_L0("failed to open ibdma driver, error = %d", 597 ibdma_err); 598 return (ibdma_err); 599 } 600 601 ops->ibdma_register = (ibdma_hdl_t (*)())ddi_modsym(ops->ibdmah, 602 "ibdma_ioc_register", &ibdma_err); 603 if (ops->ibdma_register == NULL) { 604 SRPT_DPRINTF_L0( 605 "failed to modsym ibdma_ioc_register, error = %d", 606 ibdma_err); 607 goto done; 608 } 609 610 ops->ibdma_unregister = (ibdma_status_t (*)())ddi_modsym(ops->ibdmah, 611 "ibdma_ioc_unregister", &ibdma_err); 612 if (ops->ibdma_unregister == NULL) { 613 SRPT_DPRINTF_L0( 614 "failed to modsym ibdma_ioc_unregister, error = %d", 615 ibdma_err); 616 goto done; 617 } 618 619 ops->ibdma_update = (ibdma_status_t (*)())ddi_modsym(ops->ibdmah, 620 "ibdma_ioc_update", &ibdma_err); 621 if (ops->ibdma_update == NULL) { 622 SRPT_DPRINTF_L0( 623 "failed to modsym ibdma_ioc_update, error = %d", 624 ibdma_err); 625 } 626 627 done: 628 if (ibdma_err != 0) { 629 srpt_ibdma_ops_unload(ops); 630 } 631 632 return (ibdma_err); 633 } 634 635 static void 636 srpt_ibdma_ops_unload(srpt_ibdma_ops_t *ops) 637 { 638 if (ops != NULL) { 639 if (ops->ibdmah != NULL) { 640 (void) ddi_modclose(ops->ibdmah); 641 } 642 bzero(ops, sizeof (srpt_ibdma_ops_t)); 643 } 644 } 645