1fcf3ce44SJohn Forte /* 2fcf3ce44SJohn Forte * CDDL HEADER START 3fcf3ce44SJohn Forte * 4fcf3ce44SJohn Forte * The contents of this file are subject to the terms of the 5fcf3ce44SJohn Forte * Common Development and Distribution License (the "License"). 6fcf3ce44SJohn Forte * You may not use this file except in compliance with the License. 7fcf3ce44SJohn Forte * 8fcf3ce44SJohn Forte * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fcf3ce44SJohn Forte * or http://www.opensolaris.org/os/licensing. 10fcf3ce44SJohn Forte * See the License for the specific language governing permissions 11fcf3ce44SJohn Forte * and limitations under the License. 12fcf3ce44SJohn Forte * 13fcf3ce44SJohn Forte * When distributing Covered Code, include this CDDL HEADER in each 14fcf3ce44SJohn Forte * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fcf3ce44SJohn Forte * If applicable, add the following below this CDDL HEADER, with the 16fcf3ce44SJohn Forte * fields enclosed by brackets "[]" replaced with your own identifying 17fcf3ce44SJohn Forte * information: Portions Copyright [yyyy] [name of copyright owner] 18fcf3ce44SJohn Forte * 19fcf3ce44SJohn Forte * CDDL HEADER END 20fcf3ce44SJohn Forte */ 21fcf3ce44SJohn Forte /* 2230e7468fSPeter Dunlap * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23fcf3ce44SJohn Forte * Use is subject to license terms. 24fcf3ce44SJohn Forte * 25fcf3ce44SJohn Forte * iSCSI connection interfaces 26fcf3ce44SJohn Forte */ 27fcf3ce44SJohn Forte 2830e7468fSPeter Dunlap #define ISCSI_ICS_NAMES 29fcf3ce44SJohn Forte #include "iscsi.h" 30fcf3ce44SJohn Forte #include "persistent.h" 316cefaae1SJack Meng #include <sys/bootprops.h> 326cefaae1SJack Meng 336cefaae1SJack Meng extern ib_boot_prop_t *iscsiboot_prop; 34fcf3ce44SJohn Forte 3530e7468fSPeter Dunlap static void iscsi_client_notify_task(void *cn_task_void); 36fcf3ce44SJohn Forte 3730e7468fSPeter Dunlap static void iscsi_conn_flush_active_cmds(iscsi_conn_t *icp); 38fcf3ce44SJohn Forte 39fcf3ce44SJohn Forte #define SHUTDOWN_TIMEOUT 180 /* seconds */ 40fcf3ce44SJohn Forte 416cefaae1SJack Meng extern int modrootloaded; 4230e7468fSPeter Dunlap 4330e7468fSPeter Dunlap boolean_t iscsi_conn_logging = B_FALSE; 4430e7468fSPeter Dunlap 45fcf3ce44SJohn Forte /* 46fcf3ce44SJohn Forte * +--------------------------------------------------------------------+ 47fcf3ce44SJohn Forte * | External Connection Interfaces | 48fcf3ce44SJohn Forte * +--------------------------------------------------------------------+ 49fcf3ce44SJohn Forte */ 50fcf3ce44SJohn Forte 51fcf3ce44SJohn Forte /* 52fcf3ce44SJohn Forte * iscsi_conn_create - This creates an iscsi connection structure and 53fcf3ce44SJohn Forte * associates it with a session structure. The session's sess_conn_list_rwlock 54fcf3ce44SJohn Forte * should be held as a writer before calling this function. 55fcf3ce44SJohn Forte */ 56fcf3ce44SJohn Forte iscsi_status_t 57fcf3ce44SJohn Forte iscsi_conn_create(struct sockaddr *addr, iscsi_sess_t *isp, iscsi_conn_t **icpp) 58fcf3ce44SJohn Forte { 59fcf3ce44SJohn Forte iscsi_conn_t *icp = NULL; 60fcf3ce44SJohn Forte char th_name[ISCSI_TH_MAX_NAME_LEN]; 61fcf3ce44SJohn Forte 62fcf3ce44SJohn Forte /* See if this connection already exists */ 63fcf3ce44SJohn Forte for (icp = isp->sess_conn_list; icp; icp = icp->conn_next) { 64fcf3ce44SJohn Forte 65fcf3ce44SJohn Forte /* 66fcf3ce44SJohn Forte * Compare the ioctl information to see if 67fcf3ce44SJohn Forte * its a match for this connection. (This 68fcf3ce44SJohn Forte * is done by making sure the IPs are of 69fcf3ce44SJohn Forte * the same size and then they are the 70fcf3ce44SJohn Forte * same value. 71fcf3ce44SJohn Forte */ 72fcf3ce44SJohn Forte if (bcmp(&icp->conn_base_addr, addr, 73fcf3ce44SJohn Forte SIZEOF_SOCKADDR(addr)) == 0) { 74fcf3ce44SJohn Forte /* It's a match, record this connection */ 75fcf3ce44SJohn Forte break; 76fcf3ce44SJohn Forte } 77fcf3ce44SJohn Forte } 78fcf3ce44SJohn Forte 79fcf3ce44SJohn Forte /* If icp is found return it */ 80fcf3ce44SJohn Forte if (icp != NULL) { 81fcf3ce44SJohn Forte *icpp = icp; 82fcf3ce44SJohn Forte return (ISCSI_STATUS_SUCCESS); 83fcf3ce44SJohn Forte } 84fcf3ce44SJohn Forte 85fcf3ce44SJohn Forte /* We are creating the connection, allocate, and setup */ 86fcf3ce44SJohn Forte icp = (iscsi_conn_t *)kmem_zalloc(sizeof (iscsi_conn_t), KM_SLEEP); 87fcf3ce44SJohn Forte 88fcf3ce44SJohn Forte /* 89fcf3ce44SJohn Forte * Setup connection 90fcf3ce44SJohn Forte */ 91fcf3ce44SJohn Forte icp->conn_sig = ISCSI_SIG_CONN; 92fcf3ce44SJohn Forte icp->conn_state = ISCSI_CONN_STATE_FREE; 93fcf3ce44SJohn Forte mutex_init(&icp->conn_state_mutex, NULL, MUTEX_DRIVER, NULL); 94fcf3ce44SJohn Forte cv_init(&icp->conn_state_change, NULL, CV_DRIVER, NULL); 9530e7468fSPeter Dunlap mutex_init(&icp->conn_login_mutex, NULL, MUTEX_DRIVER, NULL); 9630e7468fSPeter Dunlap cv_init(&icp->conn_login_cv, NULL, CV_DRIVER, NULL); 97fcf3ce44SJohn Forte icp->conn_state_destroy = B_FALSE; 9830e7468fSPeter Dunlap idm_sm_audit_init(&icp->conn_state_audit); 99fcf3ce44SJohn Forte icp->conn_sess = isp; 100fcf3ce44SJohn Forte 101fcf3ce44SJohn Forte mutex_enter(&iscsi_oid_mutex); 102fcf3ce44SJohn Forte icp->conn_oid = iscsi_oid++; 103fcf3ce44SJohn Forte mutex_exit(&iscsi_oid_mutex); 104fcf3ce44SJohn Forte 10530e7468fSPeter Dunlap /* 10630e7468fSPeter Dunlap * IDM CN taskq 10730e7468fSPeter Dunlap */ 10830e7468fSPeter Dunlap 10930e7468fSPeter Dunlap if (snprintf(th_name, sizeof (th_name) - 1, 11030e7468fSPeter Dunlap ISCSI_CONN_CN_TASKQ_NAME_FORMAT, 111fcf3ce44SJohn Forte icp->conn_sess->sess_hba->hba_oid, icp->conn_sess->sess_oid, 112fcf3ce44SJohn Forte icp->conn_oid) >= sizeof (th_name)) { 113fcf3ce44SJohn Forte cv_destroy(&icp->conn_state_change); 114fcf3ce44SJohn Forte mutex_destroy(&icp->conn_state_mutex); 115fcf3ce44SJohn Forte kmem_free(icp, sizeof (iscsi_conn_t)); 116fcf3ce44SJohn Forte *icpp = NULL; 117fcf3ce44SJohn Forte return (ISCSI_STATUS_INTERNAL_ERROR); 118fcf3ce44SJohn Forte } 119fcf3ce44SJohn Forte 12030e7468fSPeter Dunlap icp->conn_cn_taskq = 12130e7468fSPeter Dunlap ddi_taskq_create(icp->conn_sess->sess_hba->hba_dip, th_name, 1, 12230e7468fSPeter Dunlap TASKQ_DEFAULTPRI, 0); 12330e7468fSPeter Dunlap if (icp->conn_cn_taskq == NULL) { 12430e7468fSPeter Dunlap cv_destroy(&icp->conn_state_change); 12530e7468fSPeter Dunlap mutex_destroy(&icp->conn_state_mutex); 12630e7468fSPeter Dunlap kmem_free(icp, sizeof (iscsi_conn_t)); 12730e7468fSPeter Dunlap *icpp = NULL; 12830e7468fSPeter Dunlap return (ISCSI_STATUS_INTERNAL_ERROR); 12930e7468fSPeter Dunlap } 130fcf3ce44SJohn Forte 131fcf3ce44SJohn Forte /* Creation of the transfer thread */ 132fcf3ce44SJohn Forte if (snprintf(th_name, sizeof (th_name) - 1, ISCSI_CONN_TXTH_NAME_FORMAT, 133fcf3ce44SJohn Forte icp->conn_sess->sess_hba->hba_oid, icp->conn_sess->sess_oid, 134fcf3ce44SJohn Forte icp->conn_oid) >= sizeof (th_name)) { 135fcf3ce44SJohn Forte cv_destroy(&icp->conn_state_change); 136fcf3ce44SJohn Forte mutex_destroy(&icp->conn_state_mutex); 137fcf3ce44SJohn Forte kmem_free(icp, sizeof (iscsi_conn_t)); 13830e7468fSPeter Dunlap ddi_taskq_destroy(icp->conn_cn_taskq); 139fcf3ce44SJohn Forte *icpp = NULL; 140fcf3ce44SJohn Forte return (ISCSI_STATUS_INTERNAL_ERROR); 141fcf3ce44SJohn Forte } 142fcf3ce44SJohn Forte 143fcf3ce44SJohn Forte icp->conn_tx_thread = iscsi_thread_create(isp->sess_hba->hba_dip, 144fcf3ce44SJohn Forte th_name, iscsi_tx_thread, icp); 145fcf3ce44SJohn Forte 146fcf3ce44SJohn Forte /* setup connection queues */ 147fcf3ce44SJohn Forte iscsi_init_queue(&icp->conn_queue_active); 14830e7468fSPeter Dunlap iscsi_init_queue(&icp->conn_queue_idm_aborting); 149fcf3ce44SJohn Forte 150fcf3ce44SJohn Forte bcopy(addr, &icp->conn_base_addr, sizeof (icp->conn_base_addr)); 151fcf3ce44SJohn Forte 152fcf3ce44SJohn Forte /* Add new connection to the session connection list */ 153fcf3ce44SJohn Forte icp->conn_cid = isp->sess_conn_next_cid++; 154fcf3ce44SJohn Forte if (isp->sess_conn_list == NULL) { 155fcf3ce44SJohn Forte isp->sess_conn_list = isp->sess_conn_list_last_ptr = icp; 156fcf3ce44SJohn Forte } else { 157fcf3ce44SJohn Forte isp->sess_conn_list_last_ptr->conn_next = icp; 158fcf3ce44SJohn Forte isp->sess_conn_list_last_ptr = icp; 159fcf3ce44SJohn Forte } 160fcf3ce44SJohn Forte 161fcf3ce44SJohn Forte KSTAT_INC_SESS_CNTR_CONN(isp); 162fcf3ce44SJohn Forte (void) iscsi_conn_kstat_init(icp); 163fcf3ce44SJohn Forte 164fcf3ce44SJohn Forte *icpp = icp; 165fcf3ce44SJohn Forte 166fcf3ce44SJohn Forte return (ISCSI_STATUS_SUCCESS); 167fcf3ce44SJohn Forte } 168fcf3ce44SJohn Forte 16930e7468fSPeter Dunlap /* 17030e7468fSPeter Dunlap * iscsi_conn_online - This attempts to take a connection from 17130e7468fSPeter Dunlap * ISCSI_CONN_STATE_FREE to ISCSI_CONN_STATE_LOGGED_IN. 17230e7468fSPeter Dunlap */ 17330e7468fSPeter Dunlap iscsi_status_t 17430e7468fSPeter Dunlap iscsi_conn_online(iscsi_conn_t *icp) 17530e7468fSPeter Dunlap { 17630e7468fSPeter Dunlap iscsi_task_t *itp; 17730e7468fSPeter Dunlap iscsi_status_t rval; 17830e7468fSPeter Dunlap 17930e7468fSPeter Dunlap ASSERT(icp != NULL); 18030e7468fSPeter Dunlap ASSERT(mutex_owned(&icp->conn_state_mutex)); 18130e7468fSPeter Dunlap ASSERT(icp->conn_state == ISCSI_CONN_STATE_FREE); 18230e7468fSPeter Dunlap 18330e7468fSPeter Dunlap /* 18430e7468fSPeter Dunlap * If we are attempting to connect then for the purposes of the 18530e7468fSPeter Dunlap * other initiator code we are effectively in ISCSI_CONN_STATE_IN_LOGIN. 18630e7468fSPeter Dunlap */ 18730e7468fSPeter Dunlap iscsi_conn_update_state_locked(icp, ISCSI_CONN_STATE_IN_LOGIN); 18830e7468fSPeter Dunlap mutex_exit(&icp->conn_state_mutex); 18930e7468fSPeter Dunlap 19030e7468fSPeter Dunlap /* 19130e7468fSPeter Dunlap * Sync base connection information before login 19230e7468fSPeter Dunlap * A login redirection might have shifted the 19330e7468fSPeter Dunlap * current information from the base. 19430e7468fSPeter Dunlap */ 19530e7468fSPeter Dunlap bcopy(&icp->conn_base_addr, &icp->conn_curr_addr, 19630e7468fSPeter Dunlap sizeof (icp->conn_curr_addr)); 19730e7468fSPeter Dunlap 19830e7468fSPeter Dunlap itp = kmem_zalloc(sizeof (iscsi_task_t), KM_SLEEP); 19930e7468fSPeter Dunlap ASSERT(itp != NULL); 20030e7468fSPeter Dunlap 20130e7468fSPeter Dunlap itp->t_arg = icp; 20230e7468fSPeter Dunlap itp->t_blocking = B_TRUE; 20330e7468fSPeter Dunlap rval = iscsi_login_start(itp); 20430e7468fSPeter Dunlap kmem_free(itp, sizeof (iscsi_task_t)); 20530e7468fSPeter Dunlap 20630e7468fSPeter Dunlap mutex_enter(&icp->conn_state_mutex); 20730e7468fSPeter Dunlap 20830e7468fSPeter Dunlap return (rval); 20930e7468fSPeter Dunlap } 210fcf3ce44SJohn Forte 211fcf3ce44SJohn Forte /* 212fcf3ce44SJohn Forte * iscsi_conn_offline - This attempts to take a connection from 213fcf3ce44SJohn Forte * any state to ISCSI_CONN_STATE_FREE. 214fcf3ce44SJohn Forte */ 215fcf3ce44SJohn Forte iscsi_status_t 216fcf3ce44SJohn Forte iscsi_conn_offline(iscsi_conn_t *icp) 217fcf3ce44SJohn Forte { 218fcf3ce44SJohn Forte clock_t delay; 219fcf3ce44SJohn Forte 220fcf3ce44SJohn Forte ASSERT(icp != NULL); 221fcf3ce44SJohn Forte 222fcf3ce44SJohn Forte /* 223fcf3ce44SJohn Forte * We can only destroy a connection if its either in 224fcf3ce44SJohn Forte * a state of FREE or LOGGED. The other states are 225fcf3ce44SJohn Forte * transitionary and its unsafe to perform actions 226fcf3ce44SJohn Forte * on the connection in those states. Set a flag 227fcf3ce44SJohn Forte * on the connection to influence the transitions 228fcf3ce44SJohn Forte * to quickly complete. Then wait for a state 229fcf3ce44SJohn Forte * transition. 23030e7468fSPeter Dunlap * 23130e7468fSPeter Dunlap * ISCSI_CONN_STATE_LOGGED_IN is set immediately at the 23230e7468fSPeter Dunlap * start of CN_NOTIFY_FFP processing. icp->conn_state_ffp 23330e7468fSPeter Dunlap * is set to true at the end of ffp processing, at which 23430e7468fSPeter Dunlap * point any session updates are complete. We don't 23530e7468fSPeter Dunlap * want to start offlining the connection before we're 23630e7468fSPeter Dunlap * done completing the FFP processing since this might 23730e7468fSPeter Dunlap * interrupt the discovery process. 238fcf3ce44SJohn Forte */ 239fcf3ce44SJohn Forte delay = ddi_get_lbolt() + SEC_TO_TICK(SHUTDOWN_TIMEOUT); 240fcf3ce44SJohn Forte mutex_enter(&icp->conn_state_mutex); 241fcf3ce44SJohn Forte icp->conn_state_destroy = B_TRUE; 24230e7468fSPeter Dunlap while ((((icp->conn_state != ISCSI_CONN_STATE_FREE) && 24330e7468fSPeter Dunlap (icp->conn_state != ISCSI_CONN_STATE_LOGGED_IN)) || 24430e7468fSPeter Dunlap ((icp->conn_state == ISCSI_CONN_STATE_LOGGED_IN) && 24530e7468fSPeter Dunlap !icp->conn_state_ffp)) && 246fcf3ce44SJohn Forte (ddi_get_lbolt() < delay)) { 247fcf3ce44SJohn Forte /* wait for transition */ 248fcf3ce44SJohn Forte (void) cv_timedwait(&icp->conn_state_change, 249fcf3ce44SJohn Forte &icp->conn_state_mutex, delay); 250fcf3ce44SJohn Forte } 251fcf3ce44SJohn Forte 252fcf3ce44SJohn Forte switch (icp->conn_state) { 253fcf3ce44SJohn Forte case ISCSI_CONN_STATE_FREE: 254fcf3ce44SJohn Forte break; 255fcf3ce44SJohn Forte case ISCSI_CONN_STATE_LOGGED_IN: 25630e7468fSPeter Dunlap if (icp->conn_state_ffp) 25730e7468fSPeter Dunlap (void) iscsi_handle_logout(icp); 25830e7468fSPeter Dunlap else { 25930e7468fSPeter Dunlap icp->conn_state_destroy = B_FALSE; 26030e7468fSPeter Dunlap mutex_exit(&icp->conn_state_mutex); 26130e7468fSPeter Dunlap return (ISCSI_STATUS_INTERNAL_ERROR); 26230e7468fSPeter Dunlap } 263fcf3ce44SJohn Forte break; 264fcf3ce44SJohn Forte case ISCSI_CONN_STATE_IN_LOGIN: 265fcf3ce44SJohn Forte case ISCSI_CONN_STATE_IN_LOGOUT: 266fcf3ce44SJohn Forte case ISCSI_CONN_STATE_FAILED: 267fcf3ce44SJohn Forte case ISCSI_CONN_STATE_POLLING: 268fcf3ce44SJohn Forte default: 269fcf3ce44SJohn Forte icp->conn_state_destroy = B_FALSE; 270fcf3ce44SJohn Forte mutex_exit(&icp->conn_state_mutex); 271fcf3ce44SJohn Forte return (ISCSI_STATUS_INTERNAL_ERROR); 272fcf3ce44SJohn Forte } 273fcf3ce44SJohn Forte mutex_exit(&icp->conn_state_mutex); 274fcf3ce44SJohn Forte 275fcf3ce44SJohn Forte return (ISCSI_STATUS_SUCCESS); 276fcf3ce44SJohn Forte } 277fcf3ce44SJohn Forte 278fcf3ce44SJohn Forte /* 279fcf3ce44SJohn Forte * iscsi_conn_destroy - This destroys an iscsi connection structure 280fcf3ce44SJohn Forte * and de-associates it with the session. The connection should 281fcf3ce44SJohn Forte * already been in the ISCSI_CONN_STATE_FREE when attempting this 282fcf3ce44SJohn Forte * operation. 283fcf3ce44SJohn Forte */ 284fcf3ce44SJohn Forte iscsi_status_t 285fcf3ce44SJohn Forte iscsi_conn_destroy(iscsi_conn_t *icp) 286fcf3ce44SJohn Forte { 287fcf3ce44SJohn Forte iscsi_sess_t *isp; 288fcf3ce44SJohn Forte iscsi_conn_t *t_icp; 289fcf3ce44SJohn Forte 290fcf3ce44SJohn Forte ASSERT(icp != NULL); 291fcf3ce44SJohn Forte isp = icp->conn_sess; 292fcf3ce44SJohn Forte ASSERT(isp != NULL); 293fcf3ce44SJohn Forte 294fcf3ce44SJohn Forte if (icp->conn_state != ISCSI_CONN_STATE_FREE) { 295fcf3ce44SJohn Forte return (ISCSI_STATUS_INTERNAL_ERROR); 296fcf3ce44SJohn Forte } 297fcf3ce44SJohn Forte 298fcf3ce44SJohn Forte /* Destroy transfer thread */ 299fcf3ce44SJohn Forte iscsi_thread_destroy(icp->conn_tx_thread); 30030e7468fSPeter Dunlap ddi_taskq_destroy(icp->conn_cn_taskq); 301fcf3ce44SJohn Forte 302fcf3ce44SJohn Forte /* Terminate connection queues */ 30330e7468fSPeter Dunlap iscsi_destroy_queue(&icp->conn_queue_idm_aborting); 304fcf3ce44SJohn Forte iscsi_destroy_queue(&icp->conn_queue_active); 305fcf3ce44SJohn Forte 30630e7468fSPeter Dunlap cv_destroy(&icp->conn_login_cv); 30730e7468fSPeter Dunlap mutex_destroy(&icp->conn_login_mutex); 308fcf3ce44SJohn Forte cv_destroy(&icp->conn_state_change); 309fcf3ce44SJohn Forte mutex_destroy(&icp->conn_state_mutex); 310fcf3ce44SJohn Forte 311fcf3ce44SJohn Forte /* 312fcf3ce44SJohn Forte * Remove connection from sessions linked list. 313fcf3ce44SJohn Forte */ 314fcf3ce44SJohn Forte if (isp->sess_conn_list == icp) { 315fcf3ce44SJohn Forte /* connection first item in list */ 316fcf3ce44SJohn Forte isp->sess_conn_list = icp->conn_next; 317fcf3ce44SJohn Forte /* 318fcf3ce44SJohn Forte * check if this is also the last item in the list 319fcf3ce44SJohn Forte */ 320fcf3ce44SJohn Forte if (isp->sess_conn_list_last_ptr == icp) { 321fcf3ce44SJohn Forte isp->sess_conn_list_last_ptr = NULL; 322fcf3ce44SJohn Forte } 323fcf3ce44SJohn Forte } else { 324fcf3ce44SJohn Forte /* 325fcf3ce44SJohn Forte * search session list for icp pointing 326fcf3ce44SJohn Forte * to connection being removed. Then 327fcf3ce44SJohn Forte * update that connections next pointer. 328fcf3ce44SJohn Forte */ 329fcf3ce44SJohn Forte t_icp = isp->sess_conn_list; 330fcf3ce44SJohn Forte while (t_icp->conn_next != NULL) { 331fcf3ce44SJohn Forte if (t_icp->conn_next == icp) { 332fcf3ce44SJohn Forte break; 333fcf3ce44SJohn Forte } 334fcf3ce44SJohn Forte t_icp = t_icp->conn_next; 335fcf3ce44SJohn Forte } 336fcf3ce44SJohn Forte if (t_icp->conn_next == icp) { 337fcf3ce44SJohn Forte t_icp->conn_next = icp->conn_next; 338fcf3ce44SJohn Forte /* 339fcf3ce44SJohn Forte * if this is the last connection in the list 340fcf3ce44SJohn Forte * update the last_ptr to point to t_icp 341fcf3ce44SJohn Forte */ 342fcf3ce44SJohn Forte if (isp->sess_conn_list_last_ptr == icp) { 343fcf3ce44SJohn Forte isp->sess_conn_list_last_ptr = t_icp; 344fcf3ce44SJohn Forte } 345fcf3ce44SJohn Forte } else { 346fcf3ce44SJohn Forte /* couldn't find session */ 347fcf3ce44SJohn Forte ASSERT(FALSE); 348fcf3ce44SJohn Forte } 349fcf3ce44SJohn Forte } 350fcf3ce44SJohn Forte 351fcf3ce44SJohn Forte /* Free this Connections Data */ 352fcf3ce44SJohn Forte iscsi_conn_kstat_term(icp); 353fcf3ce44SJohn Forte kmem_free(icp, sizeof (iscsi_conn_t)); 354fcf3ce44SJohn Forte 355fcf3ce44SJohn Forte return (ISCSI_STATUS_SUCCESS); 356fcf3ce44SJohn Forte } 357fcf3ce44SJohn Forte 358fcf3ce44SJohn Forte 359fcf3ce44SJohn Forte /* 360fcf3ce44SJohn Forte * iscsi_conn_set_login_min_max - set min/max login window 361fcf3ce44SJohn Forte * 362fcf3ce44SJohn Forte * Used to set the min and max login window. Input values 363fcf3ce44SJohn Forte * are in seconds. 364fcf3ce44SJohn Forte */ 365fcf3ce44SJohn Forte void 366fcf3ce44SJohn Forte iscsi_conn_set_login_min_max(iscsi_conn_t *icp, int min, int max) 367fcf3ce44SJohn Forte { 368fcf3ce44SJohn Forte ASSERT(icp != NULL); 369fcf3ce44SJohn Forte 370fcf3ce44SJohn Forte icp->conn_login_min = ddi_get_lbolt() + SEC_TO_TICK(min); 371fcf3ce44SJohn Forte icp->conn_login_max = ddi_get_lbolt() + SEC_TO_TICK(max); 372fcf3ce44SJohn Forte } 373fcf3ce44SJohn Forte 374fcf3ce44SJohn Forte 37530e7468fSPeter Dunlap /* 37630e7468fSPeter Dunlap * Process the idm notifications 37730e7468fSPeter Dunlap */ 37830e7468fSPeter Dunlap idm_status_t 37930e7468fSPeter Dunlap iscsi_client_notify(idm_conn_t *ic, idm_client_notify_t icn, uintptr_t data) 38030e7468fSPeter Dunlap { 38130e7468fSPeter Dunlap iscsi_cn_task_t *cn; 38230e7468fSPeter Dunlap iscsi_conn_t *icp = ic->ic_handle; 38330e7468fSPeter Dunlap iscsi_sess_t *isp; 384fcf3ce44SJohn Forte 385fcf3ce44SJohn Forte /* 38630e7468fSPeter Dunlap * Don't access icp if the notification is CN_CONNECT_DESTROY 38730e7468fSPeter Dunlap * since icp may have already been freed. 388fcf3ce44SJohn Forte * 38930e7468fSPeter Dunlap * Handle CN_FFP_ENABLED and CN_CONNECT_DESTROY immediately 390fcf3ce44SJohn Forte */ 39130e7468fSPeter Dunlap switch (icn) { 39230e7468fSPeter Dunlap case CN_CONNECT_FAIL: 39330e7468fSPeter Dunlap case CN_LOGIN_FAIL: 39430e7468fSPeter Dunlap /* 39530e7468fSPeter Dunlap * Wakeup any thread waiting for login stuff to happen. 39630e7468fSPeter Dunlap */ 39730e7468fSPeter Dunlap ASSERT(icp != NULL); 39830e7468fSPeter Dunlap iscsi_login_update_state(icp, LOGIN_ERROR); 39930e7468fSPeter Dunlap return (IDM_STATUS_SUCCESS); 40030e7468fSPeter Dunlap case CN_READY_FOR_LOGIN: 40130e7468fSPeter Dunlap idm_conn_hold(ic); /* Released in CN_CONNECT_LOST */ 40230e7468fSPeter Dunlap mutex_enter(&icp->conn_state_mutex); 40330e7468fSPeter Dunlap icp->conn_state_idm_connected = B_TRUE; 40430e7468fSPeter Dunlap cv_broadcast(&icp->conn_state_change); 40530e7468fSPeter Dunlap mutex_exit(&icp->conn_state_mutex); 40630e7468fSPeter Dunlap 40730e7468fSPeter Dunlap iscsi_login_update_state(icp, LOGIN_READY); 40830e7468fSPeter Dunlap return (IDM_STATUS_SUCCESS); 40930e7468fSPeter Dunlap case CN_CONNECT_DESTROY: 41030e7468fSPeter Dunlap /* 41130e7468fSPeter Dunlap * We released any dependecies we had on this object in 41230e7468fSPeter Dunlap * either CN_LOGIN_FAIL or CN_CONNECT_LOST so we just need 41330e7468fSPeter Dunlap * to destroy the IDM connection now. 41430e7468fSPeter Dunlap */ 41530e7468fSPeter Dunlap idm_ini_conn_destroy(ic); 41630e7468fSPeter Dunlap return (IDM_STATUS_SUCCESS); 41730e7468fSPeter Dunlap } 418fcf3ce44SJohn Forte 419fcf3ce44SJohn Forte ASSERT(icp != NULL); 42030e7468fSPeter Dunlap isp = icp->conn_sess; 421fcf3ce44SJohn Forte 422fcf3ce44SJohn Forte /* 42330e7468fSPeter Dunlap * Dispatch notifications to the taskq since they often require 42430e7468fSPeter Dunlap * long blocking operations. In the case of CN_CONNECT_DESTROY 42530e7468fSPeter Dunlap * we actually just want to destroy the connection which we 42630e7468fSPeter Dunlap * can't do in the IDM taskq context. 427fcf3ce44SJohn Forte */ 42830e7468fSPeter Dunlap cn = kmem_alloc(sizeof (*cn), KM_SLEEP); 42930e7468fSPeter Dunlap 43030e7468fSPeter Dunlap cn->ct_ic = ic; 43130e7468fSPeter Dunlap cn->ct_icn = icn; 43230e7468fSPeter Dunlap cn->ct_data = data; 43330e7468fSPeter Dunlap 43430e7468fSPeter Dunlap idm_conn_hold(ic); 43530e7468fSPeter Dunlap 43630e7468fSPeter Dunlap if (ddi_taskq_dispatch(icp->conn_cn_taskq, 43730e7468fSPeter Dunlap iscsi_client_notify_task, cn, DDI_SLEEP) != DDI_SUCCESS) { 43830e7468fSPeter Dunlap idm_conn_rele(ic); 43930e7468fSPeter Dunlap cmn_err(CE_WARN, "iscsi connection(%u) failure - " 44030e7468fSPeter Dunlap "unable to schedule notify task", icp->conn_oid); 44130e7468fSPeter Dunlap iscsi_conn_update_state(icp, ISCSI_CONN_STATE_FREE); 44230e7468fSPeter Dunlap mutex_enter(&isp->sess_state_mutex); 44330e7468fSPeter Dunlap iscsi_sess_state_machine(isp, 44430e7468fSPeter Dunlap ISCSI_SESS_EVENT_N6); 44530e7468fSPeter Dunlap mutex_exit(&isp->sess_state_mutex); 44630e7468fSPeter Dunlap } 44730e7468fSPeter Dunlap 44830e7468fSPeter Dunlap return (IDM_STATUS_SUCCESS); 44930e7468fSPeter Dunlap } 45030e7468fSPeter Dunlap 45130e7468fSPeter Dunlap static void 45230e7468fSPeter Dunlap iscsi_client_notify_task(void *cn_task_void) 453fcf3ce44SJohn Forte { 45430e7468fSPeter Dunlap iscsi_cn_task_t *cn_task = cn_task_void; 45530e7468fSPeter Dunlap iscsi_conn_t *icp; 45630e7468fSPeter Dunlap iscsi_sess_t *isp; 45730e7468fSPeter Dunlap idm_conn_t *ic; 45830e7468fSPeter Dunlap idm_client_notify_t icn; 45930e7468fSPeter Dunlap uintptr_t data; 46030e7468fSPeter Dunlap idm_ffp_disable_t disable_type; 46130e7468fSPeter Dunlap boolean_t in_login; 46230e7468fSPeter Dunlap 46330e7468fSPeter Dunlap ic = cn_task->ct_ic; 46430e7468fSPeter Dunlap icn = cn_task->ct_icn; 46530e7468fSPeter Dunlap data = cn_task->ct_data; 46630e7468fSPeter Dunlap 46730e7468fSPeter Dunlap icp = ic->ic_handle; 46830e7468fSPeter Dunlap ASSERT(icp != NULL); 46930e7468fSPeter Dunlap isp = icp->conn_sess; 47030e7468fSPeter Dunlap 47130e7468fSPeter Dunlap switch (icn) { 47230e7468fSPeter Dunlap case CN_FFP_ENABLED: 47330e7468fSPeter Dunlap mutex_enter(&icp->conn_state_mutex); 47430e7468fSPeter Dunlap icp->conn_async_logout = B_FALSE; 47530e7468fSPeter Dunlap icp->conn_state_ffp = B_TRUE; 47630e7468fSPeter Dunlap cv_broadcast(&icp->conn_state_change); 47730e7468fSPeter Dunlap mutex_exit(&icp->conn_state_mutex); 47830e7468fSPeter Dunlap 47930e7468fSPeter Dunlap /* 48030e7468fSPeter Dunlap * This logic assumes that the IDM login-snooping code 48130e7468fSPeter Dunlap * and the initiator login code will agree on whether 48230e7468fSPeter Dunlap * the connection is in FFP. The reason we do this 48330e7468fSPeter Dunlap * is that we don't want to process CN_FFP_DISABLED until 48430e7468fSPeter Dunlap * CN_FFP_ENABLED has been full handled. 48530e7468fSPeter Dunlap */ 48630e7468fSPeter Dunlap mutex_enter(&icp->conn_login_mutex); 48730e7468fSPeter Dunlap while (icp->conn_login_state != LOGIN_FFP) { 48830e7468fSPeter Dunlap cv_wait(&icp->conn_login_cv, &icp->conn_login_mutex); 48930e7468fSPeter Dunlap } 49030e7468fSPeter Dunlap mutex_exit(&icp->conn_login_mutex); 49130e7468fSPeter Dunlap break; 49230e7468fSPeter Dunlap case CN_FFP_DISABLED: 49330e7468fSPeter Dunlap disable_type = (idm_ffp_disable_t)data; 49430e7468fSPeter Dunlap 49530e7468fSPeter Dunlap mutex_enter(&icp->conn_state_mutex); 49630e7468fSPeter Dunlap switch (disable_type) { 49730e7468fSPeter Dunlap case FD_SESS_LOGOUT: 49830e7468fSPeter Dunlap case FD_CONN_LOGOUT: 49930e7468fSPeter Dunlap if (icp->conn_async_logout) { 50030e7468fSPeter Dunlap /* 50130e7468fSPeter Dunlap * Our logout was in response to an 50230e7468fSPeter Dunlap * async logout request so treat this 50330e7468fSPeter Dunlap * like a connection failure (we will 50430e7468fSPeter Dunlap * try to re-establish the connection) 50530e7468fSPeter Dunlap */ 50630e7468fSPeter Dunlap iscsi_conn_update_state_locked(icp, 50730e7468fSPeter Dunlap ISCSI_CONN_STATE_FAILED); 50830e7468fSPeter Dunlap } else { 50930e7468fSPeter Dunlap /* 51030e7468fSPeter Dunlap * Logout due to to user config change, 51130e7468fSPeter Dunlap * we will not try to re-establish 51230e7468fSPeter Dunlap * the connection. 51330e7468fSPeter Dunlap */ 51430e7468fSPeter Dunlap iscsi_conn_update_state_locked(icp, 51530e7468fSPeter Dunlap ISCSI_CONN_STATE_IN_LOGOUT); 51630e7468fSPeter Dunlap /* 51730e7468fSPeter Dunlap * Hold off generating the ISCSI_SESS_EVENT_N3 51830e7468fSPeter Dunlap * event until we get the CN_CONNECT_LOST 51930e7468fSPeter Dunlap * notification. This matches the pre-IDM 52030e7468fSPeter Dunlap * implementation better. 52130e7468fSPeter Dunlap */ 52230e7468fSPeter Dunlap } 52330e7468fSPeter Dunlap break; 52430e7468fSPeter Dunlap 52530e7468fSPeter Dunlap case FD_CONN_FAIL: 526fcf3ce44SJohn Forte default: 52730e7468fSPeter Dunlap iscsi_conn_update_state_locked(icp, 52830e7468fSPeter Dunlap ISCSI_CONN_STATE_FAILED); 52930e7468fSPeter Dunlap break; 53030e7468fSPeter Dunlap } 53130e7468fSPeter Dunlap 53230e7468fSPeter Dunlap icp->conn_state_ffp = B_FALSE; 53330e7468fSPeter Dunlap cv_broadcast(&icp->conn_state_change); 53430e7468fSPeter Dunlap mutex_exit(&icp->conn_state_mutex); 53530e7468fSPeter Dunlap 53630e7468fSPeter Dunlap break; 53730e7468fSPeter Dunlap case CN_CONNECT_LOST: 53830e7468fSPeter Dunlap /* 53930e7468fSPeter Dunlap * We only care about CN_CONNECT_LOST if we've logged in. IDM 54030e7468fSPeter Dunlap * sends a flag as the data payload to indicate whether we 54130e7468fSPeter Dunlap * were trying to login. The CN_LOGIN_FAIL notification 54230e7468fSPeter Dunlap * gives us what we need to know for login failures and 54330e7468fSPeter Dunlap * otherwise we will need to keep a bunch of state to know 54430e7468fSPeter Dunlap * what CN_CONNECT_LOST means to us. 54530e7468fSPeter Dunlap */ 54630e7468fSPeter Dunlap in_login = (boolean_t)data; 54730e7468fSPeter Dunlap if (in_login) { 54830e7468fSPeter Dunlap mutex_enter(&icp->conn_state_mutex); 54930e7468fSPeter Dunlap 55030e7468fSPeter Dunlap icp->conn_state_idm_connected = B_FALSE; 55130e7468fSPeter Dunlap cv_broadcast(&icp->conn_state_change); 55230e7468fSPeter Dunlap mutex_exit(&icp->conn_state_mutex); 55330e7468fSPeter Dunlap 55430e7468fSPeter Dunlap /* Release connect hold from CN_READY_FOR_LOGIN */ 55530e7468fSPeter Dunlap idm_conn_rele(ic); 55630e7468fSPeter Dunlap break; 55730e7468fSPeter Dunlap } 55830e7468fSPeter Dunlap 55930e7468fSPeter Dunlap /* Any remaining commands are never going to finish */ 56030e7468fSPeter Dunlap iscsi_conn_flush_active_cmds(icp); 56130e7468fSPeter Dunlap 56230e7468fSPeter Dunlap /* 56330e7468fSPeter Dunlap * The connection is no longer active so cleanup any 56430e7468fSPeter Dunlap * references to the connection and release any holds so 56530e7468fSPeter Dunlap * that IDM can finish cleanup. 56630e7468fSPeter Dunlap */ 56730e7468fSPeter Dunlap mutex_enter(&icp->conn_state_mutex); 56830e7468fSPeter Dunlap if (icp->conn_state != ISCSI_CONN_STATE_FAILED) { 56930e7468fSPeter Dunlap 57030e7468fSPeter Dunlap mutex_enter(&isp->sess_state_mutex); 57130e7468fSPeter Dunlap iscsi_sess_state_machine(isp, ISCSI_SESS_EVENT_N3); 57230e7468fSPeter Dunlap mutex_exit(&isp->sess_state_mutex); 57330e7468fSPeter Dunlap 57430e7468fSPeter Dunlap iscsi_conn_update_state_locked(icp, 57530e7468fSPeter Dunlap ISCSI_CONN_STATE_FREE); 57630e7468fSPeter Dunlap } else { 57730e7468fSPeter Dunlap 57830e7468fSPeter Dunlap mutex_enter(&isp->sess_state_mutex); 57930e7468fSPeter Dunlap iscsi_sess_state_machine(isp, 58030e7468fSPeter Dunlap ISCSI_SESS_EVENT_N5); 58130e7468fSPeter Dunlap mutex_exit(&isp->sess_state_mutex); 58230e7468fSPeter Dunlap 58330e7468fSPeter Dunlap /* 58430e7468fSPeter Dunlap * If session type is NORMAL, try to reestablish the 58530e7468fSPeter Dunlap * connection. 58630e7468fSPeter Dunlap */ 58730e7468fSPeter Dunlap if (isp->sess_type == ISCSI_SESS_TYPE_NORMAL) { 58830e7468fSPeter Dunlap iscsi_conn_retry(isp, icp); 58930e7468fSPeter Dunlap } else { 59030e7468fSPeter Dunlap 59130e7468fSPeter Dunlap mutex_enter(&isp->sess_state_mutex); 59230e7468fSPeter Dunlap iscsi_sess_state_machine(isp, 59330e7468fSPeter Dunlap ISCSI_SESS_EVENT_N6); 59430e7468fSPeter Dunlap mutex_exit(&isp->sess_state_mutex); 59530e7468fSPeter Dunlap 59630e7468fSPeter Dunlap iscsi_conn_update_state_locked(icp, 59730e7468fSPeter Dunlap ISCSI_CONN_STATE_FREE); 598fcf3ce44SJohn Forte } 599fcf3ce44SJohn Forte } 600fcf3ce44SJohn Forte 60130e7468fSPeter Dunlap (void) iscsi_thread_stop(icp->conn_tx_thread); 60230e7468fSPeter Dunlap 60330e7468fSPeter Dunlap icp->conn_state_idm_connected = B_FALSE; 60430e7468fSPeter Dunlap cv_broadcast(&icp->conn_state_change); 60530e7468fSPeter Dunlap mutex_exit(&icp->conn_state_mutex); 60630e7468fSPeter Dunlap 60730e7468fSPeter Dunlap /* Release connect hold from CN_READY_FOR_LOGIN */ 60830e7468fSPeter Dunlap idm_conn_rele(ic); 60930e7468fSPeter Dunlap break; 61030e7468fSPeter Dunlap default: 61130e7468fSPeter Dunlap ISCSI_CONN_LOG(CE_WARN, 61230e7468fSPeter Dunlap "iscsi_client_notify: unknown notification: " 61330e7468fSPeter Dunlap "%x: NOT IMPLEMENTED YET: icp: %p ic: %p ", 61430e7468fSPeter Dunlap icn, (void *)icp, (void *)ic); 61530e7468fSPeter Dunlap break; 61630e7468fSPeter Dunlap } 61730e7468fSPeter Dunlap /* free the task notify structure we allocated in iscsi_client_notify */ 61830e7468fSPeter Dunlap kmem_free(cn_task, sizeof (*cn_task)); 61930e7468fSPeter Dunlap 62030e7468fSPeter Dunlap /* Release the hold we acquired in iscsi_client_notify */ 62130e7468fSPeter Dunlap idm_conn_rele(ic); 62230e7468fSPeter Dunlap } 623fcf3ce44SJohn Forte 624fcf3ce44SJohn Forte /* 625fcf3ce44SJohn Forte * iscsi_conn_sync_params - used to update connection parameters 626fcf3ce44SJohn Forte * 627fcf3ce44SJohn Forte * Used to update connection parameters with current configured 628fcf3ce44SJohn Forte * parameters in the persistent store. This should be called 629fcf3ce44SJohn Forte * before starting to make a new iscsi connection in iscsi_login. 630fcf3ce44SJohn Forte */ 631fcf3ce44SJohn Forte iscsi_status_t 632fcf3ce44SJohn Forte iscsi_conn_sync_params(iscsi_conn_t *icp) 633fcf3ce44SJohn Forte { 634fcf3ce44SJohn Forte iscsi_sess_t *isp; 635fcf3ce44SJohn Forte iscsi_hba_t *ihp; 636fcf3ce44SJohn Forte int param_id; 637fcf3ce44SJohn Forte persistent_param_t pp; 638*aff4bce5Syi zhang - Sun Microsystems - Beijing China persistent_tunable_param_t ptp; 639fcf3ce44SJohn Forte iscsi_config_sess_t *ics; 640fcf3ce44SJohn Forte int idx, size; 641fcf3ce44SJohn Forte char *name; 642fcf3ce44SJohn Forte 643fcf3ce44SJohn Forte ASSERT(icp != NULL); 644fcf3ce44SJohn Forte ASSERT((icp->conn_state == ISCSI_CONN_STATE_IN_LOGIN) || 645fcf3ce44SJohn Forte (icp->conn_state == ISCSI_CONN_STATE_FAILED) || 646fcf3ce44SJohn Forte (icp->conn_state == ISCSI_CONN_STATE_POLLING)); 647fcf3ce44SJohn Forte isp = icp->conn_sess; 648fcf3ce44SJohn Forte ASSERT(isp != NULL); 649fcf3ce44SJohn Forte ihp = isp->sess_hba; 650fcf3ce44SJohn Forte ASSERT(ihp != NULL); 651fcf3ce44SJohn Forte 652fcf3ce44SJohn Forte /* 653fcf3ce44SJohn Forte * Check if someone is trying to destroy this 654fcf3ce44SJohn Forte * connection. If so fail the sync request, 655fcf3ce44SJohn Forte * as a method of fast fail. 656fcf3ce44SJohn Forte */ 657fcf3ce44SJohn Forte if (icp->conn_state_destroy == B_TRUE) { 658fcf3ce44SJohn Forte return (ISCSI_STATUS_SHUTDOWN); 659fcf3ce44SJohn Forte } 660fcf3ce44SJohn Forte 661fcf3ce44SJohn Forte bzero(&pp, sizeof (pp)); 662fcf3ce44SJohn Forte 663fcf3ce44SJohn Forte /* First get a copy of the HBA params */ 664fcf3ce44SJohn Forte bcopy(&ihp->hba_params, &icp->conn_params, 665fcf3ce44SJohn Forte sizeof (iscsi_login_params_t)); 666*aff4bce5Syi zhang - Sun Microsystems - Beijing China bcopy(&ihp->hba_tunable_params, &icp->conn_tunable_params, 667*aff4bce5Syi zhang - Sun Microsystems - Beijing China sizeof (iscsi_tunable_params_t)); 668fcf3ce44SJohn Forte 669fcf3ce44SJohn Forte /* 670fcf3ce44SJohn Forte * Now we need to get the session configured 671fcf3ce44SJohn Forte * values from the persistent store and apply 672fcf3ce44SJohn Forte * them to our connection. 673fcf3ce44SJohn Forte */ 674fcf3ce44SJohn Forte (void) persistent_param_get((char *)isp->sess_name, &pp); 675fcf3ce44SJohn Forte for (param_id = 0; param_id < ISCSI_NUM_LOGIN_PARAM; 676fcf3ce44SJohn Forte param_id++) { 6776cefaae1SJack Meng if (iscsiboot_prop && modrootloaded && 6786cefaae1SJack Meng !iscsi_chk_bootlun_mpxio(ihp) && isp->sess_boot) { 6796cefaae1SJack Meng /* 6806cefaae1SJack Meng * iscsi boot with mpxio disabled 6816cefaae1SJack Meng * while iscsi booting target's parameter overriden 6826cefaae1SJack Meng * do no update target's parameters. 6836cefaae1SJack Meng */ 6846cefaae1SJack Meng if (pp.p_bitmap) { 6856cefaae1SJack Meng cmn_err(CE_NOTE, "Adopting " 6866cefaae1SJack Meng " default login parameters in" 6876cefaae1SJack Meng " boot session as MPxIO is disabled"); 6886cefaae1SJack Meng } 6896cefaae1SJack Meng break; 6906cefaae1SJack Meng } 691fcf3ce44SJohn Forte if (pp.p_bitmap & (1 << param_id)) { 69230e7468fSPeter Dunlap 693fcf3ce44SJohn Forte switch (param_id) { 694fcf3ce44SJohn Forte /* 695fcf3ce44SJohn Forte * Boolean parameters 696fcf3ce44SJohn Forte */ 697fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_DATA_SEQUENCE_IN_ORDER: 698fcf3ce44SJohn Forte icp->conn_params.data_pdu_in_order = 699fcf3ce44SJohn Forte pp.p_params.data_pdu_in_order; 700fcf3ce44SJohn Forte break; 701fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_IMMEDIATE_DATA: 702fcf3ce44SJohn Forte icp->conn_params.immediate_data = 703fcf3ce44SJohn Forte pp.p_params.immediate_data; 704fcf3ce44SJohn Forte break; 705fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_INITIAL_R2T: 706fcf3ce44SJohn Forte icp->conn_params.initial_r2t = 707fcf3ce44SJohn Forte pp.p_params.initial_r2t; 708fcf3ce44SJohn Forte break; 709fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_DATA_PDU_IN_ORDER: 710fcf3ce44SJohn Forte icp->conn_params.data_pdu_in_order = 711fcf3ce44SJohn Forte pp.p_params.data_pdu_in_order; 712fcf3ce44SJohn Forte break; 713fcf3ce44SJohn Forte /* 714fcf3ce44SJohn Forte * Integer parameters 715fcf3ce44SJohn Forte */ 716fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_HEADER_DIGEST: 717fcf3ce44SJohn Forte icp->conn_params.header_digest = 718fcf3ce44SJohn Forte pp.p_params.header_digest; 719fcf3ce44SJohn Forte break; 720fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_DATA_DIGEST: 721fcf3ce44SJohn Forte icp->conn_params.data_digest = 722fcf3ce44SJohn Forte pp.p_params.data_digest; 723fcf3ce44SJohn Forte break; 724fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_DEFAULT_TIME_2_RETAIN: 725fcf3ce44SJohn Forte icp->conn_params.default_time_to_retain = 726fcf3ce44SJohn Forte pp.p_params.default_time_to_retain; 727fcf3ce44SJohn Forte break; 728fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_DEFAULT_TIME_2_WAIT: 729fcf3ce44SJohn Forte icp->conn_params.default_time_to_wait = 730fcf3ce44SJohn Forte pp.p_params.default_time_to_wait; 731fcf3ce44SJohn Forte break; 732fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_MAX_RECV_DATA_SEGMENT_LENGTH: 733fcf3ce44SJohn Forte icp->conn_params.max_recv_data_seg_len = 734fcf3ce44SJohn Forte pp.p_params.max_recv_data_seg_len; 735fcf3ce44SJohn Forte break; 736fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_FIRST_BURST_LENGTH: 737fcf3ce44SJohn Forte icp->conn_params.first_burst_length = 738fcf3ce44SJohn Forte pp.p_params.first_burst_length; 739fcf3ce44SJohn Forte break; 740fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_MAX_BURST_LENGTH: 741fcf3ce44SJohn Forte icp->conn_params.max_burst_length = 742fcf3ce44SJohn Forte pp.p_params.max_burst_length; 743fcf3ce44SJohn Forte break; 744fcf3ce44SJohn Forte 745fcf3ce44SJohn Forte /* 746fcf3ce44SJohn Forte * Integer parameters which currently are unsettable 747fcf3ce44SJohn Forte */ 748fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_MAX_CONNECTIONS: 749fcf3ce44SJohn Forte /* FALLTHRU */ 750fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_OUTSTANDING_R2T: 751fcf3ce44SJohn Forte /* FALLTHRU */ 752fcf3ce44SJohn Forte case ISCSI_LOGIN_PARAM_ERROR_RECOVERY_LEVEL: 753fcf3ce44SJohn Forte /* FALLTHRU */ 754fcf3ce44SJohn Forte default: 755fcf3ce44SJohn Forte break; 756fcf3ce44SJohn Forte } 757fcf3ce44SJohn Forte } 758fcf3ce44SJohn Forte } 759fcf3ce44SJohn Forte 760*aff4bce5Syi zhang - Sun Microsystems - Beijing China if (persistent_get_tunable_param((char *)isp->sess_name, &ptp) == 761*aff4bce5Syi zhang - Sun Microsystems - Beijing China B_TRUE) { 762*aff4bce5Syi zhang - Sun Microsystems - Beijing China if (ptp.p_bitmap & ISCSI_TUNABLE_PARAM_RX_TIMEOUT_VALUE) { 763*aff4bce5Syi zhang - Sun Microsystems - Beijing China icp->conn_tunable_params.recv_login_rsp_timeout = 764*aff4bce5Syi zhang - Sun Microsystems - Beijing China ptp.p_params.recv_login_rsp_timeout; 765*aff4bce5Syi zhang - Sun Microsystems - Beijing China } 766*aff4bce5Syi zhang - Sun Microsystems - Beijing China if (ptp.p_bitmap & ISCSI_TUNABLE_PARAM_CONN_LOGIN_MAX) { 767*aff4bce5Syi zhang - Sun Microsystems - Beijing China icp->conn_tunable_params.conn_login_max = 768*aff4bce5Syi zhang - Sun Microsystems - Beijing China ptp.p_params.conn_login_max; 769*aff4bce5Syi zhang - Sun Microsystems - Beijing China } 770*aff4bce5Syi zhang - Sun Microsystems - Beijing China if (ptp.p_bitmap & ISCSI_TUNABLE_PARAM_LOGIN_POLLING_DELAY) { 771*aff4bce5Syi zhang - Sun Microsystems - Beijing China icp->conn_tunable_params.polling_login_delay = 772*aff4bce5Syi zhang - Sun Microsystems - Beijing China ptp.p_params.polling_login_delay; 773*aff4bce5Syi zhang - Sun Microsystems - Beijing China } 774*aff4bce5Syi zhang - Sun Microsystems - Beijing China } 775*aff4bce5Syi zhang - Sun Microsystems - Beijing China 776fcf3ce44SJohn Forte /* Skip binding checks on discovery sessions */ 777fcf3ce44SJohn Forte if (isp->sess_type == ISCSI_SESS_TYPE_DISCOVERY) { 778fcf3ce44SJohn Forte return (ISCSI_STATUS_SUCCESS); 779fcf3ce44SJohn Forte } 780fcf3ce44SJohn Forte 781fcf3ce44SJohn Forte /* 782fcf3ce44SJohn Forte * Now we need to get the current optional connection 783fcf3ce44SJohn Forte * binding information. 784fcf3ce44SJohn Forte */ 785fcf3ce44SJohn Forte /* setup initial buffer for configured session information */ 786fcf3ce44SJohn Forte size = sizeof (*ics); 787fcf3ce44SJohn Forte ics = kmem_zalloc(size, KM_SLEEP); 788fcf3ce44SJohn Forte ics->ics_in = 1; 789fcf3ce44SJohn Forte 790fcf3ce44SJohn Forte /* get configured sessions information */ 791fcf3ce44SJohn Forte name = (char *)isp->sess_name; 792fcf3ce44SJohn Forte if (persistent_get_config_session(name, ics) == B_FALSE) { 793fcf3ce44SJohn Forte /* 794fcf3ce44SJohn Forte * If we were unable to get target level information 795fcf3ce44SJohn Forte * then check the initiator level information. 796fcf3ce44SJohn Forte */ 797fcf3ce44SJohn Forte name = (char *)isp->sess_hba->hba_name; 798fcf3ce44SJohn Forte if (persistent_get_config_session(name, ics) == B_FALSE) { 799fcf3ce44SJohn Forte /* 800fcf3ce44SJohn Forte * No hba information is found. So assume default 801fcf3ce44SJohn Forte * one session unbound behavior. 802fcf3ce44SJohn Forte */ 803fcf3ce44SJohn Forte ics->ics_out = 1; 804fcf3ce44SJohn Forte ics->ics_bound = B_FALSE; 805fcf3ce44SJohn Forte } 806fcf3ce44SJohn Forte } 807fcf3ce44SJohn Forte 8086cefaae1SJack Meng if (iscsiboot_prop && (ics->ics_out > 1) && isp->sess_boot && 8096cefaae1SJack Meng !iscsi_chk_bootlun_mpxio(ihp)) { 8106cefaae1SJack Meng /* 8116cefaae1SJack Meng * iscsi booting session with mpxio disabled, 8126cefaae1SJack Meng * no need set multiple sessions for booting session 8136cefaae1SJack Meng */ 8146cefaae1SJack Meng ics->ics_out = 1; 8156cefaae1SJack Meng ics->ics_bound = B_FALSE; 8166cefaae1SJack Meng cmn_err(CE_NOTE, "MPxIO is disabled," 8176cefaae1SJack Meng " no need to configure multiple boot sessions"); 8186cefaae1SJack Meng } 8196cefaae1SJack Meng 820fcf3ce44SJohn Forte /* 821fcf3ce44SJohn Forte * Check to make sure this session is still a configured 822fcf3ce44SJohn Forte * session. The user might have decreased the session 823fcf3ce44SJohn Forte * count. (NOTE: byte 5 of the sess_isid is the session 824fcf3ce44SJohn Forte * count (via MS/T). This counter starts at 0.) 825fcf3ce44SJohn Forte */ 8266cefaae1SJack Meng 8276cefaae1SJack Meng 828fcf3ce44SJohn Forte idx = isp->sess_isid[5]; 8296cefaae1SJack Meng 8306cefaae1SJack Meng if (iscsiboot_prop && (idx == ISCSI_MAX_CONFIG_SESSIONS)) { 8316cefaae1SJack Meng /* 8326cefaae1SJack Meng * This is temporary session for boot session propose 8336cefaae1SJack Meng * no need to bound IP for this session 8346cefaae1SJack Meng */ 8356cefaae1SJack Meng icp->conn_bound = B_FALSE; 8366cefaae1SJack Meng kmem_free(ics, sizeof (iscsi_config_sess_t)); 8376cefaae1SJack Meng return (ISCSI_STATUS_SUCCESS); 8386cefaae1SJack Meng } 8396cefaae1SJack Meng 840fcf3ce44SJohn Forte if (ics->ics_out <= idx) { 841fcf3ce44SJohn Forte /* 842fcf3ce44SJohn Forte * No longer a configured session. Return a 843fcf3ce44SJohn Forte * failure so we don't attempt to relogin. 844fcf3ce44SJohn Forte */ 845fcf3ce44SJohn Forte return (ISCSI_STATUS_SHUTDOWN); 846fcf3ce44SJohn Forte } 847fcf3ce44SJohn Forte 848fcf3ce44SJohn Forte /* 849fcf3ce44SJohn Forte * If sessions are unbound set this information on 850fcf3ce44SJohn Forte * the connection and return success. 851fcf3ce44SJohn Forte */ 852fcf3ce44SJohn Forte if (ics->ics_bound == B_FALSE) { 853fcf3ce44SJohn Forte icp->conn_bound = B_FALSE; 854fcf3ce44SJohn Forte kmem_free(ics, sizeof (iscsi_config_sess_t)); 855fcf3ce44SJohn Forte return (ISCSI_STATUS_SUCCESS); 856fcf3ce44SJohn Forte } 857fcf3ce44SJohn Forte 858fcf3ce44SJohn Forte /* 859fcf3ce44SJohn Forte * Since the sessions are bound we need to find the matching 860fcf3ce44SJohn Forte * binding information for the session's isid. If this 861fcf3ce44SJohn Forte * session's isid is > 0 then we need to get more configured 862fcf3ce44SJohn Forte * session information to find the binding info. 863fcf3ce44SJohn Forte */ 864fcf3ce44SJohn Forte if (idx > 0) { 865fcf3ce44SJohn Forte int ics_out; 866fcf3ce44SJohn Forte 867fcf3ce44SJohn Forte ics_out = ics->ics_out; 868fcf3ce44SJohn Forte /* record new size and free last buffer */ 869fcf3ce44SJohn Forte size = ISCSI_SESSION_CONFIG_SIZE(ics_out); 870fcf3ce44SJohn Forte kmem_free(ics, sizeof (*ics)); 871fcf3ce44SJohn Forte 872fcf3ce44SJohn Forte /* allocate new buffer */ 873fcf3ce44SJohn Forte ics = kmem_zalloc(size, KM_SLEEP); 874fcf3ce44SJohn Forte ics->ics_in = ics_out; 875fcf3ce44SJohn Forte 876fcf3ce44SJohn Forte /* get configured sessions information */ 877fcf3ce44SJohn Forte if (persistent_get_config_session(name, ics) != B_TRUE) { 878fcf3ce44SJohn Forte cmn_err(CE_NOTE, "iscsi session(%d) - " 879fcf3ce44SJohn Forte "unable to get configured session information\n", 880fcf3ce44SJohn Forte isp->sess_oid); 881fcf3ce44SJohn Forte kmem_free(ics, size); 882fcf3ce44SJohn Forte return (ISCSI_STATUS_SHUTDOWN); 883fcf3ce44SJohn Forte } 884fcf3ce44SJohn Forte } 885fcf3ce44SJohn Forte 886fcf3ce44SJohn Forte /* Copy correct binding information to the connection */ 887fcf3ce44SJohn Forte icp->conn_bound = B_TRUE; 888fcf3ce44SJohn Forte if (ics->ics_bindings[idx].i_insize == sizeof (struct in_addr)) { 889fcf3ce44SJohn Forte bcopy(&ics->ics_bindings[idx].i_addr.in4, 890fcf3ce44SJohn Forte &icp->conn_bound_addr.sin4.sin_addr.s_addr, 891fcf3ce44SJohn Forte sizeof (struct in_addr)); 8921050fd6dSJames Moore icp->conn_bound_addr.sin4.sin_family = AF_INET; 893fcf3ce44SJohn Forte } else { 894fcf3ce44SJohn Forte bcopy(&ics->ics_bindings[idx].i_addr.in6, 895fcf3ce44SJohn Forte &icp->conn_bound_addr.sin6.sin6_addr.s6_addr, 896fcf3ce44SJohn Forte sizeof (struct in6_addr)); 8971050fd6dSJames Moore icp->conn_bound_addr.sin6.sin6_family = AF_INET6; 898fcf3ce44SJohn Forte } 899fcf3ce44SJohn Forte 900fcf3ce44SJohn Forte kmem_free(ics, size); 901fcf3ce44SJohn Forte 902fcf3ce44SJohn Forte return (ISCSI_STATUS_SUCCESS); 903fcf3ce44SJohn Forte } 904fcf3ce44SJohn Forte 905fcf3ce44SJohn Forte /* 906fcf3ce44SJohn Forte * +--------------------------------------------------------------------+ 907fcf3ce44SJohn Forte * | Internal Connection Interfaces | 908fcf3ce44SJohn Forte * +--------------------------------------------------------------------+ 909fcf3ce44SJohn Forte */ 910fcf3ce44SJohn Forte 911fcf3ce44SJohn Forte /* 912fcf3ce44SJohn Forte * iscsi_conn_flush_active_cmds - flush all active icmdps 913fcf3ce44SJohn Forte * for a connection. 914fcf3ce44SJohn Forte */ 915fcf3ce44SJohn Forte static void 916fcf3ce44SJohn Forte iscsi_conn_flush_active_cmds(iscsi_conn_t *icp) 917fcf3ce44SJohn Forte { 918fcf3ce44SJohn Forte iscsi_cmd_t *icmdp; 919fcf3ce44SJohn Forte iscsi_sess_t *isp; 920fcf3ce44SJohn Forte boolean_t lock_held = B_FALSE; 921fcf3ce44SJohn Forte 922fcf3ce44SJohn Forte ASSERT(icp != NULL); 923fcf3ce44SJohn Forte isp = icp->conn_sess; 924fcf3ce44SJohn Forte ASSERT(isp != NULL); 925fcf3ce44SJohn Forte 926fcf3ce44SJohn Forte if (mutex_owned(&icp->conn_queue_active.mutex)) { 927fcf3ce44SJohn Forte lock_held = B_TRUE; 928fcf3ce44SJohn Forte } else { 929fcf3ce44SJohn Forte mutex_enter(&icp->conn_queue_active.mutex); 930fcf3ce44SJohn Forte } 931fcf3ce44SJohn Forte 932fcf3ce44SJohn Forte /* Flush active queue */ 933fcf3ce44SJohn Forte icmdp = icp->conn_queue_active.head; 934fcf3ce44SJohn Forte while (icmdp != NULL) { 9352b79d384Sbing zhao - Sun Microsystems - Beijing China 9362b79d384Sbing zhao - Sun Microsystems - Beijing China mutex_enter(&icmdp->cmd_mutex); 9372b79d384Sbing zhao - Sun Microsystems - Beijing China if (icmdp->cmd_type == ISCSI_CMD_TYPE_SCSI) { 9382b79d384Sbing zhao - Sun Microsystems - Beijing China icmdp->cmd_un.scsi.pkt_stat |= STAT_ABORTED; 9392b79d384Sbing zhao - Sun Microsystems - Beijing China } 9402b79d384Sbing zhao - Sun Microsystems - Beijing China mutex_exit(&icmdp->cmd_mutex); 9412b79d384Sbing zhao - Sun Microsystems - Beijing China 942fcf3ce44SJohn Forte iscsi_cmd_state_machine(icmdp, 943fcf3ce44SJohn Forte ISCSI_CMD_EVENT_E7, isp); 944fcf3ce44SJohn Forte icmdp = icp->conn_queue_active.head; 945fcf3ce44SJohn Forte } 946fcf3ce44SJohn Forte 94730e7468fSPeter Dunlap /* Wait for active queue to drain */ 94830e7468fSPeter Dunlap while (icp->conn_queue_active.count) { 94930e7468fSPeter Dunlap mutex_exit(&icp->conn_queue_active.mutex); 95030e7468fSPeter Dunlap delay(drv_usectohz(100000)); 95130e7468fSPeter Dunlap mutex_enter(&icp->conn_queue_active.mutex); 95230e7468fSPeter Dunlap } 95330e7468fSPeter Dunlap 954fcf3ce44SJohn Forte if (lock_held == B_FALSE) { 955fcf3ce44SJohn Forte mutex_exit(&icp->conn_queue_active.mutex); 956fcf3ce44SJohn Forte } 95730e7468fSPeter Dunlap 95830e7468fSPeter Dunlap /* Wait for IDM abort queue to drain (if necessary) */ 95930e7468fSPeter Dunlap mutex_enter(&icp->conn_queue_idm_aborting.mutex); 96030e7468fSPeter Dunlap while (icp->conn_queue_idm_aborting.count) { 96130e7468fSPeter Dunlap mutex_exit(&icp->conn_queue_idm_aborting.mutex); 96230e7468fSPeter Dunlap delay(drv_usectohz(100000)); 96330e7468fSPeter Dunlap mutex_enter(&icp->conn_queue_idm_aborting.mutex); 964fcf3ce44SJohn Forte } 96530e7468fSPeter Dunlap mutex_exit(&icp->conn_queue_idm_aborting.mutex); 966fcf3ce44SJohn Forte } 967fcf3ce44SJohn Forte 968fcf3ce44SJohn Forte /* 969fcf3ce44SJohn Forte * iscsi_conn_retry - retry connect/login 970fcf3ce44SJohn Forte */ 97130e7468fSPeter Dunlap void 972fcf3ce44SJohn Forte iscsi_conn_retry(iscsi_sess_t *isp, iscsi_conn_t *icp) 973fcf3ce44SJohn Forte { 974fcf3ce44SJohn Forte iscsi_task_t *itp; 975fcf3ce44SJohn Forte 976fcf3ce44SJohn Forte ASSERT(isp != NULL); 977fcf3ce44SJohn Forte ASSERT(icp != NULL); 978fcf3ce44SJohn Forte 979fcf3ce44SJohn Forte /* set login min/max time values */ 980fcf3ce44SJohn Forte iscsi_conn_set_login_min_max(icp, 981fcf3ce44SJohn Forte ISCSI_CONN_DEFAULT_LOGIN_MIN, 982*aff4bce5Syi zhang - Sun Microsystems - Beijing China icp->conn_tunable_params.conn_login_max); 983fcf3ce44SJohn Forte 98430e7468fSPeter Dunlap ISCSI_CONN_LOG(CE_NOTE, "DEBUG: iscsi_conn_retry: icp: %p icp: %p ", 98530e7468fSPeter Dunlap (void *)icp, 98630e7468fSPeter Dunlap (void *)icp->conn_ic); 98730e7468fSPeter Dunlap 988fcf3ce44SJohn Forte /* 989fcf3ce44SJohn Forte * Sync base connection information before login. 990fcf3ce44SJohn Forte * A login redirection might have shifted the 991fcf3ce44SJohn Forte * current information from the base. 992fcf3ce44SJohn Forte */ 993fcf3ce44SJohn Forte bcopy(&icp->conn_base_addr, &icp->conn_curr_addr, 994fcf3ce44SJohn Forte sizeof (icp->conn_curr_addr)); 995fcf3ce44SJohn Forte 996fcf3ce44SJohn Forte /* schedule login task */ 997fcf3ce44SJohn Forte itp = kmem_zalloc(sizeof (iscsi_task_t), KM_SLEEP); 998fcf3ce44SJohn Forte itp->t_arg = icp; 999fcf3ce44SJohn Forte itp->t_blocking = B_FALSE; 1000fcf3ce44SJohn Forte if (ddi_taskq_dispatch(isp->sess_taskq, 1001fcf3ce44SJohn Forte (void(*)())iscsi_login_start, itp, DDI_SLEEP) != 1002fcf3ce44SJohn Forte DDI_SUCCESS) { 1003fcf3ce44SJohn Forte kmem_free(itp, sizeof (iscsi_task_t)); 100430e7468fSPeter Dunlap cmn_err(CE_WARN, "iscsi connection(%u) failure - " 100530e7468fSPeter Dunlap "unable to schedule login task", icp->conn_oid); 1006fcf3ce44SJohn Forte 100730e7468fSPeter Dunlap iscsi_conn_update_state(icp, ISCSI_CONN_STATE_FREE); 1008fcf3ce44SJohn Forte mutex_enter(&isp->sess_state_mutex); 1009fcf3ce44SJohn Forte iscsi_sess_state_machine(isp, 1010fcf3ce44SJohn Forte ISCSI_SESS_EVENT_N6); 1011fcf3ce44SJohn Forte mutex_exit(&isp->sess_state_mutex); 1012fcf3ce44SJohn Forte } 1013fcf3ce44SJohn Forte } 101430e7468fSPeter Dunlap 101530e7468fSPeter Dunlap void 101630e7468fSPeter Dunlap iscsi_conn_update_state(iscsi_conn_t *icp, iscsi_conn_state_t 101730e7468fSPeter Dunlap next_state) 101830e7468fSPeter Dunlap { 101930e7468fSPeter Dunlap mutex_enter(&icp->conn_state_mutex); 102030e7468fSPeter Dunlap (void) iscsi_conn_update_state_locked(icp, next_state); 102130e7468fSPeter Dunlap mutex_exit(&icp->conn_state_mutex); 102230e7468fSPeter Dunlap } 102330e7468fSPeter Dunlap 102430e7468fSPeter Dunlap void 102530e7468fSPeter Dunlap iscsi_conn_update_state_locked(iscsi_conn_t *icp, 102630e7468fSPeter Dunlap iscsi_conn_state_t next_state) 102730e7468fSPeter Dunlap { 102830e7468fSPeter Dunlap ASSERT(mutex_owned(&icp->conn_state_mutex)); 102930e7468fSPeter Dunlap next_state = (next_state > ISCSI_CONN_STATE_MAX) ? 103030e7468fSPeter Dunlap ISCSI_CONN_STATE_MAX : next_state; 103130e7468fSPeter Dunlap idm_sm_audit_state_change(&icp->conn_state_audit, 103230e7468fSPeter Dunlap SAS_ISCSI_CONN, icp->conn_state, next_state); 103330e7468fSPeter Dunlap switch (next_state) { 103430e7468fSPeter Dunlap case ISCSI_CONN_STATE_FREE: 103530e7468fSPeter Dunlap case ISCSI_CONN_STATE_IN_LOGIN: 103630e7468fSPeter Dunlap case ISCSI_CONN_STATE_LOGGED_IN: 103730e7468fSPeter Dunlap case ISCSI_CONN_STATE_IN_LOGOUT: 103830e7468fSPeter Dunlap case ISCSI_CONN_STATE_FAILED: 103930e7468fSPeter Dunlap case ISCSI_CONN_STATE_POLLING: 104030e7468fSPeter Dunlap ISCSI_CONN_LOG(CE_NOTE, 104130e7468fSPeter Dunlap "iscsi_conn_update_state conn %p %s(%d) -> %s(%d)", 104230e7468fSPeter Dunlap (void *)icp, 104330e7468fSPeter Dunlap iscsi_ics_name[icp->conn_state], icp->conn_state, 104430e7468fSPeter Dunlap iscsi_ics_name[next_state], next_state); 104530e7468fSPeter Dunlap icp->conn_prev_state = icp->conn_state; 104630e7468fSPeter Dunlap icp->conn_state = next_state; 104730e7468fSPeter Dunlap cv_broadcast(&icp->conn_state_change); 104830e7468fSPeter Dunlap break; 104930e7468fSPeter Dunlap default: 105030e7468fSPeter Dunlap cmn_err(CE_WARN, "Update state found illegal state: %x " 105130e7468fSPeter Dunlap "prev_state: %x", next_state, icp->conn_prev_state); 105230e7468fSPeter Dunlap ASSERT(0); 105330e7468fSPeter Dunlap } 105430e7468fSPeter Dunlap } 1055