1 /* 2 * Copyright (c) 2002-2013 Mellanox Technologies LTD. All rights reserved. 3 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 * 35 */ 36 37 /* 38 * Abstract: 39 * Implementation of osm_sm_state_mgr_t. 40 * This file implements the SM State Manager object. 41 */ 42 43 #if HAVE_CONFIG_H 44 # include <config.h> 45 #endif /* HAVE_CONFIG_H */ 46 47 #include <string.h> 48 #include <time.h> 49 #include <iba/ib_types.h> 50 #include <complib/cl_passivelock.h> 51 #include <complib/cl_debug.h> 52 #include <opensm/osm_file_ids.h> 53 #define FILE_ID OSM_FILE_SM_STATE_MGR_C 54 #include <opensm/osm_sm.h> 55 #include <opensm/osm_madw.h> 56 #include <opensm/osm_switch.h> 57 #include <opensm/osm_log.h> 58 #include <opensm/osm_subnet.h> 59 #include <opensm/osm_helper.h> 60 #include <opensm/osm_msgdef.h> 61 #include <opensm/osm_node.h> 62 #include <opensm/osm_port.h> 63 #include <vendor/osm_vendor_api.h> 64 #include <opensm/osm_helper.h> 65 #include <opensm/osm_opensm.h> 66 67 void osm_report_sm_state(osm_sm_t * sm) 68 { 69 char buf[64]; 70 const char *state_str = osm_get_sm_mgr_state_str(sm->p_subn->sm_state); 71 72 osm_log_v2(sm->p_log, OSM_LOG_SYS, FILE_ID, "Entering %s state\n", state_str); 73 snprintf(buf, sizeof(buf), "ENTERING SM %s STATE", state_str); 74 OSM_LOG_MSG_BOX(sm->p_log, OSM_LOG_VERBOSE, buf); 75 } 76 77 static boolean_t sm_state_mgr_send_master_sm_info_req(osm_sm_t * sm, uint8_t sm_state) 78 { 79 osm_madw_context_t context; 80 const osm_port_t *p_port; 81 ib_api_status_t status; 82 osm_dr_path_t dr_path; 83 ib_net64_t guid; 84 boolean_t sent_req = FALSE; 85 86 OSM_LOG_ENTER(sm->p_log); 87 88 memset(&context, 0, sizeof(context)); 89 if (sm_state == IB_SMINFO_STATE_STANDBY) { 90 /* 91 * We are in STANDBY state - this means we need to poll the 92 * master SM (according to master_guid). 93 * Send a query of SubnGet(SMInfo) to the subn 94 * master_sm_base_lid object. 95 */ 96 guid = sm->master_sm_guid; 97 } else { 98 /* 99 * We are not in STANDBY - this means we are in MASTER state - 100 * so we need to poll the SM that is saved in polling_sm_guid 101 * under sm. 102 * Send a query of SubnGet(SMInfo) to that SM. 103 */ 104 guid = sm->polling_sm_guid; 105 } 106 107 /* Verify that SM is not polling itself */ 108 if (guid == sm->p_subn->sm_port_guid) { 109 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, 110 "OpenSM doesn't poll itself\n"); 111 goto Exit; 112 } 113 114 p_port = osm_get_port_by_guid(sm->p_subn, guid); 115 116 if (p_port == NULL) { 117 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3203: " 118 "No port object for GUID 0x%016" PRIx64 "\n", 119 cl_ntoh64(guid)); 120 goto Exit; 121 } 122 123 context.smi_context.port_guid = guid; 124 context.smi_context.set_method = FALSE; 125 memcpy(&dr_path, osm_physp_get_dr_path_ptr(p_port->p_physp), sizeof(osm_dr_path_t)); 126 127 status = osm_req_get(sm, &dr_path, 128 IB_MAD_ATTR_SM_INFO, 0, FALSE, 129 ib_port_info_get_m_key(&p_port->p_physp->port_info), 130 CL_DISP_MSGID_NONE, &context); 131 132 if (status != IB_SUCCESS) 133 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3204: " 134 "Failure requesting SMInfo (%s)\n", 135 ib_get_err_str(status)); 136 else 137 sent_req = TRUE; 138 139 Exit: 140 OSM_LOG_EXIT(sm->p_log); 141 142 return (sent_req); 143 } 144 145 static void sm_state_mgr_start_polling(osm_sm_t * sm) 146 { 147 uint32_t timeout; 148 cl_status_t cl_status; 149 150 OSM_LOG_ENTER(sm->p_log); 151 152 /* 153 * Init the retry_number back to zero - need to restart counting 154 */ 155 sm->retry_number = 0; 156 157 /* 158 * Send a SubnGet(SMInfo) query to the current (or new) master found. 159 */ 160 CL_PLOCK_ACQUIRE(sm->p_lock); 161 timeout = sm->p_subn->opt.sminfo_polling_timeout; 162 sm_state_mgr_send_master_sm_info_req(sm, sm->p_subn->sm_state); 163 CL_PLOCK_RELEASE(sm->p_lock); 164 165 /* 166 * Start a timer that will wake up every sminfo_polling_timeout milliseconds. 167 * The callback of the timer will send a SubnGet(SMInfo) to the Master SM 168 * and restart the timer 169 */ 170 cl_status = cl_timer_start(&sm->polling_timer, timeout); 171 if (cl_status != CL_SUCCESS) 172 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3210: " 173 "Failed to start polling timer\n"); 174 175 OSM_LOG_EXIT(sm->p_log); 176 } 177 178 void osm_sm_state_mgr_polling_callback(IN void *context) 179 { 180 osm_sm_t *sm = context; 181 uint32_t timeout; 182 cl_status_t cl_status; 183 uint8_t sm_state; 184 185 OSM_LOG_ENTER(sm->p_log); 186 187 cl_spinlock_acquire(&sm->state_lock); 188 sm_state = sm->p_subn->sm_state; 189 cl_spinlock_release(&sm->state_lock); 190 191 CL_PLOCK_ACQUIRE(sm->p_lock); 192 timeout = sm->p_subn->opt.sminfo_polling_timeout; 193 194 /* 195 * We can be here in one of two cases: 196 * 1. We are a STANDBY sm polling on the master SM. 197 * 2. We are a MASTER sm, waiting for a handover from a remote master sm. 198 * If we are not in one of these cases - don't need to restart the poller. 199 */ 200 if (!((sm_state == IB_SMINFO_STATE_MASTER && 201 sm->polling_sm_guid != 0) || 202 sm_state == IB_SMINFO_STATE_STANDBY)) { 203 CL_PLOCK_RELEASE(sm->p_lock); 204 goto Exit; 205 } 206 207 /* 208 * If we are a STANDBY sm and the osm_exit_flag is set, then let's 209 * signal the subnet_up. This is relevant for the case of running only 210 * once. In that case - the program is stuck until this signal is 211 * received. In other cases - it is not relevant whether or not the 212 * signal is on - since we are currently in exit flow 213 */ 214 if (sm_state == IB_SMINFO_STATE_STANDBY && osm_exit_flag) { 215 CL_PLOCK_RELEASE(sm->p_lock); 216 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, 217 "Signalling subnet_up_event\n"); 218 cl_event_signal(&sm->subnet_up_event); 219 goto Exit; 220 } 221 222 /* 223 * If retry number reached the max_retry_number in the subnet opt - call 224 * osm_sm_state_mgr_process with signal OSM_SM_SIGNAL_POLLING_TIMEOUT 225 */ 226 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, "SM State %d (%s), Retry number:%d\n", 227 sm->p_subn->sm_state, osm_get_sm_mgr_state_str(sm->p_subn->sm_state), 228 sm->retry_number); 229 230 if (sm->retry_number > sm->p_subn->opt.polling_retry_number) { 231 CL_PLOCK_RELEASE(sm->p_lock); 232 OSM_LOG(sm->p_log, OSM_LOG_DEBUG, 233 "Reached polling_retry_number value in retry_number. " 234 "Go to DISCOVERY state\n"); 235 osm_sm_state_mgr_process(sm, OSM_SM_SIGNAL_POLLING_TIMEOUT); 236 goto Exit; 237 } 238 239 /* Send a SubnGet(SMInfo) request to the remote sm (depends on our state) */ 240 if (sm_state_mgr_send_master_sm_info_req(sm, sm_state)) { 241 /* Request sent, increment the retry number */ 242 sm->retry_number++; 243 } 244 245 CL_PLOCK_RELEASE(sm->p_lock); 246 247 /* restart the timer */ 248 cl_status = cl_timer_start(&sm->polling_timer, timeout); 249 if (cl_status != CL_SUCCESS) 250 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3211: " 251 "Failed to restart polling timer\n"); 252 253 Exit: 254 OSM_LOG_EXIT(sm->p_log); 255 } 256 257 static void sm_state_mgr_signal_error(osm_sm_t * sm, IN osm_sm_signal_t signal) 258 { 259 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3207: " 260 "Invalid signal %s in state %s\n", 261 osm_get_sm_mgr_signal_str(signal), 262 osm_get_sm_mgr_state_str(sm->p_subn->sm_state)); 263 } 264 265 void osm_sm_state_mgr_signal_master_is_alive(osm_sm_t * sm) 266 { 267 OSM_LOG_ENTER(sm->p_log); 268 sm->retry_number = 0; 269 OSM_LOG_EXIT(sm->p_log); 270 } 271 272 ib_api_status_t osm_sm_state_mgr_process(osm_sm_t * sm, 273 IN osm_sm_signal_t signal) 274 { 275 ib_api_status_t status = IB_SUCCESS; 276 277 CL_ASSERT(sm); 278 279 OSM_LOG_ENTER(sm->p_log); 280 281 /* 282 * The state lock prevents many race conditions from screwing 283 * up the state transition process. 284 */ 285 cl_spinlock_acquire(&sm->state_lock); 286 287 OSM_LOG(sm->p_log, OSM_LOG_DEBUG, 288 "Received signal %s in state %s\n", 289 osm_get_sm_mgr_signal_str(signal), 290 osm_get_sm_mgr_state_str(sm->p_subn->sm_state)); 291 292 switch (sm->p_subn->sm_state) { 293 case IB_SMINFO_STATE_DISCOVERING: 294 switch (signal) { 295 case OSM_SM_SIGNAL_DISCOVERY_COMPLETED: 296 /* 297 * Update the state of the SM to MASTER 298 */ 299 /* Turn on the first_time_master_sweep flag */ 300 sm->p_subn->sm_state = IB_SMINFO_STATE_MASTER; 301 osm_report_sm_state(sm); 302 /* 303 * Make sure to set the subnet master_sm_base_lid 304 * to the sm_base_lid value 305 */ 306 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock); 307 sm->p_subn->first_time_master_sweep = TRUE; 308 sm->p_subn->master_sm_base_lid = 309 sm->p_subn->sm_base_lid; 310 CL_PLOCK_RELEASE(sm->p_lock); 311 break; 312 case OSM_SM_SIGNAL_MASTER_OR_HIGHER_SM_DETECTED: 313 /* 314 * Finished all discovery actions - move to STANDBY 315 * start the polling 316 */ 317 sm->p_subn->sm_state = IB_SMINFO_STATE_STANDBY; 318 osm_report_sm_state(sm); 319 /* 320 * Since another SM is doing the LFT config - we should not 321 * ignore the results of it 322 */ 323 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock); 324 sm->p_subn->ignore_existing_lfts = FALSE; 325 CL_PLOCK_RELEASE(sm->p_lock); 326 sm_state_mgr_start_polling(sm); 327 break; 328 case OSM_SM_SIGNAL_HANDOVER: 329 /* 330 * Signal for a new sweep. We need to discover the other SM. 331 * If we already discovered this SM, and got the 332 * HANDOVER - this means the remote SM is of lower priority. 333 * In this case we will stop polling it (since it is a lower 334 * priority SM in STANDBY state). 335 */ 336 osm_sm_signal(sm, OSM_SIGNAL_SWEEP); 337 break; 338 default: 339 sm_state_mgr_signal_error(sm, signal); 340 status = IB_INVALID_PARAMETER; 341 break; 342 } 343 break; 344 345 case IB_SMINFO_STATE_STANDBY: 346 switch (signal) { 347 case OSM_SM_SIGNAL_POLLING_TIMEOUT: 348 case OSM_SM_SIGNAL_DISCOVER: 349 /* 350 * case 1: Polling timeout occured - this means that the Master SM 351 * is no longer alive. 352 * case 2: Got a signal to move to DISCOVERING 353 * Move to DISCOVERING state and start sweeping 354 */ 355 sm->p_subn->sm_state = IB_SMINFO_STATE_DISCOVERING; 356 osm_report_sm_state(sm); 357 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock); 358 sm->p_subn->coming_out_of_standby = TRUE; 359 CL_PLOCK_RELEASE(sm->p_lock); 360 osm_sm_signal(sm, OSM_SIGNAL_SWEEP); 361 break; 362 case OSM_SM_SIGNAL_DISABLE: 363 /* 364 * Update the state to NOT_ACTIVE 365 */ 366 sm->p_subn->sm_state = IB_SMINFO_STATE_NOTACTIVE; 367 osm_report_sm_state(sm); 368 break; 369 case OSM_SM_SIGNAL_HANDOVER: 370 /* 371 * Update the state to MASTER, and start sweeping 372 * OPTIONAL: send ACKNOWLEDGE 373 */ 374 /* Turn on the force_first_time_master_sweep flag */ 375 /* We want full reconfiguration to occur on the first */ 376 /* master sweep of this SM */ 377 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock); 378 /* 379 * Make sure to set the subnet master_sm_base_lid 380 * to the sm_base_lid value 381 */ 382 sm->p_subn->master_sm_base_lid = 383 sm->p_subn->sm_base_lid; 384 385 sm->p_subn->force_first_time_master_sweep = TRUE; 386 CL_PLOCK_RELEASE(sm->p_lock); 387 388 sm->p_subn->sm_state = IB_SMINFO_STATE_MASTER; 389 osm_report_sm_state(sm); 390 osm_sm_signal(sm, OSM_SIGNAL_SWEEP); 391 break; 392 case OSM_SM_SIGNAL_ACKNOWLEDGE: 393 /* 394 * Do nothing - already moved to STANDBY 395 */ 396 break; 397 default: 398 sm_state_mgr_signal_error(sm, signal); 399 status = IB_INVALID_PARAMETER; 400 break; 401 } 402 break; 403 404 case IB_SMINFO_STATE_NOTACTIVE: 405 switch (signal) { 406 case OSM_SM_SIGNAL_STANDBY: 407 /* 408 * Update the state to STANDBY 409 * start the polling 410 */ 411 sm->p_subn->sm_state = IB_SMINFO_STATE_STANDBY; 412 osm_report_sm_state(sm); 413 sm_state_mgr_start_polling(sm); 414 break; 415 default: 416 sm_state_mgr_signal_error(sm, signal); 417 status = IB_INVALID_PARAMETER; 418 break; 419 } 420 break; 421 422 case IB_SMINFO_STATE_MASTER: 423 switch (signal) { 424 case OSM_SM_SIGNAL_POLLING_TIMEOUT: 425 /* 426 * We received a polling timeout - this means that we 427 * waited for a remote master sm to send us a handover, 428 * but didn't get it, and didn't get a response from 429 * that remote sm. 430 * We want to force a heavy sweep - hopefully this 431 * occurred because the remote sm died, and we'll find 432 * this out and configure the subnet after a heavy sweep. 433 * We also want to clear the polling_sm_guid - since 434 * we are done polling on that remote sm - we are 435 * sweeping again. 436 */ 437 case OSM_SM_SIGNAL_HANDOVER: 438 /* 439 * If we received a handover in a master state - then we 440 * want to force a heavy sweep. This means that either 441 * we are in a sweep currently - in this case - no 442 * change, or we are in idle state - since we 443 * recognized a master SM before - so we want to make a 444 * heavy sweep and reconfigure the new subnet. 445 * We also want to clear the polling_sm_guid - since 446 * we are done polling on that remote sm - we got a 447 * handover from it. 448 */ 449 OSM_LOG(sm->p_log, OSM_LOG_VERBOSE, 450 "Forcing heavy sweep. Received signal %s\n", 451 osm_get_sm_mgr_signal_str(signal)); 452 CL_PLOCK_EXCL_ACQUIRE(sm->p_lock); 453 sm->polling_sm_guid = 0; 454 sm->p_subn->force_first_time_master_sweep = TRUE; 455 CL_PLOCK_RELEASE(sm->p_lock); 456 osm_sm_signal(sm, OSM_SIGNAL_SWEEP); 457 break; 458 case OSM_SM_SIGNAL_HANDOVER_SENT: 459 /* 460 * Just sent a HANDOVER signal - move to STANDBY 461 * start the polling 462 */ 463 sm->p_subn->sm_state = IB_SMINFO_STATE_STANDBY; 464 osm_report_sm_state(sm); 465 sm_state_mgr_start_polling(sm); 466 break; 467 case OSM_SM_SIGNAL_WAIT_FOR_HANDOVER: 468 /* 469 * We found a remote master SM, and we are waiting for 470 * it to handover the mastership to us. Need to start 471 * polling that SM, to make sure it is alive, if it 472 * isn't - then we should move back to discovering, 473 * since something must have happened to it. 474 */ 475 sm_state_mgr_start_polling(sm); 476 break; 477 case OSM_SM_SIGNAL_DISCOVER: 478 sm->p_subn->sm_state = IB_SMINFO_STATE_DISCOVERING; 479 osm_report_sm_state(sm); 480 break; 481 default: 482 sm_state_mgr_signal_error(sm, signal); 483 status = IB_INVALID_PARAMETER; 484 break; 485 } 486 break; 487 488 default: 489 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3208: " 490 "Invalid state %s\n", 491 osm_get_sm_mgr_state_str(sm->p_subn->sm_state)); 492 493 } 494 495 cl_spinlock_release(&sm->state_lock); 496 497 OSM_LOG_EXIT(sm->p_log); 498 return status; 499 } 500 501 ib_api_status_t osm_sm_state_mgr_check_legality(osm_sm_t * sm, 502 IN osm_sm_signal_t signal) 503 { 504 ib_api_status_t status = IB_SUCCESS; 505 506 CL_ASSERT(sm); 507 508 OSM_LOG_ENTER(sm->p_log); 509 510 /* 511 * The state lock prevents many race conditions from screwing 512 * up the state transition process. 513 */ 514 cl_spinlock_acquire(&sm->state_lock); 515 516 OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Received signal %s in state %s\n", 517 osm_get_sm_mgr_signal_str(signal), 518 osm_get_sm_mgr_state_str(sm->p_subn->sm_state)); 519 520 switch (sm->p_subn->sm_state) { 521 case IB_SMINFO_STATE_DISCOVERING: 522 switch (signal) { 523 case OSM_SM_SIGNAL_DISCOVERY_COMPLETED: 524 case OSM_SM_SIGNAL_MASTER_OR_HIGHER_SM_DETECTED: 525 case OSM_SM_SIGNAL_HANDOVER: 526 status = IB_SUCCESS; 527 break; 528 default: 529 sm_state_mgr_signal_error(sm, signal); 530 status = IB_INVALID_PARAMETER; 531 break; 532 } 533 break; 534 535 case IB_SMINFO_STATE_STANDBY: 536 switch (signal) { 537 case OSM_SM_SIGNAL_POLLING_TIMEOUT: 538 case OSM_SM_SIGNAL_DISCOVER: 539 case OSM_SM_SIGNAL_DISABLE: 540 case OSM_SM_SIGNAL_HANDOVER: 541 case OSM_SM_SIGNAL_ACKNOWLEDGE: 542 status = IB_SUCCESS; 543 break; 544 default: 545 sm_state_mgr_signal_error(sm, signal); 546 status = IB_INVALID_PARAMETER; 547 break; 548 } 549 break; 550 551 case IB_SMINFO_STATE_NOTACTIVE: 552 switch (signal) { 553 case OSM_SM_SIGNAL_STANDBY: 554 status = IB_SUCCESS; 555 break; 556 default: 557 sm_state_mgr_signal_error(sm, signal); 558 status = IB_INVALID_PARAMETER; 559 break; 560 } 561 break; 562 563 case IB_SMINFO_STATE_MASTER: 564 switch (signal) { 565 case OSM_SM_SIGNAL_HANDOVER: 566 case OSM_SM_SIGNAL_HANDOVER_SENT: 567 status = IB_SUCCESS; 568 break; 569 default: 570 sm_state_mgr_signal_error(sm, signal); 571 status = IB_INVALID_PARAMETER; 572 break; 573 } 574 break; 575 576 default: 577 OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 3209: " 578 "Invalid state %s\n", 579 osm_get_sm_mgr_state_str(sm->p_subn->sm_state)); 580 status = IB_INVALID_PARAMETER; 581 582 } 583 584 cl_spinlock_release(&sm->state_lock); 585 586 OSM_LOG_EXIT(sm->p_log); 587 return status; 588 } 589