1 /* 2 * Copyright (c) 2004-2009 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2011 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 * 35 */ 36 37 /* 38 * Abstract: 39 * Implementation of the P_Key Manager (Partition Manager). 40 * This is part of the OpenSM. 41 */ 42 43 #if HAVE_CONFIG_H 44 # include <config.h> 45 #endif /* HAVE_CONFIG_H */ 46 47 #include <string.h> 48 #include <iba/ib_types.h> 49 #include <complib/cl_qmap.h> 50 #include <complib/cl_debug.h> 51 #include <opensm/osm_file_ids.h> 52 #define FILE_ID OSM_FILE_PKEY_MGR_C 53 #include <opensm/osm_node.h> 54 #include <opensm/osm_switch.h> 55 #include <opensm/osm_partition.h> 56 #include <opensm/osm_opensm.h> 57 58 static void clear_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl, 59 uint16_t pkey_index); 60 61 /* 62 The max number of pkeys/pkey blocks for a physical port is located 63 in a different place for switch external ports (SwitchInfo) and the 64 rest of the ports (NodeInfo). 65 */ 66 static uint16_t 67 pkey_mgr_get_physp_max_pkeys(IN const osm_physp_t * p_physp) 68 { 69 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp); 70 uint16_t num_pkeys = 0; 71 72 if (!p_node->sw || (osm_physp_get_port_num(p_physp) == 0)) 73 num_pkeys = cl_ntoh16(p_node->node_info.partition_cap); 74 else 75 num_pkeys = cl_ntoh16(p_node->sw->switch_info.enforce_cap); 76 return num_pkeys; 77 } 78 79 static uint16_t 80 pkey_mgr_get_physp_max_blocks(IN const osm_physp_t * p_physp) 81 { 82 return ((pkey_mgr_get_physp_max_pkeys(p_physp) + 31) / 32); 83 } 84 85 /* 86 * Insert new pending pkey entry to the specific port pkey table 87 * pending pkeys. New entries are inserted at the back. 88 */ 89 static void 90 pkey_mgr_process_physical_port(IN osm_log_t * p_log, 91 IN osm_sm_t * sm, 92 IN const ib_net16_t pkey, 93 IN osm_physp_t * p_physp) 94 { 95 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp); 96 osm_pkey_tbl_t *p_pkey_tbl; 97 ib_net16_t *p_orig_pkey; 98 osm_pending_pkey_t *p_pending; 99 100 p_pkey_tbl = &p_physp->pkeys; 101 p_pending = (osm_pending_pkey_t *) calloc(1, sizeof(osm_pending_pkey_t)); 102 if (!p_pending) { 103 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0502: " 104 "Failed to allocate new pending pkey entry for node " 105 "0x%016" PRIx64 " port %u\n", 106 cl_ntoh64(osm_node_get_node_guid(p_node)), 107 osm_physp_get_port_num(p_physp)); 108 return; 109 } 110 p_pending->pkey = pkey; 111 if (sm->p_subn->opt.allow_both_pkeys) 112 p_orig_pkey = cl_map_get(&p_pkey_tbl->keys, pkey); 113 else 114 p_orig_pkey = cl_map_get(&p_pkey_tbl->keys, 115 ib_pkey_get_base(pkey)); 116 117 if (!p_orig_pkey) { 118 p_pending->is_new = TRUE; 119 } else { 120 CL_ASSERT(ib_pkey_get_base(*p_orig_pkey) == 121 ib_pkey_get_base(pkey)); 122 p_pending->is_new = FALSE; 123 if (osm_pkey_tbl_get_block_and_idx(p_pkey_tbl, p_orig_pkey, 124 &p_pending->block, 125 &p_pending->index) != 126 IB_SUCCESS) { 127 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0503: " 128 "Failed to obtain P_Key 0x%04x block and index " 129 "for node 0x%016" PRIx64 " port %u\n", 130 cl_ntoh16(ib_pkey_get_base(pkey)), 131 cl_ntoh64(osm_node_get_node_guid(p_node)), 132 osm_physp_get_port_num(p_physp)); 133 free(p_pending); 134 return; 135 } 136 if (p_physp->pkeys.indx0_pkey) { 137 /* 138 * Remove the pkey that should be at index 0 from 139 * accum pkey if current position is not index 0 140 */ 141 if (((sm->p_subn->opt.allow_both_pkeys && 142 pkey == p_physp->pkeys.indx0_pkey) || 143 (!sm->p_subn->opt.allow_both_pkeys && 144 ib_pkey_get_base(pkey) == ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) && 145 (p_pending->block != 0 || p_pending->index != 0)) { 146 p_pending->is_new = TRUE; 147 clear_accum_pkey_index(p_pkey_tbl, 148 p_pending->block * 149 IB_NUM_PKEY_ELEMENTS_IN_BLOCK + 150 p_pending->index); 151 } 152 153 if (p_pending->block == 0 && p_pending->index == 0) { 154 /* Move the pkey away from index 0 */ 155 if ((sm->p_subn->opt.allow_both_pkeys && 156 pkey != p_physp->pkeys.indx0_pkey) || 157 (!sm->p_subn->opt.allow_both_pkeys && 158 ib_pkey_get_base(pkey) != ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) { 159 p_pending->is_new = TRUE; 160 clear_accum_pkey_index(p_pkey_tbl, 0); 161 } 162 } 163 } else { 164 /* If index 0 is occupied by non-default, it should reoccupied by pkey 0x7FFF */ 165 if (p_pending->block == 0 && p_pending->index == 0) { 166 if (ib_pkey_get_base(pkey) != IB_DEFAULT_PARTIAL_PKEY) { 167 p_pending->is_new = TRUE; 168 clear_accum_pkey_index(p_pkey_tbl, 0); 169 } 170 /* Need to move default pkey to index 0 */ 171 } else if ((sm->p_subn->opt.allow_both_pkeys && 172 pkey == IB_DEFAULT_PKEY) || 173 (!sm->p_subn->opt.allow_both_pkeys && 174 ib_pkey_get_base(pkey) == IB_DEFAULT_PARTIAL_PKEY)) { 175 p_pending->is_new = TRUE; 176 clear_accum_pkey_index(p_pkey_tbl, 177 p_pending->block * 178 IB_NUM_PKEY_ELEMENTS_IN_BLOCK + 179 p_pending->index); 180 } 181 } 182 183 } 184 if (p_pending->is_new == TRUE) 185 cl_qlist_insert_tail(&p_pkey_tbl->pending, 186 (cl_list_item_t *) p_pending); 187 else 188 cl_qlist_insert_head(&p_pkey_tbl->pending, 189 (cl_list_item_t *) p_pending); 190 191 OSM_LOG(p_log, OSM_LOG_DEBUG, 192 "pkey 0x%04x was %s for node 0x%016" PRIx64 " port %u\n", 193 cl_ntoh16(pkey), p_pending->is_new ? "inserted" : "updated", 194 cl_ntoh64(osm_node_get_node_guid(p_node)), 195 osm_physp_get_port_num(p_physp)); 196 } 197 198 static void 199 pkey_mgr_process_partition_table(osm_log_t * p_log, osm_sm_t * sm, 200 const osm_prtn_t * p_prtn, 201 const boolean_t full) 202 { 203 const cl_map_t *p_tbl = 204 full ? &p_prtn->full_guid_tbl : &p_prtn->part_guid_tbl; 205 cl_map_iterator_t i, i_next; 206 ib_net16_t pkey = p_prtn->pkey; 207 osm_physp_t *p_physp; 208 209 if (full) 210 pkey |= cl_hton16(0x8000); 211 212 i_next = cl_map_head(p_tbl); 213 while (i_next != cl_map_end(p_tbl)) { 214 i = i_next; 215 i_next = cl_map_next(i); 216 p_physp = cl_map_obj(i); 217 if (p_physp) 218 pkey_mgr_process_physical_port(p_log, sm, pkey, 219 p_physp); 220 } 221 } 222 223 static ib_api_status_t 224 pkey_mgr_update_pkey_entry(IN osm_sm_t * sm, 225 IN const osm_physp_t * p_physp, 226 IN const ib_pkey_table_t * block, 227 IN const uint16_t block_index) 228 { 229 osm_madw_context_t context; 230 osm_node_t *p_node = osm_physp_get_node_ptr(p_physp); 231 osm_physp_t *physp0; 232 uint32_t attr_mod; 233 ib_net64_t m_key; 234 235 context.pkey_context.node_guid = osm_node_get_node_guid(p_node); 236 context.pkey_context.port_guid = osm_physp_get_port_guid(p_physp); 237 context.pkey_context.set_method = TRUE; 238 attr_mod = block_index; 239 if (osm_node_get_type(p_node) == IB_NODE_TYPE_SWITCH && 240 osm_physp_get_port_num(p_physp) != 0) { 241 attr_mod |= osm_physp_get_port_num(p_physp) << 16; 242 physp0 = osm_node_get_physp_ptr(p_node, 0); 243 m_key = ib_port_info_get_m_key(&physp0->port_info); 244 } else 245 m_key = ib_port_info_get_m_key(&p_physp->port_info); 246 return osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp), 247 (uint8_t *) block, sizeof(*block), 248 IB_MAD_ATTR_P_KEY_TABLE, 249 cl_hton32(attr_mod), FALSE, m_key, 250 CL_DISP_MSGID_NONE, &context); 251 } 252 253 static ib_api_status_t 254 pkey_mgr_enforce_partition(IN osm_log_t * p_log, osm_sm_t * sm, 255 IN osm_physp_t * p_physp, 256 IN osm_partition_enforce_type_enum enforce_type) 257 { 258 osm_madw_context_t context; 259 uint8_t payload[IB_SMP_DATA_SIZE]; 260 ib_port_info_t *p_pi; 261 ib_net64_t m_key; 262 osm_physp_t *physp0; 263 ib_api_status_t status; 264 uint8_t enforce_bits; 265 266 p_pi = &p_physp->port_info; 267 268 if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_BOTH) 269 enforce_bits = 0xc; 270 else if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_IN) 271 enforce_bits = 0x8; 272 else 273 enforce_bits = 0x4; 274 275 if ((p_pi->vl_enforce & 0xc) == enforce_bits * 276 (enforce_type != OSM_PARTITION_ENFORCE_TYPE_OFF)) { 277 OSM_LOG(p_log, OSM_LOG_DEBUG, 278 "No need to update PortInfo for " 279 "node 0x%016" PRIx64 " port %u (%s)\n", 280 cl_ntoh64(osm_node_get_node_guid 281 (osm_physp_get_node_ptr(p_physp))), 282 osm_physp_get_port_num(p_physp), 283 p_physp->p_node->print_desc); 284 return IB_SUCCESS; 285 } 286 287 memcpy(payload, p_pi, sizeof(ib_port_info_t)); 288 289 p_pi = (ib_port_info_t *) payload; 290 p_pi->vl_enforce &= ~0xc; 291 if (enforce_type != OSM_PARTITION_ENFORCE_TYPE_OFF) 292 p_pi->vl_enforce |= enforce_bits; 293 294 p_pi->state_info2 = 0; 295 ib_port_info_set_port_state(p_pi, IB_LINK_NO_CHANGE); 296 297 physp0 = osm_node_get_physp_ptr(p_physp->p_node, 0); 298 m_key = ib_port_info_get_m_key(&physp0->port_info); 299 300 context.pi_context.node_guid = 301 osm_node_get_node_guid(osm_physp_get_node_ptr(p_physp)); 302 context.pi_context.port_guid = osm_physp_get_port_guid(p_physp); 303 context.pi_context.set_method = TRUE; 304 context.pi_context.light_sweep = FALSE; 305 context.pi_context.active_transition = FALSE; 306 context.pi_context.client_rereg = FALSE; 307 308 status = osm_req_set(sm, osm_physp_get_dr_path_ptr(p_physp), 309 payload, sizeof(payload), 310 IB_MAD_ATTR_PORT_INFO, 311 cl_hton32(osm_physp_get_port_num(p_physp)), 312 FALSE, m_key, 313 CL_DISP_MSGID_NONE, &context); 314 if (status != IB_SUCCESS) 315 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0511: " 316 "Failed to set PortInfo for " 317 "node 0x%016" PRIx64 " port %u (%s)\n", 318 cl_ntoh64(osm_node_get_node_guid 319 (osm_physp_get_node_ptr(p_physp))), 320 osm_physp_get_port_num(p_physp), 321 p_physp->p_node->print_desc); 322 else 323 OSM_LOG(p_log, OSM_LOG_DEBUG, 324 "Set PortInfo for node 0x%016" PRIx64 " port %u (%s)\n", 325 cl_ntoh64(osm_node_get_node_guid 326 (osm_physp_get_node_ptr(p_physp))), 327 osm_physp_get_port_num(p_physp), 328 p_physp->p_node->print_desc); 329 return status; 330 } 331 332 static void clear_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl, 333 uint16_t pkey_index) 334 { 335 uint16_t pkey_idx_bias, pkey_idx; 336 void *ptr; 337 uintptr_t pkey_idx_ptr; 338 cl_map_iterator_t map_iter, map_iter_temp; 339 340 map_iter = cl_map_head(&p_pkey_tbl->accum_pkeys); 341 342 pkey_idx_bias = pkey_index + 1; // adjust for pkey index bias in accum_pkeys 343 344 while (map_iter != cl_map_end(&p_pkey_tbl->accum_pkeys)) { 345 map_iter_temp = cl_map_next(map_iter); 346 ptr = (uint16_t *) cl_map_obj(map_iter); 347 CL_ASSERT(ptr); 348 pkey_idx_ptr = (uintptr_t) ptr; 349 pkey_idx = pkey_idx_ptr; 350 if (pkey_idx == pkey_idx_bias) { 351 cl_map_remove_item(&p_pkey_tbl->accum_pkeys, map_iter); 352 if (p_pkey_tbl->last_pkey_idx == pkey_idx) 353 osm_pkey_find_last_accum_pkey_index(p_pkey_tbl); 354 break; 355 } 356 map_iter = map_iter_temp; 357 } 358 } 359 360 static int last_accum_pkey_index(osm_pkey_tbl_t * p_pkey_tbl, 361 uint16_t * p_block_idx, 362 uint8_t * p_pkey_idx) 363 { 364 if (p_pkey_tbl->last_pkey_idx) { 365 *p_block_idx = (p_pkey_tbl->last_pkey_idx - 1) / IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 366 *p_pkey_idx = (p_pkey_tbl->last_pkey_idx - 1) % IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 367 return 1; 368 } 369 370 return 0; 371 } 372 373 static int pkey_mgr_update_port(osm_log_t * p_log, osm_sm_t * sm, 374 const osm_port_t * const p_port) 375 { 376 osm_physp_t *p_physp; 377 osm_node_t *p_node; 378 ib_pkey_table_t *block, *new_block; 379 osm_pkey_tbl_t *p_pkey_tbl; 380 uint16_t block_index; 381 uint8_t pkey_index; 382 uint16_t last_free_block_index = 0; 383 uint8_t last_free_pkey_index = 0; 384 uint16_t num_of_blocks; 385 uint16_t max_num_of_blocks; 386 ib_api_status_t status; 387 osm_pending_pkey_t *p_pending; 388 boolean_t found; 389 ib_pkey_table_t empty_block; 390 int ret = 0, full = 0; 391 void *ptr; 392 uintptr_t pkey_idx_ptr; 393 uint16_t pkey_idx; 394 395 p_physp = p_port->p_physp; 396 if (!p_physp) 397 return FALSE; 398 399 memset(&empty_block, 0, sizeof(ib_pkey_table_t)); 400 401 p_node = osm_physp_get_node_ptr(p_physp); 402 p_pkey_tbl = &p_physp->pkeys; 403 num_of_blocks = osm_pkey_tbl_get_num_blocks(p_pkey_tbl); 404 max_num_of_blocks = pkey_mgr_get_physp_max_blocks(p_physp); 405 if (p_pkey_tbl->max_blocks > max_num_of_blocks) { 406 OSM_LOG(p_log, OSM_LOG_INFO, 407 "Max number of blocks reduced from %u to %u " 408 "for node 0x%016" PRIx64 " port %u (%s)\n", 409 p_pkey_tbl->max_blocks, max_num_of_blocks, 410 cl_ntoh64(osm_node_get_node_guid(p_node)), 411 osm_physp_get_port_num(p_physp), 412 p_physp->p_node->print_desc); 413 } 414 p_pkey_tbl->max_blocks = max_num_of_blocks; 415 416 osm_pkey_tbl_init_new_blocks(p_pkey_tbl); 417 p_pkey_tbl->used_blocks = 0; 418 419 /* 420 process every pending pkey in order - 421 first must be "updated" last are "new" 422 */ 423 p_pending = 424 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending); 425 while (p_pending != 426 (osm_pending_pkey_t *) cl_qlist_end(&p_pkey_tbl->pending)) { 427 428 found = FALSE; 429 ptr = NULL; 430 431 if (p_pending->is_new == FALSE) { 432 block_index = p_pending->block; 433 pkey_index = p_pending->index; 434 found = TRUE; 435 } else { 436 ptr = cl_map_get(&p_pkey_tbl->accum_pkeys,p_pending->pkey); 437 if (ptr != NULL) { 438 pkey_idx_ptr = (uintptr_t) ptr; 439 pkey_idx = pkey_idx_ptr; 440 pkey_idx--; /* adjust pkey index for bias */ 441 block_index = pkey_idx / IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 442 pkey_index = pkey_idx % IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 443 444 if (((sm->p_subn->opt.allow_both_pkeys && 445 p_pending->pkey == p_physp->pkeys.indx0_pkey) || 446 (!sm->p_subn->opt.allow_both_pkeys && 447 ib_pkey_get_base(p_pending->pkey) == ib_pkey_get_base(p_physp->pkeys.indx0_pkey))) || 448 ((p_pending->pkey != p_physp->pkeys.indx0_pkey && 449 pkey_idx == 0))) { 450 clear_accum_pkey_index(p_pkey_tbl, pkey_idx); 451 cl_qlist_insert_tail(&p_pkey_tbl->pending, 452 (cl_list_item_t *)p_pending); 453 p_pending = 454 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl->pending); 455 continue; 456 } else 457 found = TRUE; 458 } 459 460 if (!found) { 461 if (!p_pkey_tbl->indx0_pkey && 462 ((sm->p_subn->opt.allow_both_pkeys && 463 p_pending->pkey == IB_DEFAULT_PKEY) || 464 (!sm->p_subn->opt.allow_both_pkeys && 465 ib_pkey_get_base(p_pending->pkey) == IB_DEFAULT_PARTIAL_PKEY))) { 466 block_index = 0; 467 pkey_index = 0; 468 } else if ((sm->p_subn->opt.allow_both_pkeys && 469 p_pending->pkey == p_pkey_tbl->indx0_pkey) || 470 (!sm->p_subn->opt.allow_both_pkeys && 471 ib_pkey_get_base(p_pending->pkey) == 472 ib_pkey_get_base(p_pkey_tbl->indx0_pkey))) { 473 block_index = 0; 474 pkey_index = 0; 475 } else if (last_accum_pkey_index(p_pkey_tbl, 476 &last_free_block_index, 477 &last_free_pkey_index)) { 478 block_index = last_free_block_index; 479 pkey_index = last_free_pkey_index + 1; 480 if (pkey_index >= IB_NUM_PKEY_ELEMENTS_IN_BLOCK) { 481 block_index++; 482 pkey_index -= IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 483 } 484 } else { 485 block_index = 0; 486 pkey_index = 1; 487 } 488 489 if (block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index >= pkey_mgr_get_physp_max_pkeys(p_physp)) { 490 if ((sm->p_subn->opt.allow_both_pkeys && 491 p_pending->pkey != IB_DEFAULT_PKEY) || 492 (!sm->p_subn->opt.allow_both_pkeys && 493 ib_pkey_get_base(p_pending->pkey) != IB_DEFAULT_PARTIAL_PKEY)) { 494 last_free_block_index = 0; 495 last_free_pkey_index = 1; 496 found = osm_pkey_find_next_free_entry(p_pkey_tbl, &last_free_block_index, &last_free_pkey_index); 497 } else 498 found = FALSE; 499 if (!found) 500 full = 1; 501 else { 502 block_index = last_free_block_index; 503 pkey_index = last_free_pkey_index; 504 if (block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index >= pkey_mgr_get_physp_max_pkeys(p_physp)) { 505 full = 1; 506 found = FALSE; 507 } else { 508 OSM_LOG(p_log, OSM_LOG_INFO, 509 "Reusing PKeyTable block index %u pkey index %u " 510 "for pkey 0x%x on 0x%016" PRIx64 " port %u (%s)\n", 511 block_index, 512 pkey_index, 513 cl_ntoh16(p_pending->pkey), 514 cl_ntoh64(osm_node_get_node_guid(p_node)), 515 osm_physp_get_port_num(p_physp), 516 p_physp->p_node->print_desc); 517 518 clear_accum_pkey_index(p_pkey_tbl, block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index); 519 } 520 } 521 if (full) 522 OSM_LOG(p_log, OSM_LOG_ERROR, 523 "ERR 0512: " 524 "Failed to set PKey 0x%04x because Pkey table is full " 525 "for node 0x%016" PRIx64 " port %u (%s)\n", 526 cl_ntoh16(p_pending->pkey), 527 cl_ntoh64(osm_node_get_node_guid(p_node)), 528 osm_physp_get_port_num(p_physp), 529 p_physp->p_node->print_desc); 530 } else 531 found = TRUE; 532 } 533 } 534 535 if (found) { 536 if (IB_SUCCESS != 537 osm_pkey_tbl_set_new_entry(p_pkey_tbl, block_index, 538 pkey_index, 539 p_pending->pkey)) { 540 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0505: " 541 "Failed to set PKey 0x%04x in block %u idx %u " 542 "for node 0x%016" PRIx64 " port %u (%s)\n", 543 cl_ntoh16(p_pending->pkey), block_index, 544 pkey_index, 545 cl_ntoh64(osm_node_get_node_guid 546 (p_node)), 547 osm_physp_get_port_num(p_physp), 548 p_physp->p_node->print_desc); 549 } 550 if (ptr == NULL && 551 CL_SUCCESS != 552 osm_pkey_tbl_set_accum_pkeys(p_pkey_tbl, 553 p_pending->pkey, 554 block_index * IB_NUM_PKEY_ELEMENTS_IN_BLOCK + pkey_index)) { 555 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0508: " 556 "Failed to set accum_pkeys PKey 0x%04x " 557 "in block %u idx %u for node 0x%016" 558 PRIx64 " port %u (%s)\n", 559 cl_ntoh16(p_pending->pkey), block_index, 560 pkey_index, 561 cl_ntoh64(osm_node_get_node_guid(p_node)), 562 osm_physp_get_port_num(p_physp), 563 p_physp->p_node->print_desc); 564 } 565 } 566 free(p_pending); 567 p_pending = 568 (osm_pending_pkey_t *) cl_qlist_remove_head(&p_pkey_tbl-> 569 pending); 570 } 571 572 p_pkey_tbl->indx0_pkey = 0; 573 /* now look for changes and store */ 574 for (block_index = 0; block_index < num_of_blocks; block_index++) { 575 block = osm_pkey_tbl_block_get(p_pkey_tbl, block_index); 576 new_block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index); 577 if (!new_block) 578 new_block = &empty_block; 579 if (block && !memcmp(new_block, block, sizeof(*block))) 580 continue; 581 582 status = 583 pkey_mgr_update_pkey_entry(sm, p_physp, new_block, 584 block_index); 585 if (status == IB_SUCCESS) 586 OSM_LOG(p_log, OSM_LOG_DEBUG, 587 "Updated pkey table block %u for node 0x%016" 588 PRIx64 " port %u (%s)\n", block_index, 589 cl_ntoh64(osm_node_get_node_guid(p_node)), 590 osm_physp_get_port_num(p_physp), 591 p_physp->p_node->print_desc); 592 else { 593 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0506: " 594 "pkey_mgr_update_pkey_entry() failed to update " 595 "pkey table block %u for node 0x%016" PRIx64 596 " port %u (%s)\n", block_index, 597 cl_ntoh64(osm_node_get_node_guid(p_node)), 598 osm_physp_get_port_num(p_physp), 599 p_physp->p_node->print_desc); 600 ret = -1; 601 } 602 } 603 604 return ret; 605 } 606 607 static int last_used_pkey_index(const osm_port_t * const p_port, 608 const osm_pkey_tbl_t * p_pkey_tbl, 609 uint16_t * p_last_index) 610 { 611 ib_pkey_table_t *last_block; 612 uint16_t index, last_index = 0; 613 614 CL_ASSERT(p_last_index); 615 616 last_block = osm_pkey_tbl_new_block_get(p_pkey_tbl, 617 p_pkey_tbl->used_blocks - 1); 618 if (!last_block) 619 return 1; 620 621 if (p_pkey_tbl->used_blocks == p_pkey_tbl->max_blocks) 622 last_index = cl_ntoh16(p_port->p_node->node_info.partition_cap) % IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 623 if (last_index == 0) 624 last_index = IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 625 index = last_index; 626 do { 627 index--; 628 if (!ib_pkey_is_invalid(last_block->pkey_entry[index])) 629 break; 630 } while (index != 0); 631 632 *p_last_index = index; 633 return 0; 634 } 635 636 static int update_peer_block(osm_log_t * p_log, osm_sm_t * sm, 637 osm_physp_t * peer, 638 osm_pkey_tbl_t * p_peer_pkey_tbl, 639 ib_pkey_table_t * new_peer_block, 640 uint16_t peer_block_idx, osm_node_t * p_node) 641 { 642 int ret = 0; 643 ib_pkey_table_t *peer_block; 644 645 peer_block = osm_pkey_tbl_block_get(p_peer_pkey_tbl, peer_block_idx); 646 if (!peer_block || 647 memcmp(peer_block, new_peer_block, sizeof(*peer_block))) { 648 if (pkey_mgr_update_pkey_entry(sm, peer, new_peer_block, 649 peer_block_idx) != IB_SUCCESS) { 650 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0509: " 651 "pkey_mgr_update_pkey_entry() failed to update " 652 "pkey table block %u for node 0x%016" 653 PRIx64 " port %u (%s)\n", 654 peer_block_idx, 655 cl_ntoh64(osm_node_get_node_guid(p_node)), 656 osm_physp_get_port_num(peer), 657 p_node->print_desc); 658 ret = -1; 659 } 660 } 661 662 return ret; 663 } 664 665 static int new_pkey_exists(osm_pkey_tbl_t * p_pkey_tbl, ib_net16_t pkey) 666 { 667 uint16_t num_blocks; 668 uint16_t block_index; 669 ib_pkey_table_t *block; 670 uint16_t pkey_idx; 671 672 num_blocks = (uint16_t) cl_ptr_vector_get_size(&p_pkey_tbl->new_blocks); 673 for (block_index = 0; block_index < num_blocks; block_index++) { 674 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index); 675 if (!block) 676 continue; 677 678 for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 679 pkey_idx++) { 680 if (block->pkey_entry[pkey_idx] == pkey) 681 return 1; 682 } 683 } 684 return 0; 685 } 686 687 static int pkey_mgr_update_peer_port(osm_log_t * p_log, osm_sm_t * sm, 688 const osm_subn_t * p_subn, 689 const osm_port_t * const p_port, 690 osm_partition_enforce_type_enum enforce_type) 691 { 692 osm_physp_t *p_physp, *peer; 693 osm_node_t *p_node; 694 ib_pkey_table_t *block; 695 const osm_pkey_tbl_t *p_pkey_tbl; 696 osm_pkey_tbl_t *p_peer_pkey_tbl; 697 uint16_t block_index, peer_block_idx; 698 uint16_t peer_max_blocks; 699 uint16_t last_index; 700 ib_pkey_table_t new_peer_block; 701 uint16_t pkey_idx, peer_pkey_idx; 702 ib_net16_t pkey, full_pkey; 703 int ret = 0, loop_exit = 0; 704 705 p_physp = p_port->p_physp; 706 if (!p_physp) 707 return -1; 708 peer = osm_physp_get_remote(p_physp); 709 if (!peer) 710 return -1; 711 p_node = osm_physp_get_node_ptr(peer); 712 if (!p_node->sw || !p_node->sw->switch_info.enforce_cap) 713 return 0; 714 715 if (enforce_type == OSM_PARTITION_ENFORCE_TYPE_OFF) { 716 pkey_mgr_enforce_partition(p_log, sm, peer, OSM_PARTITION_ENFORCE_TYPE_OFF); 717 return ret; 718 } 719 720 p_pkey_tbl = osm_physp_get_pkey_tbl(p_physp); 721 peer_max_blocks = pkey_mgr_get_physp_max_blocks(peer); 722 p_peer_pkey_tbl = &peer->pkeys; 723 peer_block_idx = 0; 724 peer_pkey_idx = 0; 725 for (block_index = 0; block_index < p_pkey_tbl->used_blocks; 726 block_index++) { 727 if (loop_exit) 728 break; 729 block = osm_pkey_tbl_new_block_get(p_pkey_tbl, block_index); 730 if (!block) 731 continue; 732 for (pkey_idx = 0; pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 733 pkey_idx++) { 734 pkey = block->pkey_entry[pkey_idx]; 735 if (ib_pkey_is_invalid(pkey)) 736 continue; 737 if (!ib_pkey_is_full_member(pkey)) { 738 full_pkey = pkey | IB_PKEY_TYPE_MASK; 739 if (new_pkey_exists(&p_physp->pkeys, full_pkey)) 740 continue; 741 } 742 new_peer_block.pkey_entry[peer_pkey_idx] = pkey; 743 if (peer_block_idx >= peer_max_blocks) { 744 loop_exit = 1; 745 break; 746 } 747 if (++peer_pkey_idx == IB_NUM_PKEY_ELEMENTS_IN_BLOCK) { 748 if (update_peer_block(p_log, sm, peer, 749 p_peer_pkey_tbl, 750 &new_peer_block, 751 peer_block_idx, p_node)) 752 ret = -1; 753 peer_pkey_idx = 0; 754 peer_block_idx++; 755 } 756 } 757 } 758 759 if (peer_block_idx < peer_max_blocks) { 760 if (peer_pkey_idx) { 761 /* Handle partial last block */ 762 for (; peer_pkey_idx < IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 763 peer_pkey_idx++) 764 new_peer_block.pkey_entry[peer_pkey_idx] = 0; 765 if (update_peer_block(p_log, sm, peer, p_peer_pkey_tbl, 766 &new_peer_block, peer_block_idx, 767 p_node)) 768 ret = -1; 769 } else 770 peer_block_idx--; 771 772 p_peer_pkey_tbl->used_blocks = peer_block_idx + 1; 773 if (p_peer_pkey_tbl->used_blocks == peer_max_blocks) { 774 /* Is last used pkey index beyond switch peer port capacity ? */ 775 if (!last_used_pkey_index(p_port, p_peer_pkey_tbl, 776 &last_index)) { 777 last_index += peer_block_idx * IB_NUM_PKEY_ELEMENTS_IN_BLOCK; 778 if (cl_ntoh16(p_node->sw->switch_info.enforce_cap) <= last_index) { 779 OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0507: " 780 "Not enough pkey entries (%u <= %u) on switch 0x%016" 781 PRIx64 " port %u (%s). Clearing Enforcement bit\n", 782 cl_ntoh16(p_node->sw->switch_info.enforce_cap), 783 last_index, 784 cl_ntoh64(osm_node_get_node_guid(p_node)), 785 osm_physp_get_port_num(peer), 786 p_node->print_desc); 787 enforce_type = OSM_PARTITION_ENFORCE_TYPE_OFF; 788 ret = -1; 789 } 790 } 791 } 792 } else { 793 p_peer_pkey_tbl->used_blocks = peer_max_blocks; 794 enforce_type = OSM_PARTITION_ENFORCE_TYPE_OFF; 795 } 796 797 if (!ret) 798 OSM_LOG(p_log, OSM_LOG_DEBUG, 799 "Pkey table was successfully updated for node 0x%016" 800 PRIx64 " port %u (%s)\n", 801 cl_ntoh64(osm_node_get_node_guid(p_node)), 802 osm_physp_get_port_num(peer), p_node->print_desc); 803 804 if (pkey_mgr_enforce_partition(p_log, sm, peer, enforce_type)) 805 ret = -1; 806 807 return ret; 808 } 809 810 int osm_pkey_mgr_process(IN osm_opensm_t * p_osm) 811 { 812 cl_qmap_t *p_tbl; 813 cl_map_item_t *p_next; 814 osm_prtn_t *p_prtn; 815 osm_port_t *p_port; 816 osm_switch_t *p_sw; 817 osm_physp_t *p_physp; 818 osm_node_t *p_remote_node; 819 uint8_t i; 820 int ret = 0; 821 822 CL_ASSERT(p_osm); 823 824 OSM_LOG_ENTER(&p_osm->log); 825 826 CL_PLOCK_EXCL_ACQUIRE(&p_osm->lock); 827 828 if (osm_prtn_make_partitions(&p_osm->log, &p_osm->subn) != IB_SUCCESS) { 829 OSM_LOG(&p_osm->log, OSM_LOG_ERROR, "ERR 0510: " 830 "osm_prtn_make_partitions() failed\n"); 831 ret = -1; 832 goto _err; 833 } 834 835 /* populate the pending pkey entries by scanning all partitions */ 836 p_tbl = &p_osm->subn.prtn_pkey_tbl; 837 p_next = cl_qmap_head(p_tbl); 838 while (p_next != cl_qmap_end(p_tbl)) { 839 p_prtn = (osm_prtn_t *) p_next; 840 p_next = cl_qmap_next(p_next); 841 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm, 842 p_prtn, FALSE); 843 pkey_mgr_process_partition_table(&p_osm->log, &p_osm->sm, 844 p_prtn, TRUE); 845 } 846 847 /* calculate and set new pkey tables */ 848 p_tbl = &p_osm->subn.port_guid_tbl; 849 p_next = cl_qmap_head(p_tbl); 850 while (p_next != cl_qmap_end(p_tbl)) { 851 p_port = (osm_port_t *) p_next; 852 p_next = cl_qmap_next(p_next); 853 if (pkey_mgr_update_port(&p_osm->log, &p_osm->sm, p_port)) 854 ret = -1; 855 if ((osm_node_get_type(p_port->p_node) != IB_NODE_TYPE_SWITCH) 856 && pkey_mgr_update_peer_port(&p_osm->log, &p_osm->sm, 857 &p_osm->subn, p_port, 858 p_osm->subn.opt.part_enforce_enum)) 859 ret = -1; 860 } 861 862 /* clear partition enforcement on inter-switch links */ 863 p_tbl = &p_osm->subn.sw_guid_tbl; 864 p_next = cl_qmap_head(p_tbl); 865 while (p_next != cl_qmap_end(p_tbl)) { 866 p_sw = (osm_switch_t *) p_next; 867 p_next = cl_qmap_next(p_next); 868 for (i = 1; i < p_sw->num_ports; i++) { 869 p_physp = osm_node_get_physp_ptr(p_sw->p_node, i); 870 if (p_physp && p_physp->p_remote_physp) 871 p_remote_node = p_physp->p_remote_physp->p_node; 872 else 873 continue; 874 875 if (osm_node_get_type(p_remote_node) != IB_NODE_TYPE_SWITCH) 876 continue; 877 878 if(! (p_physp->port_info.vl_enforce & 0xc )) 879 continue; 880 881 /* clear partition enforcement */ 882 if (pkey_mgr_enforce_partition(&p_osm->log, &p_osm->sm, p_physp, OSM_PARTITION_ENFORCE_TYPE_OFF)) 883 ret = -1; 884 } 885 } 886 _err: 887 CL_PLOCK_RELEASE(&p_osm->lock); 888 OSM_LOG_EXIT(&p_osm->log); 889 return ret; 890 } 891