1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2019-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "nvlink.h"
25 #include "nvlink_export.h"
26 #include "nvlink_os.h"
27 #include "../nvlink_ctx.h"
28 #include "../nvlink_helper.h"
29
30 /**
31 * For a given link, return the associated intranode connection
32 *
33 * @param[in] endpoint NVLink Link pointer
34 * @param[out] conn Connection associated with the link
35 */
36 void
nvlink_core_get_intranode_conn(nvlink_link * endpoint,nvlink_intranode_conn ** conn)37 nvlink_core_get_intranode_conn
38 (
39 nvlink_link *endpoint,
40 nvlink_intranode_conn **conn
41 )
42 {
43 nvlink_intranode_conn *tmpConn = NULL;
44
45 if ((endpoint == NULL) || (conn == NULL))
46 {
47 return;
48 }
49
50 FOR_EACH_CONNECTION(tmpConn, nvlinkLibCtx.nv_intraconn_head, node)
51 {
52 if (tmpConn->end0 == endpoint || tmpConn->end1 == endpoint)
53 {
54 *conn = tmpConn;
55 break;
56 }
57 }
58 }
59
60 /**
61 * For a given local link, return the associated internode connection
62 *
63 * @param[in] localLink NVLink Link pointer
64 * @param[out] conn Connection associated with the link
65 */
66 void
nvlink_core_get_internode_conn(nvlink_link * localLink,nvlink_internode_conn ** conn)67 nvlink_core_get_internode_conn
68 (
69 nvlink_link *localLink,
70 nvlink_internode_conn **conn
71 )
72 {
73 nvlink_internode_conn *tmpConn = NULL;
74
75 if ((localLink == NULL) || (conn == NULL))
76 {
77 return;
78 }
79
80 FOR_EACH_CONNECTION(tmpConn, nvlinkLibCtx.nv_interconn_head, node)
81 {
82 if (tmpConn->local_end == localLink)
83 {
84 *conn = tmpConn;
85 break;
86 }
87 }
88 }
89
90 /**
91 * Add a new intranode connection to the list of connections
92 *
93 * @param[in] end0 NVLink Link pointer for end0
94 * @param[in] end1 NVLink Link pointer for end1
95 *
96 * return NVL_SUCCESS if the conn was added successfully
97 */
98 NvlStatus
nvlink_core_add_intranode_conn(nvlink_link * end0,nvlink_link * end1)99 nvlink_core_add_intranode_conn
100 (
101 nvlink_link *end0,
102 nvlink_link *end1
103 )
104 {
105 nvlink_intranode_conn *conn = NULL;
106
107 if ((end0 == NULL) || (end1 == NULL))
108 {
109 return NVL_BAD_ARGS;
110 }
111
112 // don't do anything if we have an intranode connecction
113 nvlink_core_get_intranode_conn(end0, &conn);
114
115 if (conn != NULL)
116 {
117 // Verify that the other end of the connection is indeed end1
118 conn->end0 == end0 ?
119 nvlink_assert(conn->end1 == end1) :
120 nvlink_assert(conn->end0 == end1);
121 return NVL_SUCCESS;
122 }
123
124 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
125 "Adding new NVLink intranode connection between %s:%s and %s:%s\n",
126 end0->dev->deviceName, end0->linkName,
127 end1->dev->deviceName, end1->linkName));
128
129 // create a new intranode connection object
130 conn = (nvlink_intranode_conn*)nvlink_malloc(sizeof(nvlink_intranode_conn));
131 if (conn == NULL)
132 {
133 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
134 "Adding NVLink intranode connection failed "
135 "due to memory allocation error\n"));
136 return NVL_NO_MEM;
137 }
138
139 nvlink_memset(conn, 0, sizeof(nvlink_intranode_conn));
140
141 // Initialize the node for the connection
142 nvListInit(&conn->node);
143
144 // Initialize the connection endpoints
145 conn->end0 = end0;
146 conn->end1 = end1;
147
148 // Add the connection to the list of connections
149 nvListAppend(&conn->node, &nvlinkLibCtx.nv_intraconn_head.node);
150
151 //
152 // Update the count of connected endpoints
153 // Loopback link, increment by 1
154 // Non loopback link, increment by 2
155 //
156 nvlinkLibCtx.connectedEndpoints = ( end0 == end1 ?
157 nvlinkLibCtx.connectedEndpoints + 1:
158 nvlinkLibCtx.connectedEndpoints + 2 );
159
160 return NVL_SUCCESS;
161 }
162
163 /**
164 * Add a new internode connection to the list of internode connections
165 *
166 * Note: As of now, no stats/count for internode connections.
167 *
168 * @param[in] localLink NVLink Link pointer for one end
169 * @param[in] remoteEndPoint Remote endpoint
170 *
171 * return NVL_SUCCESS if the conn was added succesfully
172 */
173 NvlStatus
nvlink_core_add_internode_conn(nvlink_link * localLink,nvlink_remote_endpoint_info * remoteEndPoint)174 nvlink_core_add_internode_conn
175 (
176 nvlink_link *localLink,
177 nvlink_remote_endpoint_info *remoteEndPoint
178 )
179 {
180 nvlink_internode_conn *conn = NULL;
181
182 if ((localLink == NULL) || (remoteEndPoint == NULL))
183 {
184 return NVL_BAD_ARGS;
185 }
186
187 // Don't do anything if we have an internode connecction for local link
188 nvlink_core_get_internode_conn(localLink, &conn);
189 if (conn != NULL)
190 {
191 return NVL_SUCCESS;
192 }
193
194 // create a new connection
195 conn = (nvlink_internode_conn *)nvlink_malloc(sizeof(nvlink_internode_conn));
196 if (conn == NULL)
197 {
198 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
199 "Adding nvlink internode connection failed"
200 " due to memory allocation error\n"));
201 return NVL_NO_MEM;
202 }
203
204 nvlink_memset(conn, 0, sizeof(nvlink_internode_conn));
205
206 // initialize the node for the connection list
207 nvListInit(&conn->node);
208
209 // copy/assign the connection endpoints information
210 conn->local_end = localLink;
211 nvlink_memcpy(&conn->remote_end,
212 remoteEndPoint,
213 sizeof(nvlink_remote_endpoint_info));
214
215 // add the connection to the list of internode connections
216 nvListAppend(&conn->node, &nvlinkLibCtx.nv_interconn_head.node);
217
218 return NVL_SUCCESS;
219 }
220
221 /**
222 * Remove the connection from the list of intranode connections
223 *
224 * @param[in] conn NVLink connection pointer
225 */
226 void
nvlink_core_remove_intranode_conn(nvlink_intranode_conn * conn)227 nvlink_core_remove_intranode_conn
228 (
229 nvlink_intranode_conn *conn
230 )
231 {
232 if (conn == NULL)
233 return;
234
235 // Remove the connection from the list of connections
236 nvListDel(&conn->node);
237
238 //
239 // Update the count of connected endpoints
240 // Loopback link, decrement by 1
241 // Non loopback link, decrement by 2
242 //
243 nvlinkLibCtx.connectedEndpoints = ( conn->end0 == conn->end1 ?
244 nvlinkLibCtx.connectedEndpoints - 1:
245 nvlinkLibCtx.connectedEndpoints - 2 );
246
247 //
248 // Update the count of notConnected endpoints
249 // Loopback link, do nothing
250 // Non-loopback link, increment by 1
251 //
252 nvlinkLibCtx.notConnectedEndpoints = ( conn->end0 != conn->end1 ?
253 nvlinkLibCtx.notConnectedEndpoints + 1:
254 nvlinkLibCtx.notConnectedEndpoints );
255
256 nvlink_free((void *)conn);
257 }
258
259 /**
260 * Remove the connection from the list of internode connections
261 *
262 * @param[in] localLink NVLink link pointer
263 */
264 void
nvlink_core_remove_internode_conn(nvlink_link * localLink)265 nvlink_core_remove_internode_conn
266 (
267 nvlink_link *localLink
268 )
269 {
270 nvlink_internode_conn *conn = NULL;
271
272 if (localLink == NULL)
273 return;
274
275 nvlink_core_get_internode_conn(localLink, &conn);
276
277 if (conn != NULL)
278 {
279 nvListDel(&conn->node);
280 nvlink_free((void *)conn);
281 }
282 }
283
284 /**
285 * Check if the given intranode connection is in the specified mode
286 *
287 * @param[in] conn NVLink Connection pointer
288 * @param[in] linkMode Link mode
289 *
290 * return NVL_SUCCESS if the conn is in the given state
291 */
292 NvlStatus
nvlink_core_check_intranode_conn_state(nvlink_intranode_conn * conn,NvU64 linkMode)293 nvlink_core_check_intranode_conn_state
294 (
295 nvlink_intranode_conn *conn,
296 NvU64 linkMode
297 )
298 {
299 if (conn == NULL)
300 {
301 return NVL_BAD_ARGS;
302 }
303
304 switch (linkMode)
305 {
306 case NVLINK_LINKSTATE_OFF:
307 {
308 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_OFF)) &&
309 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_OFF)))
310 {
311 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
312 "%s: Link already in OFF state. ",
313 __FUNCTION__));
314 nvlink_core_print_intranode_conn(conn);
315 return NVL_SUCCESS;
316 }
317
318 // Check if only one end of connection is OFF
319 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_OFF)) ||
320 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_OFF)))
321 {
322 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_ERRORS,
323 "%s: Link is in bad state. ",
324 __FUNCTION__));
325 nvlink_core_print_intranode_conn(conn);
326 return NVL_ERR_INVALID_STATE;
327 }
328
329 return NVL_ERR_GENERIC;
330 }
331
332 case NVLINK_LINKSTATE_RESET:
333 {
334 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_RESET)) &&
335 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_RESET)))
336 {
337 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
338 "%s: Link already in RESET state. ",
339 __FUNCTION__));
340 nvlink_core_print_intranode_conn(conn);
341 return NVL_SUCCESS;
342 }
343
344 // Check if only one end of connection is RESET
345 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_RESET)) ||
346 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_RESET)))
347 {
348 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_ERRORS,
349 "%s: Link is in bad state. ",
350 __FUNCTION__));
351 nvlink_core_print_intranode_conn(conn);
352 return NVL_ERR_INVALID_STATE;
353 }
354
355 return NVL_ERR_GENERIC;
356 }
357
358 case NVLINK_LINKSTATE_SAFE:
359 {
360 // Check if both ends and their sublinks are already in SAFE mode
361 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_SAFE)) &&
362 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_SAFE)))
363 {
364 if ((nvlink_core_check_tx_sublink_state(conn->end0,
365 NVLINK_SUBLINK_STATE_TX_OFF)) &&
366 (nvlink_core_check_tx_sublink_state(conn->end1,
367 NVLINK_SUBLINK_STATE_TX_OFF)) &&
368 (nvlink_core_check_rx_sublink_state(conn->end0,
369 NVLINK_SUBLINK_STATE_RX_OFF)) &&
370 (nvlink_core_check_rx_sublink_state(conn->end1,
371 NVLINK_SUBLINK_STATE_RX_OFF)))
372 {
373 //
374 // If links are in safe, check if sublinks are in off
375 // if so, we had performed pseudo-clean shutdown
376 //
377 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
378 "%s: Link is not in SAFE mode. ",
379 __FUNCTION__));
380 nvlink_core_print_intranode_conn(conn);
381 return NVL_ERR_GENERIC;
382 }
383 else if (!((nvlink_core_check_tx_sublink_state(conn->end0,
384 NVLINK_SUBLINK_STATE_TX_SAFE)) &&
385 (nvlink_core_check_tx_sublink_state(conn->end1,
386 NVLINK_SUBLINK_STATE_TX_SAFE)) &&
387 (nvlink_core_check_rx_sublink_state(conn->end0,
388 NVLINK_SUBLINK_STATE_RX_SAFE)) &&
389 (nvlink_core_check_rx_sublink_state(conn->end1,
390 NVLINK_SUBLINK_STATE_RX_SAFE))))
391 {
392 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_ERRORS,
393 "%s: Sublinks are in bad state. ",
394 __FUNCTION__));
395 nvlink_core_print_intranode_conn(conn);
396 return NVL_ERR_INVALID_STATE;
397 }
398
399 return NVL_SUCCESS;
400 }
401
402 // Check if only one end of connection is in SAFE mode
403 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_SAFE)) ||
404 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_SAFE)))
405 {
406 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_ERRORS,
407 "%s: Link is in bad state. ",
408 __FUNCTION__));
409 nvlink_core_print_intranode_conn(conn);
410 return NVL_ERR_INVALID_STATE;
411 }
412
413 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
414 "%s: Link is not in SAFE mode. ",
415 __FUNCTION__));
416 nvlink_core_print_intranode_conn(conn);
417 return NVL_ERR_GENERIC;
418 }
419
420 case NVLINK_LINKSTATE_HS:
421 {
422 // Check if both ends and their sublinks are already in HS mode
423 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_HS)) &&
424 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_HS)))
425 {
426 // In NVLINK4.0, corelib doesn't control sublink state transitions
427 if (conn->end0->version < NVLINK_DEVICE_VERSION_40 &&
428 !((nvlink_core_check_tx_sublink_state(conn->end0,
429 NVLINK_SUBLINK_STATE_TX_HS)) &&
430 (nvlink_core_check_tx_sublink_state(conn->end1,
431 NVLINK_SUBLINK_STATE_TX_HS)) &&
432 (nvlink_core_check_rx_sublink_state(conn->end0,
433 NVLINK_SUBLINK_STATE_RX_HS)) &&
434 (nvlink_core_check_rx_sublink_state(conn->end1,
435 NVLINK_SUBLINK_STATE_RX_HS))))
436 {
437 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_ERRORS,
438 "%s: Sublinks are in bad state. ",
439 __FUNCTION__));
440 nvlink_core_print_intranode_conn(conn);
441 return NVL_ERR_INVALID_STATE;
442 }
443
444 return NVL_SUCCESS;
445 }
446
447 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_HS)) ||
448 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_HS)))
449 {
450 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_ERRORS,
451 "%s: Link is in bad state. ",
452 __FUNCTION__));
453 nvlink_core_print_intranode_conn(conn);
454 return NVL_ERR_INVALID_STATE;
455 }
456
457 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
458 "%s: Link is not in HIGH SPEED mode. ",
459 __FUNCTION__));
460 nvlink_core_print_intranode_conn(conn);
461 return NVL_ERR_GENERIC;
462 }
463
464 case NVLINK_LINKSTATE_SLEEP:
465 {
466 // Check if both ends of connection are already in SLEEP mode
467 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_SLEEP)) &&
468 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_SLEEP)))
469 {
470 return NVL_SUCCESS;
471 }
472
473 // Check if only one end of connection is in SLEEP mode
474 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_SLEEP)) ||
475 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_SLEEP)))
476 {
477 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_ERRORS,
478 "%s: Link is in bad state. ",
479 __FUNCTION__));
480 nvlink_core_print_intranode_conn(conn);
481 return NVL_ERR_INVALID_STATE;
482 }
483
484 NVLINK_PRINT((DBG_MODULE_NVLINK_CORE, NVLINK_DBG_LEVEL_INFO,
485 "%s: Link is not in SLEEP mode. ",
486 __FUNCTION__));
487 nvlink_core_print_intranode_conn(conn);
488
489 return NVL_ERR_GENERIC;
490 }
491 case NVLINK_LINKSTATE_ACTIVE_PENDING:
492 {
493 // Check if both ends of connection are already in ACTIVE_PENDING
494 if ((nvlink_core_check_link_state(conn->end0, NVLINK_LINKSTATE_ACTIVE_PENDING)) &&
495 (nvlink_core_check_link_state(conn->end1, NVLINK_LINKSTATE_ACTIVE_PENDING)))
496 {
497 return NVL_SUCCESS;
498 }
499 break;
500 }
501 }
502
503 return NVL_SUCCESS;
504 }
505
506 /**
507 * Copy the intranode connection's remote endpoint information into
508 * the nvlink_conn_info structure passed in
509 *
510 * @param[in] remote_end NVLink Link pointer
511 * @param[in] conn_info Details of remote endpoint
512 */
513 void
nvlink_core_copy_intranode_conn_info(nvlink_link * remote_end,nvlink_conn_info * conn_info)514 nvlink_core_copy_intranode_conn_info
515 (
516 nvlink_link *remote_end,
517 nvlink_conn_info *conn_info
518 )
519 {
520 if ((remote_end == NULL) || (conn_info == NULL))
521 {
522 return;
523 }
524
525 // copy the remote device pci information
526 conn_info->domain = remote_end->dev->pciInfo.domain;
527 conn_info->bus = remote_end->dev->pciInfo.bus;
528 conn_info->device = remote_end->dev->pciInfo.device;
529 conn_info->function = remote_end->dev->pciInfo.function;
530 conn_info->pciDeviceId = remote_end->dev->pciInfo.pciDeviceId;
531 conn_info->chipSid = remote_end->localSid;
532
533 // copy the device type
534 conn_info->deviceType = remote_end->dev->type;
535
536 // copy the remote device uuid
537 if (remote_end->dev->uuid != NULL)
538 {
539 nvlink_memcpy(conn_info->devUuid, remote_end->dev->uuid, NV_UUID_LEN);
540 }
541
542 // copy the remote link number
543 conn_info->linkNumber = remote_end->linkNumber;
544 }
545
546 /**
547 * Copy the internode connection's remote endpoint information into
548 * the nvlink_conn_info structure passed in
549 *
550 * @param[in] remote_end NVLink Link pointer
551 * @param[in] conn_info Details of remote endpoint
552 */
553 void
nvlink_core_copy_internode_conn_info(nvlink_remote_endpoint_info * remote_end,nvlink_conn_info * conn_info)554 nvlink_core_copy_internode_conn_info
555 (
556 nvlink_remote_endpoint_info *remote_end,
557 nvlink_conn_info *conn_info
558 )
559 {
560 if ((remote_end == NULL) || (conn_info == NULL))
561 {
562 return;
563 }
564
565 // copy the remote device pci information
566 conn_info->domain = remote_end->pciInfo.domain;
567 conn_info->bus = remote_end->pciInfo.bus;
568 conn_info->device = remote_end->pciInfo.device;
569 conn_info->function = remote_end->pciInfo.function;
570 conn_info->pciDeviceId = 0;
571
572 // copy the device type
573 conn_info->deviceType = remote_end->devType;
574
575 // copy the remote device uuid
576 nvlink_memcpy(conn_info->devUuid, remote_end->devUuid, NV_UUID_LEN);
577
578 // copy the remote link number
579 conn_info->linkNumber = remote_end->linkIndex;
580 }
581