1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2015-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "conftest.h"
25
26 #include "nvlink_os.h"
27 #include "nvlink_linux.h"
28 #include "nvlink_errors.h"
29 #include "nvlink_export.h"
30 #include "nv-linux.h"
31 #include "nv-procfs.h"
32 #include "nv-time.h"
33 #include "nvlink_caps.h"
34
35 #include <linux/module.h>
36 #include <linux/major.h>
37 #include <linux/cdev.h>
38 #include <linux/device.h>
39 #include <linux/string.h>
40 #include <linux/mutex.h>
41
42 #define MAX_ERROR_STRING 512
43
44 typedef struct nvlink_file_private
45 {
46 struct
47 {
48 /* A duped file descriptor for fabric_mgmt capability */
49 int fabric_mgmt;
50 } capability_fds;
51 } nvlink_file_private_t;
52
53 #define NVLINK_SET_FILE_PRIVATE(filp, data) ((filp)->private_data = (data))
54 #define NVLINK_GET_FILE_PRIVATE(filp) ((nvlink_file_private_t *)(filp)->private_data)
55
56 typedef struct
57 {
58 struct mutex lock;
59 NvBool initialized;
60 struct cdev cdev;
61 dev_t devno;
62 int opened;
63 int major_devnum;
64 } _nvlink_drvctx;
65
66
67 // nvlink driver local state
68 static _nvlink_drvctx nvlink_drvctx;
69
70 #if defined(CONFIG_PROC_FS)
71 #define NV_DEFINE_SINGLE_NVLINK_PROCFS_FILE(name) \
72 NV_DEFINE_SINGLE_PROCFS_FILE_READ_ONLY(name, nv_system_pm_lock)
73 #endif
74
75 #define NVLINK_PROCFS_DIR "driver/nvidia-nvlink"
76
77 static struct proc_dir_entry *nvlink_procfs_dir = NULL;
78
79 #if defined(CONFIG_PROC_FS)
80 static int nvlink_is_procfs_available = 1;
81 #else
82 static int nvlink_is_procfs_available = 0;
83 #endif
84
85 static struct proc_dir_entry *nvlink_permissions = NULL;
86
nv_procfs_read_permissions(struct seq_file * s,void * v)87 static int nv_procfs_read_permissions(struct seq_file *s, void *v)
88 {
89 // Restrict device node permissions - 0666.
90 seq_printf(s, "%s: %u\n", "DeviceFileMode", 438);
91
92 return 0;
93 }
94
95 NV_DEFINE_SINGLE_NVLINK_PROCFS_FILE(permissions);
96
nvlink_permissions_exit(void)97 static void nvlink_permissions_exit(void)
98 {
99 if (!nvlink_permissions)
100 {
101 return;
102 }
103
104 proc_remove(nvlink_permissions);
105 nvlink_permissions = NULL;
106 }
107
nvlink_permissions_init(void)108 static int nvlink_permissions_init(void)
109 {
110 if (!nvlink_procfs_dir)
111 {
112 return -EACCES;
113 }
114
115 nvlink_permissions = NV_CREATE_PROC_FILE("permissions",
116 nvlink_procfs_dir,
117 permissions,
118 NULL);
119 if (!nvlink_permissions)
120 {
121 return -EACCES;
122 }
123
124 return 0;
125 }
126
nvlink_procfs_exit(void)127 static void nvlink_procfs_exit(void)
128 {
129 nvlink_permissions_exit();
130
131 if (!nvlink_procfs_dir)
132 {
133 return;
134 }
135
136 proc_remove(nvlink_procfs_dir);
137 nvlink_procfs_dir = NULL;
138 }
139
nvlink_procfs_init(void)140 static int nvlink_procfs_init(void)
141 {
142 int rc = 0;
143
144 if (!nvlink_is_procfs_available)
145 {
146 return -EACCES;
147 }
148
149 nvlink_procfs_dir = NV_CREATE_PROC_DIR(NVLINK_PROCFS_DIR, NULL);
150 if (!nvlink_procfs_dir)
151 {
152 return -EACCES;
153 }
154
155 rc = nvlink_permissions_init();
156 if (rc < 0)
157 {
158 goto cleanup;
159 }
160
161 return 0;
162
163 cleanup:
164
165 nvlink_procfs_exit();
166
167 return rc;
168 }
169
nvlink_fops_open(struct inode * inode,struct file * filp)170 static int nvlink_fops_open(struct inode *inode, struct file *filp)
171 {
172 int rc = 0;
173 nvlink_file_private_t *private = NULL;
174
175 nvlink_print(NVLINK_DBG_INFO, "nvlink driver open\n");
176
177 mutex_lock(&nvlink_drvctx.lock);
178
179 // nvlink lib driver is currently exclusive open.
180 if (nvlink_drvctx.opened)
181 {
182 rc = -EBUSY;
183 goto open_error;
184 }
185
186 private = (nvlink_file_private_t *)nvlink_malloc(sizeof(*private));
187 if (private == NULL)
188 {
189 rc = -ENOMEM;
190 goto open_error;
191 }
192
193 private->capability_fds.fabric_mgmt = -1;
194 NVLINK_SET_FILE_PRIVATE(filp, private);
195
196 // mark our state as opened
197 nvlink_drvctx.opened = NV_TRUE;
198
199 open_error:
200 mutex_unlock(&nvlink_drvctx.lock);
201 return rc;
202 }
203
nvlink_fops_release(struct inode * inode,struct file * filp)204 static int nvlink_fops_release(struct inode *inode, struct file *filp)
205 {
206 nvlink_file_private_t *private = NVLINK_GET_FILE_PRIVATE(filp);
207
208 nvlink_print(NVLINK_DBG_INFO, "nvlink driver close\n");
209
210 if (private == NULL)
211 return -ENOMEM;
212
213 mutex_lock(&nvlink_drvctx.lock);
214
215 if (private->capability_fds.fabric_mgmt > 0)
216 {
217 nvlink_cap_release(private->capability_fds.fabric_mgmt);
218 private->capability_fds.fabric_mgmt = -1;
219 }
220
221 nvlink_free(filp->private_data);
222 NVLINK_SET_FILE_PRIVATE(filp, NULL);
223
224 // mark the device as not opened
225 nvlink_drvctx.opened = NV_FALSE;
226
227 mutex_unlock(&nvlink_drvctx.lock);
228
229 return 0;
230 }
231
nvlink_fops_ioctl(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)232 static int nvlink_fops_ioctl(struct inode *inode,
233 struct file *filp,
234 unsigned int cmd,
235 unsigned long arg)
236 {
237 nvlink_ioctrl_params ctrl_params = {0};
238 int param_size = _IOC_SIZE(cmd);
239 void *param_buf = NULL;
240 NvlStatus ret_val = 0;
241 int rc = 0;
242
243 // no buffer for simple _IO types
244 if (param_size)
245 {
246 // allocate a buffer to hold user input
247 param_buf = kzalloc(param_size, GFP_KERNEL);
248 if (NULL == param_buf)
249 {
250 rc = -ENOMEM;
251 goto nvlink_ioctl_fail;
252 }
253
254 // copy user input to kernel buffers. Simple _IOR() ioctls can skip this step.
255 if (_IOC_DIR(cmd) & _IOC_WRITE)
256 {
257 // copy user input to local buffer
258 if (copy_from_user(param_buf, (const void *)arg, param_size))
259 {
260 rc = -EFAULT;
261 goto nvlink_ioctl_fail;
262 }
263 }
264 }
265
266 ctrl_params.osPrivate = filp->private_data;
267 ctrl_params.cmd = _IOC_NR(cmd);
268 ctrl_params.buf = param_buf;
269 ctrl_params.size = param_size;
270
271 ret_val = nvlink_lib_ioctl_ctrl(&ctrl_params);
272 if (NVL_SUCCESS != ret_val)
273 {
274 rc = -EINVAL;
275 goto nvlink_ioctl_fail;
276 }
277
278 // no copy for write-only ioctl
279 if ((param_size) && (_IOC_DIR(cmd) & _IOC_READ))
280 {
281 if (copy_to_user((void *)arg, ctrl_params.buf, ctrl_params.size))
282 {
283 rc = -EFAULT;
284 goto nvlink_ioctl_fail;
285 }
286 }
287
288 nvlink_ioctl_fail:
289 if (param_buf)
290 {
291 kfree(param_buf);
292 }
293 return rc;
294 }
295
296 #define NV_FILE_INODE(file) (file)->f_inode
297
nvlink_fops_unlocked_ioctl(struct file * file,unsigned int cmd,unsigned long arg)298 static long nvlink_fops_unlocked_ioctl(struct file *file,
299 unsigned int cmd,
300 unsigned long arg)
301 {
302 return nvlink_fops_ioctl(NV_FILE_INODE(file), file, cmd, arg);
303 }
304
305
306 static const struct file_operations nvlink_fops = {
307 .owner = THIS_MODULE,
308 .open = nvlink_fops_open,
309 .release = nvlink_fops_release,
310 .unlocked_ioctl = nvlink_fops_unlocked_ioctl,
311 };
312
nvlink_core_init(void)313 int __init nvlink_core_init(void)
314 {
315 NvlStatus ret_val;
316 int rc;
317
318 if (NV_TRUE == nvlink_drvctx.initialized)
319 {
320 nvlink_print(NVLINK_DBG_ERRORS, "nvlink core interface already initialized\n");
321 return -EBUSY;
322 }
323
324 mutex_init(&nvlink_drvctx.lock);
325
326 ret_val = nvlink_lib_initialize();
327 if (NVL_SUCCESS != ret_val)
328 {
329 nvlink_print(NVLINK_DBG_ERRORS, "Failed to initialize driver : %d\n", ret_val);
330 rc = -ENODEV;
331 goto nvlink_lib_initialize_fail;
332 }
333
334 rc = alloc_chrdev_region(&nvlink_drvctx.devno, 0, NVLINK_NUM_MINOR_DEVICES,
335 NVLINK_DEVICE_NAME);
336 if (rc < 0)
337 {
338 nvlink_print(NVLINK_DBG_ERRORS, "alloc_chrdev_region failed: %d\n", rc);
339 goto alloc_chrdev_region_fail;
340 }
341
342 nvlink_drvctx.major_devnum = MAJOR(nvlink_drvctx.devno);
343 nvlink_print(NVLINK_DBG_INFO, "Nvlink Core is being initialized, major device number %d\n",
344 nvlink_drvctx.major_devnum);
345
346 cdev_init(&nvlink_drvctx.cdev, &nvlink_fops);
347 nvlink_drvctx.cdev.owner = THIS_MODULE;
348 rc = cdev_add(&nvlink_drvctx.cdev, nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES);
349 if (rc < 0)
350 {
351 nvlink_print(NVLINK_DBG_ERRORS, " Unable to create cdev\n");
352 goto cdev_add_fail;
353 }
354
355 rc = nvlink_procfs_init();
356 if (rc < 0)
357 {
358 goto procfs_init_fail;
359 }
360
361 rc = nvlink_cap_init(NVLINK_PROCFS_DIR);
362 if (rc < 0)
363 {
364 nvlink_print(NVLINK_DBG_ERRORS, " Unable to create capability\n");
365 goto cap_init_fail;
366 }
367
368 nvlink_drvctx.initialized = NV_TRUE;
369
370 return 0;
371
372 cap_init_fail:
373 nvlink_procfs_exit();
374
375 procfs_init_fail:
376 cdev_del(&nvlink_drvctx.cdev);
377
378 cdev_add_fail:
379 unregister_chrdev_region(nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES);
380
381 alloc_chrdev_region_fail:
382 nvlink_lib_unload();
383
384 nvlink_lib_initialize_fail:
385 nv_mutex_destroy(&nvlink_drvctx.lock);
386 return rc;
387 }
388
nvlink_core_exit(void)389 void nvlink_core_exit(void)
390 {
391 if (NV_FALSE == nvlink_drvctx.initialized)
392 {
393 return;
394 }
395
396 nvlink_cap_exit();
397
398 nvlink_procfs_exit();
399
400 cdev_del(&nvlink_drvctx.cdev);
401
402 unregister_chrdev_region(nvlink_drvctx.devno, NVLINK_NUM_MINOR_DEVICES);
403
404 nvlink_lib_unload();
405
406 nv_mutex_destroy(&nvlink_drvctx.lock);
407
408 nvlink_print(NVLINK_DBG_INFO, "Unregistered Nvlink Core, major device number %d\n",
409 nvlink_drvctx.major_devnum);
410 }
411
412 void
nvlink_print(const char * file,int line,const char * function,int log_level,const char * fmt,...)413 nvlink_print
414 (
415 const char *file,
416 int line,
417 const char *function,
418 int log_level,
419 const char *fmt,
420 ...
421 )
422 {
423 va_list arglist;
424 char nv_string[MAX_ERROR_STRING];
425 char *sys_log_level;
426
427 switch (log_level) {
428 case NVLINK_DBG_LEVEL_INFO:
429 sys_log_level = KERN_INFO;
430 break;
431 case NVLINK_DBG_LEVEL_SETUP:
432 sys_log_level = KERN_DEBUG;
433 break;
434 case NVLINK_DBG_LEVEL_USERERRORS:
435 sys_log_level = KERN_NOTICE;
436 break;
437 case NVLINK_DBG_LEVEL_WARNINGS:
438 sys_log_level = KERN_WARNING;
439 break;
440 case NVLINK_DBG_LEVEL_ERRORS:
441 sys_log_level = KERN_ERR;
442 break;
443 default:
444 sys_log_level = KERN_INFO;
445 break;
446 }
447
448 va_start(arglist, fmt);
449 vsnprintf(nv_string, sizeof(nv_string), fmt, arglist);
450 va_end(arglist);
451
452 nv_string[sizeof(nv_string) - 1] = '\0';
453 printk("%snvidia-nvlink: %s", sys_log_level, nv_string);
454 }
455
nvlink_malloc(NvLength size)456 void * nvlink_malloc(NvLength size)
457 {
458 return kmalloc(size, GFP_KERNEL);
459 }
460
nvlink_free(void * ptr)461 void nvlink_free(void *ptr)
462 {
463 return kfree(ptr);
464 }
465
nvlink_strcpy(char * dest,const char * src)466 char * nvlink_strcpy(char *dest, const char *src)
467 {
468 return strcpy(dest, src);
469 }
470
nvlink_strcmp(const char * dest,const char * src)471 int nvlink_strcmp(const char *dest, const char *src)
472 {
473 return strcmp(dest, src);
474 }
475
nvlink_strlen(const char * s)476 NvLength nvlink_strlen(const char *s)
477 {
478 return strlen(s);
479 }
480
nvlink_snprintf(char * dest,NvLength size,const char * fmt,...)481 int nvlink_snprintf(char *dest, NvLength size, const char *fmt, ...)
482 {
483 va_list arglist;
484 int chars_written;
485
486 va_start(arglist, fmt);
487 chars_written = vsnprintf(dest, size, fmt, arglist);
488 va_end(arglist);
489
490 return chars_written;
491 }
492
nvlink_memRd32(const volatile void * address)493 NvU32 nvlink_memRd32(const volatile void * address)
494 {
495 return (*(const volatile NvU32*)(address));
496 }
497
nvlink_memWr32(volatile void * address,NvU32 data)498 void nvlink_memWr32(volatile void *address, NvU32 data)
499 {
500 (*(volatile NvU32 *)(address)) = data;
501 }
502
nvlink_memRd64(const volatile void * address)503 NvU64 nvlink_memRd64(const volatile void * address)
504 {
505 return (*(const volatile NvU64 *)(address));
506 }
507
nvlink_memWr64(volatile void * address,NvU64 data)508 void nvlink_memWr64(volatile void *address, NvU64 data)
509 {
510 (*(volatile NvU64 *)(address)) = data;
511 }
512
nvlink_memset(void * dest,int value,NvLength size)513 void * nvlink_memset(void *dest, int value, NvLength size)
514 {
515 return memset(dest, value, size);
516 }
517
nvlink_memcpy(void * dest,const void * src,NvLength size)518 void * nvlink_memcpy(void *dest, const void *src, NvLength size)
519 {
520 return memcpy(dest, src, size);
521 }
522
nvlink_memcmp(const void * s1,const void * s2,NvLength size)523 int nvlink_memcmp(const void *s1, const void *s2, NvLength size)
524 {
525 return memcmp(s1, s2, size);
526 }
527
528 /*
529 * Sleep for specified milliseconds. Yields the CPU to scheduler.
530 */
nvlink_sleep(unsigned int ms)531 void nvlink_sleep(unsigned int ms)
532 {
533 NV_STATUS status;
534
535 status = nv_sleep_ms(ms);
536
537 if (status != NV_OK)
538 {
539 if (printk_ratelimit())
540 {
541 nvlink_print(NVLINK_DBG_ERRORS, "NVLink: requested sleep duration"
542 " %d msec exceeded %d msec\n",
543 ms, NV_MAX_ISR_DELAY_MS);
544 WARN_ON(1);
545 }
546 }
547 }
548
nvlink_assert(int cond)549 void nvlink_assert(int cond)
550 {
551 if ((cond) == 0x0)
552 {
553 if (printk_ratelimit())
554 {
555 nvlink_print(NVLINK_DBG_ERRORS, "NVLink: Assertion failed!\n");
556 WARN_ON(1);
557 }
558
559 dbg_breakpoint();
560 }
561 }
562
nvlink_allocLock(void)563 void * nvlink_allocLock(void)
564 {
565 struct semaphore *sema;
566
567 sema = nvlink_malloc(sizeof(*sema));
568 if (sema == NULL)
569 {
570 nvlink_print(NVLINK_DBG_ERRORS, "Failed to allocate sema!\n");
571 return NULL;
572 }
573 sema_init(sema, 1);
574
575 return sema;
576 }
577
nvlink_acquireLock(void * hLock)578 void nvlink_acquireLock(void *hLock)
579 {
580 down(hLock);
581 }
582
nvlink_releaseLock(void * hLock)583 void nvlink_releaseLock(void *hLock)
584 {
585 up(hLock);
586 }
587
nvlink_freeLock(void * hLock)588 void nvlink_freeLock(void *hLock)
589 {
590 if (NULL == hLock)
591 {
592 return;
593 }
594
595 NVLINK_FREE(hLock);
596 }
597
nvlink_isLockOwner(void * hLock)598 NvBool nvlink_isLockOwner(void *hLock)
599 {
600 return NV_TRUE;
601 }
602
nvlink_acquire_fabric_mgmt_cap(void * osPrivate,NvU64 capDescriptor)603 NvlStatus nvlink_acquire_fabric_mgmt_cap(void *osPrivate, NvU64 capDescriptor)
604 {
605 int dup_fd = -1;
606 nvlink_file_private_t *private_data = (nvlink_file_private_t *)osPrivate;
607
608 if (private_data == NULL)
609 {
610 return NVL_BAD_ARGS;
611 }
612
613 dup_fd = nvlink_cap_acquire((int)capDescriptor,
614 NVLINK_CAP_FABRIC_MANAGEMENT);
615 if (dup_fd < 0)
616 {
617 return NVL_ERR_OPERATING_SYSTEM;
618 }
619
620 private_data->capability_fds.fabric_mgmt = dup_fd;
621 return NVL_SUCCESS;
622 }
623
nvlink_is_fabric_manager(void * osPrivate)624 int nvlink_is_fabric_manager(void *osPrivate)
625 {
626 nvlink_file_private_t *private_data = (nvlink_file_private_t *)osPrivate;
627
628 /* Make sure that fabric mgmt capbaility fd is valid */
629 if ((private_data == NULL) ||
630 (private_data->capability_fds.fabric_mgmt < 0))
631 {
632 return 0;
633 }
634
635 return 1;
636 }
637
nvlink_is_admin(void)638 int nvlink_is_admin(void)
639 {
640 return NV_IS_SUSER();
641 }
642
nvlink_get_platform_time(void)643 NvU64 nvlink_get_platform_time(void)
644 {
645 return nv_ktime_get_raw_ns();
646 }
647