1 /*******************************************************************************
2     Copyright (c) 2015-2022 NVIDIA Corporation
3 
4     Permission is hereby granted, free of charge, to any person obtaining a copy
5     of this software and associated documentation files (the "Software"), to
6     deal in the Software without restriction, including without limitation the
7     rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8     sell copies of the Software, and to permit persons to whom the Software is
9     furnished to do so, subject to the following conditions:
10 
11         The above copyright notice and this permission notice shall be
12         included in all copies or substantial portions of the Software.
13 
14     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15     IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16     FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17     THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20     DEALINGS IN THE SOFTWARE.
21 
22 *******************************************************************************/
23 
24 #include "uvm_rm_mem.h"
25 #include "uvm_test.h"
26 #include "uvm_test_ioctl.h"
27 #include "uvm_va_space.h"
28 #include "uvm_kvmalloc.h"
29 
30 static NV_STATUS map_cpu(uvm_rm_mem_t *rm_mem)
31 {
32     void *cpu_va;
33 
34     // Map
35     TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem));
36     TEST_CHECK_RET(uvm_rm_mem_mapped_on_cpu(rm_mem));
37 
38     // Mapping if already mapped is OK
39     TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem));
40 
41     // Unmap
42     uvm_rm_mem_unmap_cpu(rm_mem);
43 
44     // Unmapping already unmapped also OK
45     uvm_rm_mem_unmap_cpu(rm_mem);
46 
47     // Map again
48     TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem));
49 
50     cpu_va = uvm_rm_mem_get_cpu_va(rm_mem);
51     TEST_CHECK_RET(cpu_va != NULL);
52 
53     // Check that the CPU VA is writable.
54     // memset and memcpy might cause alignment faults on aarch64.
55     // See Bug 2668765 for more details. Since this is a test ioctl and
56     // therefore not a critical performance path, it's safe to use memset_io on
57     // all platforms.
58     memset_io(cpu_va, 0, rm_mem->size);
59 
60     return NV_OK;
61 }
62 
63 static NV_STATUS check_alignment(uvm_rm_mem_t *rm_mem, uvm_gpu_t *gpu, NvU64 alignment)
64 {
65     // Alignment requirements only apply to mappings in the UVM-owned VA space
66     if (alignment != 0) {
67         NvU64 gpu_uvm_va = uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu);
68 
69         TEST_CHECK_RET(IS_ALIGNED(gpu_uvm_va, alignment));
70     }
71 
72     return NV_OK;
73 }
74 
75 static NV_STATUS map_gpu_owner(uvm_rm_mem_t *rm_mem, NvU64 alignment)
76 {
77     uvm_gpu_t *gpu = rm_mem->gpu_owner;
78     NvU64 gpu_uvm_va;
79     NvU64 gpu_proxy_va = 0;
80 
81     // The memory should have been automatically mapped in the GPU owner
82     TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu));
83 
84     gpu_uvm_va = uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu);
85 
86     // In SR-IOV heavy, there are two VA spaces per GPU, so there are two
87     // mappings for a single rm_mem object on a GPU, even if the memory is
88     // located in vidmem.
89     if (uvm_gpu_uses_proxy_channel_pool(gpu)) {
90         TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu));
91 
92         gpu_proxy_va = uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu);
93     }
94     else {
95         TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu));
96     }
97 
98     TEST_NV_CHECK_RET(check_alignment(rm_mem, gpu, alignment));
99 
100     // Mappings are not ref counted, so additional map calls are no-ops; the
101     // GPU VA should remain the same for all the applicable VA spaces.
102     TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment));
103 
104     TEST_CHECK_RET(gpu_uvm_va == uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu));
105 
106     if (uvm_gpu_uses_proxy_channel_pool(gpu))
107         TEST_CHECK_RET(gpu_proxy_va == uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu));
108 
109     // Unmapping the GPU owner is a no-op
110     uvm_rm_mem_unmap_gpu(rm_mem, gpu);
111 
112     TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu));
113     TEST_CHECK_RET(gpu_uvm_va == uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu));
114 
115     if (uvm_gpu_uses_proxy_channel_pool(gpu)) {
116         TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu));
117         TEST_CHECK_RET(gpu_proxy_va == uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu));
118     }
119     else {
120         TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu));
121     }
122 
123     return NV_OK;
124 }
125 
126 static NV_STATUS map_other_gpus(uvm_rm_mem_t *rm_mem, uvm_va_space_t *va_space, NvU64 alignment)
127 {
128     uvm_gpu_t *gpu_owner = rm_mem->gpu_owner;
129     uvm_gpu_t *gpu;
130 
131     for_each_va_space_gpu(gpu, va_space) {
132         if (gpu == gpu_owner)
133             continue;
134 
135         TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment));
136         TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu));
137 
138         // Mappings are not ref counted, so additional map calls are no-ops
139         TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment));
140 
141         // The previous GPU map calls added mappings to the proxy VA space
142         // when in SR-IOV heavy mode
143         TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu) == uvm_gpu_uses_proxy_channel_pool(gpu));
144 
145         // Unmapping removes all mappings
146         uvm_rm_mem_unmap_gpu(rm_mem, gpu);
147         TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu(rm_mem, gpu));
148         TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu));
149 
150         // Additional unmappings are no-ops
151         uvm_rm_mem_unmap_gpu(rm_mem, gpu);
152         TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu(rm_mem, gpu));
153         TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu));
154 
155         // Subsequent mappings should behave as they did in the beginning.
156         TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment));
157         TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu));
158 
159         TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu) == uvm_gpu_uses_proxy_channel_pool(gpu));
160 
161         TEST_NV_CHECK_RET(check_alignment(rm_mem, gpu, alignment));
162     }
163 
164     return NV_OK;
165 }
166 
167 static NV_STATUS test_all_gpus_in_va(uvm_va_space_t *va_space)
168 {
169     uvm_gpu_t *gpu;
170     uvm_rm_mem_t *rm_mem = NULL;
171     NV_STATUS status = NV_OK;
172 
173     // Create allocations of these types
174     static const uvm_rm_mem_type_t mem_types[] = { UVM_RM_MEM_TYPE_SYS, UVM_RM_MEM_TYPE_GPU };
175 
176     // Create allocations of these sizes
177     static const size_t sizes[] = { 1, 4, 16, 128, 1024, 4096, 1024 * 1024, 4 * 1024 * 1024 };
178     static const NvU64 alignments[] = { 0,
179                                         8,
180                                         UVM_PAGE_SIZE_4K >> 1,
181                                         UVM_PAGE_SIZE_4K,
182                                         UVM_PAGE_SIZE_4K << 1,
183                                         UVM_PAGE_SIZE_64K,
184                                         UVM_PAGE_SIZE_2M,
185                                         UVM_PAGE_SIZE_2M << 3,
186                                         UVM_PAGE_SIZE_2M << 5 };
187 
188     uvm_assert_rwsem_locked(&va_space->lock);
189 
190     TEST_CHECK_RET(!uvm_processor_mask_empty(&va_space->registered_gpus));
191 
192     for_each_va_space_gpu(gpu, va_space) {
193         int i, j, k;
194 
195         for (i = 0; i < ARRAY_SIZE(sizes); ++i) {
196             for (j = 0; j < ARRAY_SIZE(mem_types); ++j) {
197                 for (k = 0; k < ARRAY_SIZE(alignments); ++k) {
198 
199                     // Create an allocation in the GPU's address space
200                     TEST_NV_CHECK_RET(uvm_rm_mem_alloc(gpu, mem_types[j], sizes[i], alignments[k], &rm_mem));
201 
202                     // Test CPU mappings
203                     TEST_NV_CHECK_GOTO(map_cpu(rm_mem), error);
204 
205                     // Test mappings in the GPU owning the allocation
206                     TEST_NV_CHECK_GOTO(map_gpu_owner(rm_mem, alignments[k]), error);
207 
208                     // For sysmem allocations, test mappings on all other GPUs
209                     if (rm_mem->type == UVM_RM_MEM_TYPE_SYS)
210                         TEST_NV_CHECK_GOTO(map_other_gpus(rm_mem, va_space, alignments[k]), error);
211 
212                     uvm_rm_mem_free(rm_mem);
213                 }
214             }
215         }
216     }
217 
218     return NV_OK;
219 
220 error:
221     uvm_rm_mem_free(rm_mem);
222 
223     return status;
224 }
225 
226 NV_STATUS uvm_test_rm_mem_sanity(UVM_TEST_RM_MEM_SANITY_PARAMS *params, struct file *filp)
227 {
228     NV_STATUS status;
229     uvm_va_space_t *va_space = uvm_va_space_get(filp);
230 
231     uvm_va_space_down_read_rm(va_space);
232 
233     status = test_all_gpus_in_va(va_space);
234 
235     uvm_va_space_up_read_rm(va_space);
236 
237     return status;
238 }
239