1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24
25 #include "core/core.h"
26 #include "gpu/gpu.h"
27 #include "os/os.h"
28 #include "lib/base_utils.h"
29 #include "lib/zlib/inflate.h"
30 #include "nvRmReg.h"
31 #include "virtualization/hypervisor/hypervisor.h"
32
33 /**
34 * @brief Changes the user-space permissions for a given register address range
35 *
36 * @param pGpu
37 * @param[in] offset byte address of register address range start
38 * @param[in] size size in bytes of register address range
39 * @param[in] bAllow whether or not to allow register access from user space
40 *
41 * @return NV_OK if success, error otherwise
42 */
43 NV_STATUS
gpuSetUserRegisterAccessPermissions_IMPL(OBJGPU * pGpu,NvU32 offset,NvU32 size,NvBool bAllow)44 gpuSetUserRegisterAccessPermissions_IMPL(OBJGPU *pGpu, NvU32 offset, NvU32 size, NvBool bAllow)
45 {
46 NvU32 mapSize = pGpu->userRegisterAccessMapSize * 8; // total number of bits
47 NvU32 bitOffset;
48 NvU32 bitSize;
49
50 NV_ASSERT_OR_RETURN(pGpu->pUserRegisterAccessMap != NULL, NV_ERR_INVALID_STATE);
51 NV_ASSERT_OR_RETURN((offset & 3) == 0, NV_ERR_INVALID_ARGUMENT);
52 NV_ASSERT_OR_RETURN((size & 3) == 0, NV_ERR_INVALID_ARGUMENT);
53
54 NV_PRINTF(LEVEL_INFO, "%sllowing access to 0x%x-0x%x\n",
55 (bAllow ? "A" : "Disa"), offset, (offset + size - 1));
56
57 NV_PRINTF(LEVEL_INFO, "Byte 0x%x Bit 0x%x through Byte 0x%x Bit 0x%x\n",
58 offset / 4 / 8, offset / 4 % 8, (offset + size) / 4 / 8,
59 (offset + size - 1) / 4 % 8);
60
61 bitOffset = offset/sizeof(NvU32);
62 bitSize = size/sizeof(NvU32);
63
64 NV_ASSERT_OR_RETURN(bitOffset < mapSize, NV_ERR_INVALID_ARGUMENT);
65 NV_ASSERT_OR_RETURN((bitOffset+bitSize) <= mapSize, NV_ERR_INVALID_ARGUMENT);
66
67 // Deal with bits up to first byte.
68 for (; bitOffset%8 != 0 && bitSize; bitOffset++, bitSize--)
69 {
70 nvBitFieldSet((NvU32*) pGpu->pUserRegisterAccessMap,
71 pGpu->userRegisterAccessMapSize / sizeof(NvU32),
72 bitOffset, bAllow);
73 }
74
75 if (!bitSize)
76 return NV_OK;
77
78 // Deal with any full bytes.
79 portMemSet(pGpu->pUserRegisterAccessMap + bitOffset/8, bAllow ? 0xff : 0x0, NV_ALIGN_DOWN(bitSize, 8)/8);
80
81 bitOffset += NV_ALIGN_DOWN(bitSize, 8);
82 bitSize -= NV_ALIGN_DOWN(bitSize, 8);
83
84 // Any remaining bits
85 for (; bitSize; bitOffset++, bitSize--)
86 {
87 nvBitFieldSet((NvU32*) pGpu->pUserRegisterAccessMap,
88 pGpu->userRegisterAccessMapSize / sizeof(NvU32),
89 bitOffset, bAllow);
90 }
91
92 return NV_OK;
93 }
94
95 /**
96 * @brief Changes the user-space permissions for the given (in bulk) register address ranges
97 *
98 * @param pGpu
99 * @param[in] pOffsetsSizesArr flat array of (register offset, register size in bytes) pairs
100 * @param[in] arrSizeBytes size in bytes of the pOffsetsSizesArr array
101 * @param[in] bAllow whether or not to allow register access from user space
102 *
103 * @return NV_OK if success, error otherwise
104 */
105 NV_STATUS
gpuSetUserRegisterAccessPermissionsInBulk_IMPL(OBJGPU * pGpu,const NvU32 * pOffsetsSizesArr,NvU32 arrSizeBytes,NvBool bAllow)106 gpuSetUserRegisterAccessPermissionsInBulk_IMPL(OBJGPU *pGpu, const NvU32 *pOffsetsSizesArr,
107 NvU32 arrSizeBytes, NvBool bAllow)
108 {
109 NV_ASSERT_OR_RETURN((arrSizeBytes & (2 * sizeof(NvU32) - 1)) == 0, NV_ERR_INVALID_ARGUMENT);
110 NvU32 numElements = arrSizeBytes / sizeof(NvU32);
111
112 NvU32 i;
113 NV_STATUS status;
114 for (i = 0; i < numElements; i += 2)
115 {
116 status = gpuSetUserRegisterAccessPermissions(pGpu,
117 pOffsetsSizesArr[i], pOffsetsSizesArr[i + 1], bAllow);
118
119 if (status != NV_OK)
120 {
121 return status;
122 }
123 }
124
125 return NV_OK;
126 }
127
128 /**
129 * @brief returns if a given register address can be accessed from userspace.
130 *
131 * @param pGpu
132 * @param[in] offset Register offset to test, must be dword aligned.
133 *
134 * @return NV_TRUE if register is accessible, NV_FALSE if not.
135 */
136 NvBool
gpuGetUserRegisterAccessPermissions_IMPL(OBJGPU * pGpu,NvU32 offset)137 gpuGetUserRegisterAccessPermissions_IMPL(OBJGPU *pGpu, NvU32 offset)
138 {
139 NvU32 bitOffset = offset / sizeof(NvU32);
140
141 if (!pGpu->pUserRegisterAccessMap)
142 {
143 //
144 // If very early in the init sequence, everything is accessible, since
145 // we can't have gotten any user originating accesses yet.
146 //
147 if (!gpuIsFullyConstructed(pGpu))
148 return NV_TRUE;
149
150 NV_ASSERT_FAILED("No user register access map available to read");
151 return NV_FALSE;
152 }
153
154 if (bitOffset >= (pGpu->userRegisterAccessMapSize * 8))
155 {
156 NV_PRINTF(LEVEL_ERROR, "Parameter `offset` = %u is out of bounds.\n",
157 offset);
158 return NV_FALSE;
159 }
160
161 if ((offset % 4) != 0)
162 {
163 NV_PRINTF(LEVEL_ERROR,
164 "Parameter `offset` = %u must be 4-byte aligned.\n", offset);
165 return NV_FALSE;
166 }
167
168 // pGpu->pUserRegisterAccessMap is pageable, must not be at raised IRQ
169 NV_ASSERT_OR_RETURN(!osIsRaisedIRQL(), NV_ERR_INVALID_IRQ_LEVEL);
170
171 return nvBitFieldTest((NvU32*) pGpu->pUserRegisterAccessMap, pGpu->userRegisterAccessMapSize / sizeof(NvU32), bitOffset);
172 }
173
174
_getIsProfilingPrivileged(OBJGPU * pGpu)175 static NvBool _getIsProfilingPrivileged(OBJGPU *pGpu)
176 {
177 // On a vGPU Host, RmProfilingAdminOnly is always set to 1
178 if (hypervisorIsVgxHyper())
179 {
180 //
181 // Setting the value at this point to make the behavior same for
182 // debug/develop/release drivers on vGPU host.
183 //
184 return NV_TRUE;
185 }
186 #if defined(DEBUG) || defined(DEVELOP)
187 return NV_FALSE;
188 #else
189 NvU32 data32;
190 if (NV_OK == osReadRegistryDword(pGpu, NV_REG_STR_RM_PROFILING_ADMIN_ONLY, &data32))
191 {
192 return (data32 == NV_REG_STR_RM_PROFILING_ADMIN_ONLY_TRUE);
193 }
194
195 return NV_TRUE;
196 #endif
197 }
198 /**
199 * @brief Constructs the bitmap used to control whether a register can be accessed by user space.
200 *
201 * Bitmap contains a single bit per 32b register.
202 *
203 * @param pGpu
204 *
205 * @return NV_OK if success, error otherwise
206 */
207 NV_STATUS
gpuConstructUserRegisterAccessMap_IMPL(OBJGPU * pGpu)208 gpuConstructUserRegisterAccessMap_IMPL(OBJGPU *pGpu)
209 {
210 NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS *pParams = NULL;
211 NV_STATUS status = NV_OK;
212 NvU32 compressedSize = 0;
213 NvU32 profilingRangesSize = 0;
214 const NvU8 *compressedData = NULL;
215 const NvU32 *profilingRangesArr = NULL;
216
217 if (IS_VIRTUAL(pGpu))
218 {
219 // Usermode access maps unused in Guest RM. Initialize this boolean and leave.
220 pGpu->bRmProfilingPrivileged = _getIsProfilingPrivileged(pGpu);
221 return NV_OK;
222 }
223
224 NV_ASSERT(pGpu->userRegisterAccessMapSize == 0);
225
226 {
227 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
228 pParams = portMemAllocPaged(sizeof(*pParams));
229 NV_ASSERT_OR_RETURN(pParams != NULL, NV_ERR_NO_MEMORY);
230 portMemSet(pParams, 0, sizeof(*pParams));
231
232 NV_ASSERT_OK_OR_GOTO(status,
233 pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
234 NV2080_CTRL_CMD_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP,
235 pParams, sizeof(*pParams)), done);
236
237 pGpu->userRegisterAccessMapSize = pParams->userRegisterAccessMapSize;
238 compressedSize = pParams->compressedSize;
239 profilingRangesSize = pParams->profilingRangesSize;
240 compressedData = (const NvU8*)pParams->compressedData;
241 profilingRangesArr = (const NvU32*)pParams->profilingRanges;
242 }
243
244 //
245 // We round up to a 32b multiple to be used with bitfield helpers.
246 // (Of course it should already be a 32b multiple, but just to be sure.)
247 //
248 pGpu->userRegisterAccessMapSize = NV_ALIGN_UP(pGpu->userRegisterAccessMapSize, sizeof(NvU32));
249 if (pGpu->userRegisterAccessMapSize == 0)
250 {
251 NV_PRINTF(LEVEL_INFO,
252 "User Register Access Map unsupported for this chip.\n");
253 status = NV_OK;
254 goto done;
255 }
256
257 pGpu->pUserRegisterAccessMap = portMemAllocPaged(pGpu->userRegisterAccessMapSize);
258 if (pGpu->pUserRegisterAccessMap == NULL)
259 {
260 status = NV_ERR_NO_MEMORY;
261 goto done;
262 }
263
264 pGpu->pUnrestrictedRegisterAccessMap = portMemAllocPaged(pGpu->userRegisterAccessMapSize);
265 if (pGpu->pUnrestrictedRegisterAccessMap == NULL)
266 {
267 status = NV_ERR_NO_MEMORY;
268 goto done;
269 }
270
271 NV_PRINTF(LEVEL_INFO, "Allocated User Register Access Map of 0x%xB @%p\n",
272 pGpu->userRegisterAccessMapSize, pGpu->pUserRegisterAccessMap);
273
274 if (!pGpu->bUseRegisterAccessMap || compressedSize == 0)
275 {
276 NV_PRINTF(LEVEL_INFO,
277 "GPU/Platform does not have restricted user register access! Allowing all registers.\n");
278 portMemSet(pGpu->pUserRegisterAccessMap, 0xFF, pGpu->userRegisterAccessMapSize);
279 }
280 else
281 {
282 NV_ASSERT_OK_OR_GOTO(status,
283 gpuInitRegisterAccessMap(pGpu, pGpu->pUserRegisterAccessMap,
284 pGpu->userRegisterAccessMapSize, compressedData, compressedSize), done);
285 }
286
287 // copy permissions from user access map
288 if (portMemCopy(pGpu->pUnrestrictedRegisterAccessMap, pGpu->userRegisterAccessMapSize,
289 pGpu->pUserRegisterAccessMap, pGpu->userRegisterAccessMapSize) == NULL)
290 {
291 NV_PRINTF(LEVEL_ERROR,
292 "Failed to initialize unrestricted register access map\n");
293 status = NV_ERR_INVALID_ADDRESS;
294 goto done;
295 }
296
297 pGpu->bRmProfilingPrivileged = _getIsProfilingPrivileged(pGpu);
298 if (pGpu->bRmProfilingPrivileged && profilingRangesSize > 0)
299 {
300 // remove profiling registers from user map
301 status = gpuSetUserRegisterAccessPermissionsInBulk(
302 pGpu, profilingRangesArr, profilingRangesSize, NV_FALSE);
303 if (status != NV_OK)
304 {
305 pGpu->bRmProfilingPrivileged = NV_FALSE;
306 goto done;
307 }
308 }
309
310 done:
311 if (status != NV_OK)
312 {
313 portMemFree(pGpu->pUserRegisterAccessMap);
314 pGpu->pUserRegisterAccessMap = NULL;
315
316 portMemFree(pGpu->pUnrestrictedRegisterAccessMap);
317 pGpu->pUnrestrictedRegisterAccessMap = NULL;
318
319 pGpu->userRegisterAccessMapSize = 0;
320 }
321 portMemFree(pParams);
322
323 return status;
324 }
325
326
327 /**
328 * @brief Initializes the register access map
329 *
330 * Extracts compressed data representing access map.
331 *
332 * @param pGpu
333 *
334 * @return NV_OK if success, error otherwise
335 */
336 NV_STATUS
gpuInitRegisterAccessMap_IMPL(OBJGPU * pGpu,NvU8 * pAccessMap,NvU32 accessMapSize,const NvU8 * pComprData,const NvU32 comprDataSize)337 gpuInitRegisterAccessMap_IMPL(OBJGPU *pGpu, NvU8 *pAccessMap, NvU32 accessMapSize, const NvU8 *pComprData, const NvU32 comprDataSize)
338 {
339 PGZ_INFLATE_STATE pGzState = NULL;
340 NvU32 inflatedBytes = 0;
341
342 NV_ASSERT_OR_RETURN(pAccessMap != NULL, NV_ERR_INVALID_STATE);
343 NV_ASSERT_OR_RETURN(accessMapSize != 0, NV_ERR_INVALID_STATE);
344 NV_ASSERT_OR_RETURN(pComprData != NULL, NV_ERR_INVALID_STATE);
345 NV_ASSERT_OR_RETURN(comprDataSize != 0, NV_ERR_INVALID_STATE);
346
347 //
348 // Strip off gzlib 10-byte header
349 // XXX this really belongs in the RM GZ library
350 //
351 pComprData += 10;
352
353 NV_ASSERT_OK_OR_RETURN(utilGzAllocate((NvU8*)pComprData, accessMapSize, &pGzState));
354
355 NV_ASSERT(pGzState);
356
357 inflatedBytes = utilGzGetData(pGzState, 0, accessMapSize, pAccessMap);
358
359 utilGzDestroy(pGzState);
360
361 if (inflatedBytes != accessMapSize)
362 {
363 NV_PRINTF(LEVEL_ERROR,
364 "failed to get inflated data, got %u bytes, expecting %u\n",
365 inflatedBytes, pGpu->userRegisterAccessMapSize);
366 DBG_BREAKPOINT();
367 return NV_ERR_INFLATE_COMPRESSED_DATA_FAILED;
368 }
369 return NV_OK;
370 }
371