1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 
25 #include "core/core.h"
26 #include "gpu/gpu.h"
27 #include "os/os.h"
28 #include "lib/base_utils.h"
29 #include "lib/zlib/inflate.h"
30 #include "nvRmReg.h"
31 #include "virtualization/hypervisor/hypervisor.h"
32 
33 /**
34  * @brief Changes the user-space permissions for a given register address range
35  *
36  * @param pGpu
37  * @param[in] offset  byte address of register address range start
38  * @param[in] size  size in bytes of register address range
39  * @param[in] bAllow whether or not to allow register access from user space
40  *
41  * @return NV_OK if success, error otherwise
42  */
43 NV_STATUS
gpuSetUserRegisterAccessPermissions_IMPL(OBJGPU * pGpu,NvU32 offset,NvU32 size,NvBool bAllow)44 gpuSetUserRegisterAccessPermissions_IMPL(OBJGPU *pGpu, NvU32 offset, NvU32 size, NvBool bAllow)
45 {
46     NvU32 mapSize = pGpu->userRegisterAccessMapSize * 8;  // total number of bits
47     NvU32 bitOffset;
48     NvU32 bitSize;
49 
50     NV_ASSERT_OR_RETURN(pGpu->pUserRegisterAccessMap != NULL, NV_ERR_INVALID_STATE);
51     NV_ASSERT_OR_RETURN((offset & 3) == 0, NV_ERR_INVALID_ARGUMENT);
52     NV_ASSERT_OR_RETURN((size & 3) == 0, NV_ERR_INVALID_ARGUMENT);
53 
54     NV_PRINTF(LEVEL_INFO, "%sllowing access to 0x%x-0x%x\n",
55               (bAllow ? "A" : "Disa"), offset, (offset + size - 1));
56 
57     NV_PRINTF(LEVEL_INFO, "Byte 0x%x Bit 0x%x through Byte 0x%x Bit 0x%x\n",
58               offset / 4 / 8, offset / 4 % 8, (offset + size) / 4 / 8,
59               (offset + size - 1) / 4 % 8);
60 
61     bitOffset = offset/sizeof(NvU32);
62     bitSize = size/sizeof(NvU32);
63 
64     NV_ASSERT_OR_RETURN(bitOffset < mapSize, NV_ERR_INVALID_ARGUMENT);
65     NV_ASSERT_OR_RETURN((bitOffset+bitSize) <= mapSize, NV_ERR_INVALID_ARGUMENT);
66 
67     // Deal with bits  up to first byte.
68     for (; bitOffset%8 != 0 && bitSize; bitOffset++, bitSize--)
69     {
70         nvBitFieldSet((NvU32*) pGpu->pUserRegisterAccessMap,
71                 pGpu->userRegisterAccessMapSize / sizeof(NvU32),
72                 bitOffset, bAllow);
73     }
74 
75     if (!bitSize)
76         return NV_OK;
77 
78     // Deal with any full bytes.
79     portMemSet(pGpu->pUserRegisterAccessMap + bitOffset/8, bAllow ? 0xff : 0x0, NV_ALIGN_DOWN(bitSize, 8)/8);
80 
81     bitOffset += NV_ALIGN_DOWN(bitSize, 8);
82     bitSize -= NV_ALIGN_DOWN(bitSize, 8);
83 
84     // Any remaining bits
85     for (; bitSize; bitOffset++, bitSize--)
86     {
87         nvBitFieldSet((NvU32*) pGpu->pUserRegisterAccessMap,
88                 pGpu->userRegisterAccessMapSize / sizeof(NvU32),
89                 bitOffset, bAllow);
90     }
91 
92     return NV_OK;
93 }
94 
95 /**
96  * @brief Changes the user-space permissions for the given (in bulk) register address ranges
97  *
98  * @param pGpu
99  * @param[in] pOffsetsSizesArr  flat array of (register offset, register size in bytes) pairs
100  * @param[in] arrSizeBytes  size in bytes of the pOffsetsSizesArr array
101  * @param[in] bAllow  whether or not to allow register access from user space
102  *
103  * @return NV_OK if success, error otherwise
104  */
105 NV_STATUS
gpuSetUserRegisterAccessPermissionsInBulk_IMPL(OBJGPU * pGpu,const NvU32 * pOffsetsSizesArr,NvU32 arrSizeBytes,NvBool bAllow)106 gpuSetUserRegisterAccessPermissionsInBulk_IMPL(OBJGPU *pGpu, const NvU32 *pOffsetsSizesArr,
107                                                NvU32 arrSizeBytes, NvBool bAllow)
108 {
109     NV_ASSERT_OR_RETURN((arrSizeBytes & (2 * sizeof(NvU32) - 1)) == 0, NV_ERR_INVALID_ARGUMENT);
110     NvU32 numElements = arrSizeBytes / sizeof(NvU32);
111 
112     NvU32 i;
113     NV_STATUS status;
114     for (i = 0; i < numElements; i += 2)
115     {
116         status = gpuSetUserRegisterAccessPermissions(pGpu,
117                     pOffsetsSizesArr[i], pOffsetsSizesArr[i + 1], bAllow);
118 
119         if (status != NV_OK)
120         {
121             return status;
122         }
123     }
124 
125     return NV_OK;
126 }
127 
128 /**
129  * @brief returns if a given register address can be accessed from userspace.
130  *
131  * @param pGpu
132  * @param[in]  offset Register offset to test, must be dword aligned.
133  *
134  * @return NV_TRUE if register is accessible, NV_FALSE if not.
135  */
136 NvBool
gpuGetUserRegisterAccessPermissions_IMPL(OBJGPU * pGpu,NvU32 offset)137 gpuGetUserRegisterAccessPermissions_IMPL(OBJGPU *pGpu, NvU32 offset)
138 {
139     NvU32 bitOffset = offset / sizeof(NvU32);
140 
141     if (!pGpu->pUserRegisterAccessMap)
142     {
143         //
144         // If very early in the init sequence, everything is accessible, since
145         // we can't have gotten any user originating accesses yet.
146         //
147         if (!gpuIsFullyConstructed(pGpu))
148             return NV_TRUE;
149 
150         NV_ASSERT_FAILED("No user register access map available to read");
151         return NV_FALSE;
152     }
153 
154     if (bitOffset >= (pGpu->userRegisterAccessMapSize * 8))
155     {
156         NV_PRINTF(LEVEL_ERROR, "Parameter `offset` = %u is out of bounds.\n",
157                   offset);
158         return NV_FALSE;
159     }
160 
161     if ((offset % 4) != 0)
162     {
163         NV_PRINTF(LEVEL_ERROR,
164                   "Parameter `offset` = %u must be 4-byte aligned.\n", offset);
165         return NV_FALSE;
166     }
167 
168     // pGpu->pUserRegisterAccessMap is pageable, must not be at raised IRQ
169     NV_ASSERT_OR_RETURN(!osIsRaisedIRQL(), NV_ERR_INVALID_IRQ_LEVEL);
170 
171     return nvBitFieldTest((NvU32*) pGpu->pUserRegisterAccessMap, pGpu->userRegisterAccessMapSize / sizeof(NvU32), bitOffset);
172 }
173 
174 
_getIsProfilingPrivileged(OBJGPU * pGpu)175 static NvBool _getIsProfilingPrivileged(OBJGPU *pGpu)
176 {
177     // On a vGPU Host, RmProfilingAdminOnly is always set to 1
178     if (hypervisorIsVgxHyper())
179     {
180         //
181         // Setting the value at this point to make the behavior same for
182         // debug/develop/release drivers on vGPU host.
183         //
184         return NV_TRUE;
185     }
186 #if defined(DEBUG) || defined(DEVELOP)
187     return NV_FALSE;
188 #else
189     NvU32 data32;
190     if (NV_OK == osReadRegistryDword(pGpu, NV_REG_STR_RM_PROFILING_ADMIN_ONLY, &data32))
191     {
192         return (data32 == NV_REG_STR_RM_PROFILING_ADMIN_ONLY_TRUE);
193     }
194 
195     return NV_TRUE;
196 #endif
197 }
198 /**
199  * @brief Constructs the bitmap used to control whether a register can be accessed by user space.
200  *
201  * Bitmap contains a single bit per 32b register.
202  *
203  * @param pGpu
204  *
205  * @return NV_OK if success, error otherwise
206  */
207 NV_STATUS
gpuConstructUserRegisterAccessMap_IMPL(OBJGPU * pGpu)208 gpuConstructUserRegisterAccessMap_IMPL(OBJGPU *pGpu)
209 {
210     NV2080_CTRL_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP_PARAMS *pParams = NULL;
211     NV_STATUS    status              = NV_OK;
212     NvU32        compressedSize      = 0;
213     NvU32        profilingRangesSize = 0;
214     const NvU8  *compressedData      = NULL;
215     const NvU32 *profilingRangesArr  = NULL;
216 
217     if (IS_VIRTUAL(pGpu))
218     {
219         // Usermode access maps unused in Guest RM. Initialize this boolean and leave.
220         pGpu->bRmProfilingPrivileged = _getIsProfilingPrivileged(pGpu);
221         return NV_OK;
222     }
223 
224     NV_ASSERT(pGpu->userRegisterAccessMapSize == 0);
225 
226     {
227         RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
228         pParams = portMemAllocPaged(sizeof(*pParams));
229         NV_ASSERT_OR_RETURN(pParams != NULL, NV_ERR_NO_MEMORY);
230         portMemSet(pParams, 0, sizeof(*pParams));
231 
232         NV_ASSERT_OK_OR_GOTO(status,
233             pRmApi->Control(pRmApi, pGpu->hInternalClient, pGpu->hInternalSubdevice,
234                             NV2080_CTRL_CMD_INTERNAL_GPU_GET_USER_REGISTER_ACCESS_MAP,
235                             pParams, sizeof(*pParams)), done);
236 
237         pGpu->userRegisterAccessMapSize = pParams->userRegisterAccessMapSize;
238         compressedSize      = pParams->compressedSize;
239         profilingRangesSize = pParams->profilingRangesSize;
240         compressedData      = (const NvU8*)pParams->compressedData;
241         profilingRangesArr  = (const NvU32*)pParams->profilingRanges;
242     }
243 
244     //
245     // We round up to a 32b multiple to be used with bitfield helpers.
246     // (Of course it should already be a 32b multiple, but just to be sure.)
247     //
248     pGpu->userRegisterAccessMapSize = NV_ALIGN_UP(pGpu->userRegisterAccessMapSize, sizeof(NvU32));
249     if (pGpu->userRegisterAccessMapSize == 0)
250     {
251         NV_PRINTF(LEVEL_INFO,
252                   "User Register Access Map unsupported for this chip.\n");
253         status = NV_OK;
254         goto done;
255     }
256 
257     pGpu->pUserRegisterAccessMap = portMemAllocPaged(pGpu->userRegisterAccessMapSize);
258     if (pGpu->pUserRegisterAccessMap == NULL)
259     {
260         status = NV_ERR_NO_MEMORY;
261         goto done;
262     }
263 
264     pGpu->pUnrestrictedRegisterAccessMap = portMemAllocPaged(pGpu->userRegisterAccessMapSize);
265     if (pGpu->pUnrestrictedRegisterAccessMap == NULL)
266     {
267         status = NV_ERR_NO_MEMORY;
268         goto done;
269     }
270 
271     NV_PRINTF(LEVEL_INFO, "Allocated User Register Access Map of 0x%xB @%p\n",
272               pGpu->userRegisterAccessMapSize, pGpu->pUserRegisterAccessMap);
273 
274     if (!pGpu->bUseRegisterAccessMap || compressedSize == 0)
275     {
276         NV_PRINTF(LEVEL_INFO,
277                   "GPU/Platform does not have restricted user register access!  Allowing all registers.\n");
278         portMemSet(pGpu->pUserRegisterAccessMap, 0xFF, pGpu->userRegisterAccessMapSize);
279     }
280     else
281     {
282         NV_ASSERT_OK_OR_GOTO(status,
283             gpuInitRegisterAccessMap(pGpu, pGpu->pUserRegisterAccessMap,
284                 pGpu->userRegisterAccessMapSize, compressedData, compressedSize), done);
285     }
286 
287     // copy permissions from user access map
288     if (portMemCopy(pGpu->pUnrestrictedRegisterAccessMap, pGpu->userRegisterAccessMapSize,
289                     pGpu->pUserRegisterAccessMap, pGpu->userRegisterAccessMapSize) == NULL)
290     {
291         NV_PRINTF(LEVEL_ERROR,
292                   "Failed to initialize unrestricted register access map\n");
293         status = NV_ERR_INVALID_ADDRESS;
294         goto done;
295     }
296 
297     pGpu->bRmProfilingPrivileged = _getIsProfilingPrivileged(pGpu);
298     if (pGpu->bRmProfilingPrivileged && profilingRangesSize > 0)
299     {
300         // remove profiling registers from user map
301         status = gpuSetUserRegisterAccessPermissionsInBulk(
302                     pGpu, profilingRangesArr, profilingRangesSize, NV_FALSE);
303         if (status != NV_OK)
304         {
305             pGpu->bRmProfilingPrivileged = NV_FALSE;
306             goto done;
307         }
308     }
309 
310 done:
311     if (status != NV_OK)
312     {
313         portMemFree(pGpu->pUserRegisterAccessMap);
314         pGpu->pUserRegisterAccessMap = NULL;
315 
316         portMemFree(pGpu->pUnrestrictedRegisterAccessMap);
317         pGpu->pUnrestrictedRegisterAccessMap = NULL;
318 
319         pGpu->userRegisterAccessMapSize = 0;
320     }
321     portMemFree(pParams);
322 
323     return status;
324 }
325 
326 
327 /**
328  * @brief Initializes the register access map
329  *
330  * Extracts compressed data representing access map.
331  *
332  * @param pGpu
333  *
334  * @return NV_OK if success, error otherwise
335  */
336 NV_STATUS
gpuInitRegisterAccessMap_IMPL(OBJGPU * pGpu,NvU8 * pAccessMap,NvU32 accessMapSize,const NvU8 * pComprData,const NvU32 comprDataSize)337 gpuInitRegisterAccessMap_IMPL(OBJGPU *pGpu, NvU8 *pAccessMap, NvU32 accessMapSize, const NvU8 *pComprData, const NvU32 comprDataSize)
338 {
339     PGZ_INFLATE_STATE pGzState = NULL;
340     NvU32 inflatedBytes        = 0;
341 
342     NV_ASSERT_OR_RETURN(pAccessMap != NULL, NV_ERR_INVALID_STATE);
343     NV_ASSERT_OR_RETURN(accessMapSize != 0, NV_ERR_INVALID_STATE);
344     NV_ASSERT_OR_RETURN(pComprData != NULL, NV_ERR_INVALID_STATE);
345     NV_ASSERT_OR_RETURN(comprDataSize != 0, NV_ERR_INVALID_STATE);
346 
347     //
348     // Strip off gzlib 10-byte header
349     // XXX this really belongs in the RM GZ library
350     //
351     pComprData += 10;
352 
353     NV_ASSERT_OK_OR_RETURN(utilGzAllocate((NvU8*)pComprData, accessMapSize, &pGzState));
354 
355     NV_ASSERT(pGzState);
356 
357     inflatedBytes = utilGzGetData(pGzState, 0, accessMapSize, pAccessMap);
358 
359     utilGzDestroy(pGzState);
360 
361     if (inflatedBytes != accessMapSize)
362     {
363         NV_PRINTF(LEVEL_ERROR,
364                   "failed to get inflated data, got %u bytes, expecting %u\n",
365                   inflatedBytes, pGpu->userRegisterAccessMapSize);
366         DBG_BREAKPOINT();
367         return NV_ERR_INFLATE_COMPRESSED_DATA_FAILED;
368     }
369     return NV_OK;
370 }
371