1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2017-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /*!
25 * Provides the implementation for all TU102+ specific KernelFalcon interfaces.
26 */
27
28 #include "gpu/falcon/kernel_falcon.h"
29 #include "os/os.h"
30
31 #include "published/turing/tu102/dev_riscv_pri.h"
32 #include "published/turing/tu102/dev_falcon_v4.h"
33 #include "published/turing/tu102/dev_fbif_v4.h"
34
35 /*!
36 * Read a Falcon register.
37 *
38 * @param[in] pGpu OBJGPU pointer
39 * @param[in] pKernelFlcn KernelFalcon pointer
40 * @param[in] offset Offset into the Falcon register space.
41 *
42 * @returns The value of the register.
43 */
44 NvU32
kflcnRegRead_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn,NvU32 offset)45 kflcnRegRead_TU102
46 (
47 OBJGPU *pGpu,
48 KernelFalcon *pKernelFlcn,
49 NvU32 offset
50 )
51 {
52 return REG_INST_DEVIDX_RD32_EX(pGpu, DEVICE_INDEX_GPU, 0,
53 pKernelFlcn->registerBase + offset, NULL);
54 }
55
56 /*!
57 * Write a Falcon register.
58 *
59 * @param[in] pGpu OBJGPU pointer
60 * @param[in] pKernelFlcn KernelFalcon pointer
61 * @param[in] offset Offset into the Falcon register space.
62 * @param[in] data Data to write to the register.
63 */
64 void
kflcnRegWrite_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn,NvU32 offset,NvU32 data)65 kflcnRegWrite_TU102
66 (
67 OBJGPU *pGpu,
68 KernelFalcon *pKernelFlcn,
69 NvU32 offset,
70 NvU32 data
71 )
72 {
73 REG_INST_DEVIDX_WR32_EX(pGpu, DEVICE_INDEX_GPU, 0,
74 pKernelFlcn->registerBase + offset, data, NULL);
75 }
76
77 /*!
78 * Read a RISCV register.
79 *
80 * @param[in] pGpu OBJGPU pointer
81 * @param[in] pKernelFlcn KernelFalcon pointer
82 * @param[in] offset Offset into the RISCV register space.
83 *
84 * @returns The value of the register.
85 */
86 NvU32
kflcnRiscvRegRead_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn,NvU32 offset)87 kflcnRiscvRegRead_TU102
88 (
89 OBJGPU *pGpu,
90 KernelFalcon *pKernelFlcn,
91 NvU32 offset
92 )
93 {
94 return REG_INST_DEVIDX_RD32_EX(pGpu, DEVICE_INDEX_GPU, 0,
95 pKernelFlcn->riscvRegisterBase + offset, NULL);
96 }
97
98 /*!
99 * Write a RISCV register.
100 *
101 * @param[in] pGpu OBJGPU pointer
102 * @param[in] pKernelFlcn KernelFalcon pointer
103 * @param[in] offset Offset into the RISCV register space.
104 * @param[in] data Data to write to the register.
105 */
106 void
kflcnRiscvRegWrite_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn,NvU32 offset,NvU32 data)107 kflcnRiscvRegWrite_TU102
108 (
109 OBJGPU *pGpu,
110 KernelFalcon *pKernelFlcn,
111 NvU32 offset,
112 NvU32 data
113 )
114 {
115 REG_INST_DEVIDX_WR32_EX(pGpu, DEVICE_INDEX_GPU, 0,
116 pKernelFlcn->riscvRegisterBase + offset, data, NULL);
117 }
118
119 /*!
120 * Check the existence of RISCV CPU.
121 */
122 NvBool
kflcnIsRiscvCpuEnabled_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn)123 kflcnIsRiscvCpuEnabled_TU102
124 (
125 OBJGPU *pGpu,
126 KernelFalcon *pKernelFlcn
127 )
128 {
129 NvU32 reg = kflcnRegRead_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_HWCFG2);
130
131 return FLD_TEST_DRF(_PFALCON, _FALCON_HWCFG2, _RISCV, _ENABLE, reg);
132 }
133
134 /*!
135 * Function to check if RISCV is active.
136 */
137 NvBool
kflcnIsRiscvActive_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn)138 kflcnIsRiscvActive_TU102
139 (
140 OBJGPU *pGpu,
141 KernelFalcon *pKernelFlcn
142 )
143 {
144 NvU32 val;
145
146 val = kflcnRiscvRegRead_HAL(pGpu, pKernelFlcn, NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS);
147
148 return FLD_TEST_DRF(_PRISCV, _RISCV_CORE_SWITCH_RISCV_STATUS, _ACTIVE_STAT, _ACTIVE, val);
149 }
150
151 /*!
152 * Reset falcon using secure reset.
153 * This leaves the falcon in falcon mode after reset.
154 */
155 void
kflcnReset_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn)156 kflcnReset_TU102
157 (
158 OBJGPU *pGpu,
159 KernelFalcon *pKernelFlcn
160 )
161 {
162 NV_ASSERT_OR_RETURN_VOID(kflcnPreResetWait_HAL(pGpu, pKernelFlcn) == NV_OK);
163 NV_ASSERT_OK(kflcnResetHw(pGpu, pKernelFlcn));
164 kflcnWaitForResetToFinish_HAL(pGpu, pKernelFlcn);
165 kflcnSwitchToFalcon_HAL(pGpu, pKernelFlcn);
166 kflcnRegWrite_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_RM,
167 pGpu->chipId0);
168 }
169
170 /*!
171 * Reset falcon using secure reset, ready to run riscv.
172 */
173 void
kflcnResetIntoRiscv_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn)174 kflcnResetIntoRiscv_TU102
175 (
176 OBJGPU *pGpu,
177 KernelFalcon *pKernelFlcn
178 )
179 {
180 //
181 // Turing and GA100 do not have an explicit core switch,
182 // the core will be ready to run riscv after reset.
183 //
184 kflcnReset_TU102(pGpu, pKernelFlcn);
185 }
186
187 /*!
188 * Start a Falcon CPU.
189 */
190 void
kflcnStartCpu_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn)191 kflcnStartCpu_TU102(OBJGPU *pGpu, KernelFalcon *pKernelFlcn)
192 {
193 if (FLD_TEST_DRF(_PFALCON, _FALCON_CPUCTL, _ALIAS_EN, _TRUE,
194 kflcnRegRead_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_CPUCTL)))
195 {
196 kflcnRegWrite_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_CPUCTL_ALIAS,
197 DRF_DEF(_PFALCON, _FALCON_CPUCTL_ALIAS, _STARTCPU, _TRUE));
198 }
199 else
200 {
201 kflcnRegWrite_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_CPUCTL,
202 DRF_DEF(_PFALCON, _FALCON_CPUCTL, _STARTCPU, _TRUE));
203 }
204 }
205
206 /*!
207 * Disables context requirement of Falcon.
208 */
209 void
kflcnDisableCtxReq_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn)210 kflcnDisableCtxReq_TU102
211 (
212 OBJGPU *pGpu,
213 KernelFalcon *pKernelFlcn
214 )
215 {
216 NvU32 data = 0;
217
218 data = GPU_REG_RD32(pGpu, pKernelFlcn->fbifBase + NV_PFALCON_FBIF_CTL);
219 data = FLD_SET_DRF(_PFALCON, _FBIF_CTL, _ALLOW_PHYS_NO_CTX, _ALLOW, data);
220
221 // Allow physical address without CTX
222 GPU_REG_WR32(pGpu, pKernelFlcn->fbifBase + NV_PFALCON_FBIF_CTL, data);
223
224 // Allow issue DMA request without block bind
225 kflcnRegWrite_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_DMACTL, 0x0);
226 }
227
228 /*!
229 * Checks if Falcon memory scrubbing is finished.
230 *
231 * @param pGpu OBJGPU pointer
232 * @param pVoid void pointer to a KernelFalcon instance
233 */
234 static NvBool
_kflcnMemScrubbingFinished(OBJGPU * pGpu,void * pVoid)235 _kflcnMemScrubbingFinished
236 (
237 OBJGPU *pGpu,
238 void *pVoid
239 )
240 {
241 NvBool bResult = NV_FALSE;
242 NvU32 dmaCtrl = 0;
243 KernelFalcon *pKernelFlcn = reinterpretCast(pVoid, KernelFalcon *);
244
245 dmaCtrl = kflcnRegRead_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_DMACTL);
246
247 if (FLD_TEST_DRF(_PFALCON, _FALCON_DMACTL, _DMEM_SCRUBBING, _DONE, dmaCtrl) &&
248 FLD_TEST_DRF(_PFALCON, _FALCON_DMACTL, _IMEM_SCRUBBING, _DONE, dmaCtrl))
249 {
250 bResult = NV_TRUE;
251 }
252
253 return bResult;
254 }
255
256 /*!
257 * Wait for Falcon reset to finish.
258 */
259 NV_STATUS
kflcnWaitForResetToFinish_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn)260 kflcnWaitForResetToFinish_TU102
261 (
262 OBJGPU *pGpu,
263 KernelFalcon *pKernelFlcn
264 )
265 {
266 // Skip the wait if we are in the GPU reset path
267 if (API_GPU_IN_RESET_SANITY_CHECK(pGpu))
268 return NV_ERR_GPU_IN_FULLCHIP_RESET;
269
270 //
271 // We could potentially bypass the polling if we are going to read from IMEM or DMEM.
272 // But waiting ensures we avoid pri timouts. See bug 623410.
273 //
274 return gpuTimeoutCondWait(pGpu, _kflcnMemScrubbingFinished, pKernelFlcn, NULL);
275 }
276
277 /*!
278 * Wait for Falcon to halt.
279 *
280 * @param[in] pGpu OBJGPU pointer
281 * @param[in] pKernelFlcn KernelFalcon pointer
282 * @param[in] timeoutUs Timeout value
283 *
284 * @returns NV_ERR_TIMEOUT if falcon fails to halt.
285 */
286 NV_STATUS
kflcnWaitForHalt_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn,NvU32 timeoutUs,NvU32 flags)287 kflcnWaitForHalt_TU102
288 (
289 OBJGPU *pGpu,
290 KernelFalcon *pKernelFlcn,
291 NvU32 timeoutUs,
292 NvU32 flags
293 )
294 {
295 NV_STATUS status = NV_OK;
296 RMTIMEOUT timeout;
297
298 gpuSetTimeout(pGpu, timeoutUs, &timeout, flags);
299
300 while (!FLD_TEST_DRF(_PFALCON, _FALCON, _CPUCTL_HALTED, _TRUE,
301 kflcnRegRead_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_CPUCTL)))
302 {
303 status = gpuCheckTimeout(pGpu, &timeout);
304 if (status == NV_ERR_TIMEOUT)
305 {
306 NV_PRINTF(LEVEL_ERROR, "Timeout waiting for Falcon to halt\n");
307 DBG_BREAKPOINT();
308 break;
309 }
310 osSpinLoop();
311 }
312
313 return status;
314 }
315
316 /*!
317 * Read the IRQ status of the RISCV Falcon.
318 *
319 * @return IRQ status mask
320 */
321 NvU32
kflcnReadIntrStatus_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn)322 kflcnReadIntrStatus_TU102
323 (
324 OBJGPU *pGpu,
325 KernelFalcon *pKernelFlcn
326 )
327 {
328 return ((kflcnRegRead_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_IRQSTAT) &
329 kflcnRegRead_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_IRQMASK) &
330 kflcnRegRead_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_IRQDEST)) |
331 (kflcnRegRead_HAL(pGpu, pKernelFlcn, NV_PFALCON_FALCON_IRQSTAT) &
332 kflcnRiscvRegRead_HAL(pGpu, pKernelFlcn, NV_PRISCV_RISCV_IRQMASK) &
333 kflcnRiscvRegRead_HAL(pGpu, pKernelFlcn, NV_PRISCV_RISCV_IRQDEST)));
334 }
335
336
337 /*!
338 * Mask a IMEM address to have only the BLK and OFFSET bits set.
339 *
340 * @param[in] addr IMEM address
341 */
342 NvU32
kflcnMaskImemAddr_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn,NvU32 addr)343 kflcnMaskImemAddr_TU102
344 (
345 OBJGPU *pGpu,
346 KernelFalcon *pKernelFlcn,
347 NvU32 addr
348
349 )
350 {
351 return (addr & (DRF_SHIFTMASK(NV_PFALCON_FALCON_IMEMC_OFFS) |
352 DRF_SHIFTMASK(NV_PFALCON_FALCON_IMEMC_BLK)));
353 }
354
355 /*!
356 * Mask a DMEM address to have only the BLK and OFFSET bits set.
357 *
358 * @param[in] addr DMEM address
359 */
360 NvU32
kflcnMaskDmemAddr_TU102(OBJGPU * pGpu,KernelFalcon * pKernelFlcn,NvU32 addr)361 kflcnMaskDmemAddr_TU102
362 (
363 OBJGPU *pGpu,
364 KernelFalcon *pKernelFlcn,
365 NvU32 addr
366 )
367 {
368 return (addr & (DRF_SHIFTMASK(NV_PFALCON_FALCON_DMEMC_OFFS) |
369 DRF_SHIFTMASK(NV_PFALCON_FALCON_DMEMC_BLK)));
370 }
371