1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2020-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "core/core.h"
25 #include "kernel/gpu/intr/intr.h"
26 #include "gpu/gpu.h"
27 #include "os/os.h"
28
29 #include "published/hopper/gh100/dev_vm.h"
30 #include "published/hopper/gh100/dev_vm_addendum.h"
31
32 // Compile time asserts to make sure we don't write beyond the leaf register array
33
34 ct_assert(NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF__SIZE_1 == NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_SET__SIZE_1);
35 ct_assert(NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF__SIZE_1 == NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_CLEAR__SIZE_1);
36
37
38 NV_STATUS
intrInitSubtreeMap_GH100(OBJGPU * pGpu,Intr * pIntr)39 intrInitSubtreeMap_GH100
40 (
41 OBJGPU *pGpu,
42 Intr *pIntr
43 )
44 {
45 NV2080_INTR_CATEGORY_SUBTREE_MAP *pCategoryEngine =
46 &pIntr->subtreeMap[NV2080_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE];
47 pCategoryEngine->subtreeStart = NV_CPU_INTR_STALL_SUBTREE_START;
48 pCategoryEngine->subtreeEnd = NV_CPU_INTR_STALL_SUBTREE_START;
49
50 NV2080_INTR_CATEGORY_SUBTREE_MAP *pCategoryEngineNotification =
51 &pIntr->subtreeMap[NV2080_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION];
52 pCategoryEngineNotification->subtreeStart = NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_SUBTREE(0);
53 pCategoryEngineNotification->subtreeEnd = NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_SUBTREE(0);
54
55 NV2080_INTR_CATEGORY_SUBTREE_MAP *pCategoryRunlistLocked =
56 &pIntr->subtreeMap[NV2080_INTR_CATEGORY_RUNLIST];
57 pCategoryRunlistLocked->subtreeStart = NV_CPU_INTR_STALL_SUBTREE_LAST_SWRL;
58 pCategoryRunlistLocked->subtreeEnd = NV_CPU_INTR_STALL_SUBTREE_LAST_SWRL;
59
60 //
61 // Don't reprogram NV_RUNLIST_INTR_TREE_LOCKLESS (runlist notification)
62 // vectors to a different subtree than their init values in Hopper unlike
63 // Ampere. INTR_TOP is covered with a "nonstall" mask corresponding to these
64 // leaf ranges.
65 // It is okay to reprogram the vectors for individual runlists within the
66 // same subtree.
67 //
68 NV2080_INTR_CATEGORY_SUBTREE_MAP *pCategoryRunlistNotification =
69 &pIntr->subtreeMap[NV2080_INTR_CATEGORY_RUNLIST_NOTIFICATION];
70 pCategoryRunlistNotification->subtreeStart = NV_CPU_INTR_STALL_SUBTREE_LAST;
71 pCategoryRunlistNotification->subtreeEnd = NV_CPU_INTR_STALL_SUBTREE_LAST;
72
73 NV2080_INTR_CATEGORY_SUBTREE_MAP *pCategoryUvmOwned =
74 &pIntr->subtreeMap[NV2080_INTR_CATEGORY_UVM_OWNED];
75 pCategoryUvmOwned->subtreeStart = NV_CPU_INTR_UVM_SUBTREE_START;
76 pCategoryUvmOwned->subtreeEnd = NV_CPU_INTR_UVM_SUBTREE_LAST;
77
78 NV2080_INTR_CATEGORY_SUBTREE_MAP *pCategoryUvmShared =
79 &pIntr->subtreeMap[NV2080_INTR_CATEGORY_UVM_SHARED];
80 pCategoryUvmShared->subtreeStart = NV_CPU_INTR_UVM_SHARED_SUBTREE_START;
81 pCategoryUvmShared->subtreeEnd = NV_CPU_INTR_UVM_SHARED_SUBTREE_LAST;
82
83 return NV_OK;
84 }
85
86
87 /*!
88 * @brief Gets the number of leaf registers used
89 */
90 NvU32
intrGetNumLeaves_GH100(OBJGPU * pGpu,Intr * pIntr)91 intrGetNumLeaves_GH100(OBJGPU *pGpu, Intr *pIntr)
92 {
93 ct_assert((NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_END(NV_CPU_INTR_STALL_SUBTREE_LAST) + 1) <= NV_MAX_INTR_LEAVES);
94 return (NV_CTRL_INTR_SUBTREE_TO_LEAF_IDX_END(NV_CPU_INTR_STALL_SUBTREE_LAST) + 1);
95 }
96
97 /*!
98 * @brief Gets the value of VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF__SIZE_1
99 */
100 NvU32
intrGetLeafSize_GH100(OBJGPU * pGpu,Intr * pIntr)101 intrGetLeafSize_GH100(OBJGPU *pGpu, Intr *pIntr)
102 {
103 return NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF__SIZE_1;
104 }
105
106
107 NvU64
intrGetIntrTopNonStallMask_GH100(OBJGPU * pGpu,Intr * pIntr)108 intrGetIntrTopNonStallMask_GH100
109 (
110 OBJGPU *pGpu,
111 Intr *pIntr
112 )
113 {
114 // TODO Bug 3823562 Remove all these asserts
115 // Compile-time assert against the highest set bit that will be returned
116 #define NV_CPU_INTR_NOSTALL_SUBTREE_HIGHEST NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_TOP_SUBTREE(5)
117
118 ct_assert(NV_CPU_INTR_NOSTALL_SUBTREE_HIGHEST < NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF__SIZE_1);
119 ct_assert(NV_CPU_INTR_NOSTALL_SUBTREE_HIGHEST < NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_SET__SIZE_1);
120 ct_assert(NV_CPU_INTR_NOSTALL_SUBTREE_HIGHEST < NV_VIRTUAL_FUNCTION_PRIV_CPU_INTR_LEAF_EN_CLEAR__SIZE_1);
121
122
123 NvU64 mask = 0;
124 mask |= intrGetIntrTopCategoryMask(pIntr,
125 NV2080_INTR_CATEGORY_ESCHED_DRIVEN_ENGINE_NOTIFICATION);
126
127 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_SWRL_GRANULAR_LOCKING))
128 {
129 mask |= intrGetIntrTopCategoryMask(pIntr,
130 NV2080_INTR_CATEGORY_RUNLIST_NOTIFICATION);
131 }
132
133 // Sanity check that Intr.subtreeMap is initialized
134 NV_ASSERT(mask != 0);
135 return mask;
136 }
137
138
139 /*!
140 * @brief Sanity check that the given stall engine interrupt vector is in the right tree
141 *
142 * @param[in] pGpu OBJGPU pointer
143 * @param[in] pIntr Intr pointer
144 * @param[in] vector stall interrupt vector
145 */
146 void
intrSanityCheckEngineIntrStallVector_GH100(OBJGPU * pGpu,Intr * pIntr,NvU32 vector,NvU16 mcEngine)147 intrSanityCheckEngineIntrStallVector_GH100
148 (
149 OBJGPU *pGpu,
150 Intr *pIntr,
151 NvU32 vector,
152 NvU16 mcEngine
153 )
154 {
155 //
156 // The leafIndex must be within the engine stall tree: leaf 6 and 7 on Hopper.
157 // Try to catch this on pre-release drivers. Don't need this on release drivers since this
158 // is only to catch issues during code development. Should never happen in practice.
159 //
160 if ((NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(vector) != 6) &&
161 (NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(vector) != 7))
162 {
163 NV_PRINTF(LEVEL_ERROR, "MC_ENGINE_IDX %u has invalid stall intr vector %u\n", mcEngine, vector);
164 DBG_BREAKPOINT();
165 }
166 }
167
168 /*!
169 * @brief Sanity check that the given notification engine interrupt vector is in the right tree
170 *
171 * @param[in] pGpu OBJGPU pointer
172 * @param[in] pIntr Intr pointer
173 * @param[in] vector stall interrupt vector
174 */
175 void
intrSanityCheckEngineIntrNotificationVector_GH100(OBJGPU * pGpu,Intr * pIntr,NvU32 vector,NvU16 mcEngine)176 intrSanityCheckEngineIntrNotificationVector_GH100
177 (
178 OBJGPU *pGpu,
179 Intr *pIntr,
180 NvU32 vector,
181 NvU16 mcEngine
182 )
183 {
184 //
185 // The leafIndex must be within the engine notification tree: leaf 0 and 1 on Hopper.
186 // Try to catch this on pre-release drivers. Don't need this on release drivers since this
187 // is only to catch issues during code development. Should never happen in practice.
188 //
189 if ((NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(vector) != 0) &&
190 (NV_CTRL_INTR_GPU_VECTOR_TO_LEAF_REG(vector) != 1))
191 {
192 NV_PRINTF(LEVEL_ERROR, "MC_ENGINE_IDX %u has invalid notification intr vector %u\n", mcEngine, vector);
193 DBG_BREAKPOINT();
194 }
195 }
196