1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 #if !defined(SRT_BUILD)
24 #define NVOC_KERN_GMMU_H_PRIVATE_ACCESS_ALLOWED
25 #include "gpu/mmu/kern_gmmu.h"
26 #endif
27 #include "mmu/gmmu_fmt.h"
28 #include "published/hopper/gh100/dev_mmu.h"
29 #include "published/hopper/gh100/hwproject.h"
30 
31 /*!
32  *  NUM_VA_BITS = 57
33  *
34  *  PD4[56]
35  *  |
36  *  v
37  *  PD3 [55:47]
38  *  |
39  *  v
40  *  PD2 [46:38]
41  *  |
42  *  v
43  *  PD1 [37:29] / PT_512M [37:29] (512MB page)
44  *  |
45  *  v
46  *  PD0 [28:21] / PT_LARGE [28:21] (2MB page)
47  *  |                       \
48  *  |                        \
49  *  v                         v
50  *  PT_SMALL       PT_BIG (64KB page)
51  *  [20:12]               [20:16]
52  */
53 void kgmmuFmtInitLevels_GH10X(KernelGmmu    *pKernelGmmu,
54                               MMU_FMT_LEVEL *pLevels,
55                               const NvU32    numLevels,
56                               const NvU32    version,
57                               const NvU32    bigPageShift)
58 {
59     NV_ASSERT_OR_RETURN_VOID(version == GMMU_FMT_VERSION_3);
60     NV_ASSERT_OR_RETURN_VOID(numLevels >= 7);
61 
62     // Page directory 4 (root).
63     pLevels[0].virtAddrBitHi  = 56;
64     pLevels[0].virtAddrBitLo  = 56;
65     pLevels[0].entrySize      = NV_MMU_VER3_PDE__SIZE;
66     pLevels[0].numSubLevels   = 1;
67     pLevels[0].subLevels      = pLevels + 1;
68     pLevels[0].pageLevelIdTag = MMU_FMT_PT_SURF_ID_PD0;
69 
70     // Page directory 3.
71     pLevels[1].virtAddrBitHi  = 55;
72     pLevels[1].virtAddrBitLo  = 47;
73     pLevels[1].entrySize      = NV_MMU_VER3_PDE__SIZE;
74     pLevels[1].numSubLevels   = 1;
75     pLevels[1].subLevels      = pLevels + 2;
76     pLevels[1].pageLevelIdTag = MMU_FMT_PT_SURF_ID_PD1;
77 
78     // Page directory 2.
79     pLevels[2].virtAddrBitHi  = 46;
80     pLevels[2].virtAddrBitLo  = 38;
81     pLevels[2].entrySize      = NV_MMU_VER3_PDE__SIZE;
82     pLevels[2].numSubLevels   = 1;
83     pLevels[2].subLevels      = pLevels + 3;
84     pLevels[2].pageLevelIdTag = MMU_FMT_PT_SURF_ID_PD2;
85 
86     // Page directory 1.
87     pLevels[3].virtAddrBitHi  = 37;
88     pLevels[3].virtAddrBitLo  = 29;
89     pLevels[3].entrySize      = NV_MMU_VER3_PDE__SIZE;
90     pLevels[3].numSubLevels   = 1;
91     pLevels[3].subLevels      = pLevels + 4;
92     // Page directory 1 can hold a PTE pointing to a 512MB Page
93     pLevels[3].bPageTable     = NV_TRUE;
94     pLevels[3].pageLevelIdTag = MMU_FMT_PT_SURF_ID_PD3;
95 
96     // Page directory 0.
97     pLevels[4].virtAddrBitHi  = 28;
98     pLevels[4].virtAddrBitLo  = 21;
99     pLevels[4].entrySize      = NV_MMU_VER3_DUAL_PDE__SIZE;
100     pLevels[4].numSubLevels   = 2;
101     pLevels[4].bPageTable     = NV_TRUE;
102     pLevels[4].subLevels      = pLevels + 5;
103     pLevels[4].pageLevelIdTag = MMU_FMT_PT_SURF_ID_PD4;
104 
105     // Big page table.
106     pLevels[5].virtAddrBitHi  = 20;
107     pLevels[5].virtAddrBitLo  = (NvU8)bigPageShift;
108     pLevels[5].entrySize      = NV_MMU_VER3_PTE__SIZE;
109     pLevels[5].bPageTable     = NV_TRUE;
110     pLevels[5].pageLevelIdTag = MMU_FMT_PT_SURF_ID_PT_BIG;
111 
112     // Small page table.
113     pLevels[6].virtAddrBitHi  = 20;
114     pLevels[6].virtAddrBitLo  = 12;
115     pLevels[6].entrySize      = NV_MMU_VER3_PTE__SIZE;
116     pLevels[6].bPageTable     = NV_TRUE;
117     pLevels[5].pageLevelIdTag = MMU_FMT_PT_SURF_ID_PT_4K;
118 }
119 
120 void kgmmuFmtInitPdeMulti_GH10X(KernelGmmu                *pKernelGmmu,
121                                 GMMU_FMT_PDE_MULTI        *pPdeMulti,
122                                 const NvU32                version,
123                                 const NV_FIELD_ENUM_ENTRY *pPdeApertures)
124 {
125     GMMU_FMT_PDE *pPdeBig   = &pPdeMulti->subLevels[0];
126     GMMU_FMT_PDE *pPdeSmall = &pPdeMulti->subLevels[1];
127 
128     NV_ASSERT_OR_RETURN_VOID(version == GMMU_FMT_VERSION_3);
129 
130     // Dual PDE - big part.
131     pPdeBig->version = GMMU_FMT_VERSION_3;
132     INIT_FIELD_APERTURE(&pPdeBig->fldAperture, NV_MMU_VER3_DUAL_PDE_APERTURE_BIG,
133                         pPdeApertures);
134     INIT_FIELD_ADDRESS(&pPdeBig->fldAddr, NV_MMU_VER3_DUAL_PDE_ADDRESS_BIG,
135                        NV_MMU_VER3_DUAL_PDE_ADDRESS_BIG_SHIFT);
136     INIT_FIELD_DESC32(&pPdeBig->fldPdePcf, NV_MMU_VER3_DUAL_PDE_PCF_BIG);
137 
138     // Dual PDE - small part.
139     pPdeSmall->version = GMMU_FMT_VERSION_3;
140     INIT_FIELD_APERTURE(&pPdeSmall->fldAperture, NV_MMU_VER3_DUAL_PDE_APERTURE_SMALL,
141                         pPdeApertures);
142     INIT_FIELD_ADDRESS(&pPdeSmall->fldAddr, NV_MMU_VER3_DUAL_PDE_ADDRESS_SMALL,
143                        NV_MMU_VER3_DUAL_PDE_ADDRESS_SHIFT);
144     INIT_FIELD_DESC32(&pPdeSmall->fldPdePcf, NV_MMU_VER3_DUAL_PDE_PCF_SMALL);
145 }
146 
147 void kgmmuFmtInitPde_GH10X(KernelGmmu *pKernelGmmu,
148                            GMMU_FMT_PDE *pPde,
149                            const NvU32 version,
150                            const NV_FIELD_ENUM_ENTRY *pPdeApertures)
151 {
152     NV_ASSERT_OR_RETURN_VOID(version == GMMU_FMT_VERSION_3);
153 
154     pPde->version = GMMU_FMT_VERSION_3;
155     INIT_FIELD_APERTURE(&pPde->fldAperture, NV_MMU_VER3_PDE_APERTURE, pPdeApertures);
156     INIT_FIELD_DESC32(&pPde->fldPdePcf, NV_MMU_VER3_PDE_PCF);
157     INIT_FIELD_ADDRESS(&pPde->fldAddr, NV_MMU_VER3_PDE_ADDRESS, NV_MMU_VER3_PDE_ADDRESS_SHIFT);
158 }
159 
160 void kgmmuFmtInitPte_GH10X(KernelGmmu *pKernelGmmu,
161                            GMMU_FMT_PTE *pPte,
162                            const NvU32 version,
163                            const NV_FIELD_ENUM_ENTRY *pPteApertures,
164                            const NvBool bUnifiedAperture)
165 {
166     NV_ASSERT_OR_RETURN_VOID(version == GMMU_FMT_VERSION_3);
167 
168     pPte->version = GMMU_FMT_VERSION_3;
169     INIT_FIELD_BOOL(&pPte->fldValid, NV_MMU_VER3_PTE_VALID);
170     INIT_FIELD_APERTURE(&pPte->fldAperture, NV_MMU_VER3_PTE_APERTURE, pPteApertures);
171     INIT_FIELD_DESC32(&pPte->fldPeerIndex, NV_MMU_VER3_PTE_PEER_ID);
172     INIT_FIELD_DESC32(&pPte->fldKind, NV_MMU_VER3_PTE_KIND);
173     INIT_FIELD_DESC32(&pPte->fldPtePcf, NV_MMU_VER3_PTE_PCF);
174 
175     INIT_FIELD_ADDRESS(&pPte->fldAddrVidmem, NV_MMU_VER3_PTE_ADDRESS_VID, NV_MMU_VER3_PTE_ADDRESS_SHIFT);
176     INIT_FIELD_ADDRESS(&pPte->fldAddrPeer, NV_MMU_VER3_PTE_ADDRESS_PEER, NV_MMU_VER3_PTE_ADDRESS_SHIFT);
177     INIT_FIELD_ADDRESS(&pPte->fldAddrSysmem, NV_MMU_VER3_PTE_ADDRESS_SYS, NV_MMU_VER3_PTE_ADDRESS_SHIFT);
178 }
179