1 /*
2 * Copyright (c) 2019 Jordan Hargrave <jordan_hargrave@hotmail.com>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16 #ifndef __amd_iommu_h__
17 #define __amd_iommu_h__
18
19 #define DEV_TAB_BASE_REG 0x0000
20 #define CMD_BASE_REG 0x0008
21 #define EVT_BASE_REG 0x0010
22
23 #define EXCL_BASE_REG 0x0020
24 #define EXCL_LIMIT_REG 0x0028
25
26 /* Extended Feature Register */
27 #define EXTFEAT_REG 0x0030
28 #define EFR_PREFSUP (1L << 0)
29 #define EFR_PPRSUP (1L << 1)
30 #define EFR_NXSUP (1L << 3)
31 #define EFR_GTSUP (1L << 4)
32 #define EFR_IASUP (1L << 6)
33 #define EFR_GASUP (1L << 7)
34 #define EFR_HESUP (1L << 8)
35 #define EFR_PCSUP (1L << 9)
36 #define EFR_HATS_SHIFT 10
37 #define EFR_HATS_MASK 0x3
38 #define EFR_GATS_SHIFT 12
39 #define EFR_GATS_MASK 0x3
40 #define EFR_GLXSUP_SHIFT 14
41 #define EFR_GLXSUP_MASK 0x3
42 #define EFR_SMIFSUP_SHIFT 16
43 #define EFR_SMIFSUP_MASK 0x3
44 #define EFR_SMIFRC_SHIFT 18
45 #define EFR_SMIFRC_MASK 0x7
46 #define EFR_GAMSUP_SHIFT 21
47 #define EFR_GAMSUP_MASK 0x7
48
49 #define CMD_HEAD_REG 0x2000
50 #define CMD_TAIL_REG 0x2008
51 #define EVT_HEAD_REG 0x2010
52 #define EVT_TAIL_REG 0x2018
53
54 #define IOMMUSTS_REG 0x2020
55
56 #define DEV_TAB_MASK 0x000FFFFFFFFFF000LL
57 #define DEV_TAB_LEN 0x1FF
58
59 /* IOMMU Control */
60 #define IOMMUCTL_REG 0x0018
61 #define CTL_IOMMUEN (1L << 0)
62 #define CTL_HTTUNEN (1L << 1)
63 #define CTL_EVENTLOGEN (1L << 2)
64 #define CTL_EVENTINTEN (1L << 3)
65 #define CTL_COMWAITINTEN (1L << 4)
66 #define CTL_INVTIMEOUT_SHIFT 5
67 #define CTL_INVTIMEOUT_MASK 0x7
68 #define CTL_INVTIMEOUT_NONE 0
69 #define CTL_INVTIMEOUT_1MS 1
70 #define CTL_INVTIMEOUT_10MS 2
71 #define CTL_INVTIMEOUT_100MS 3
72 #define CTL_INVTIMEOUT_1S 4
73 #define CTL_INVTIMEOUT_10S 5
74 #define CTL_INVTIMEOUT_100S 6
75 #define CTL_PASSPW (1L << 8)
76 #define CTL_RESPASSPW (1L << 9)
77 #define CTL_COHERENT (1L << 10)
78 #define CTL_ISOC (1L << 11)
79 #define CTL_CMDBUFEN (1L << 12)
80 #define CTL_PPRLOGEN (1L << 13)
81 #define CTL_PPRINTEN (1L << 14)
82 #define CTL_PPREN (1L << 15)
83 #define CTL_GTEN (1L << 16)
84 #define CTL_GAEN (1L << 17)
85 #define CTL_CRW_SHIFT 18
86 #define CTL_CRW_MASK 0xF
87 #define CTL_SMIFEN (1L << 22)
88 #define CTL_SLFWBDIS (1L << 23)
89 #define CTL_SMIFLOGEN (1L << 24)
90 #define CTL_GAMEN_SHIFT 25
91 #define CTL_GAMEN_MASK 0x7
92 #define CTL_GALOGEN (1L << 28)
93 #define CTL_GAINTEN (1L << 29)
94 #define CTL_DUALPPRLOGEN_SHIFT 30
95 #define CTL_DUALPPRLOGEN_MASK 0x3
96 #define CTL_DUALEVTLOGEN_SHIFT 32
97 #define CTL_DUALEVTLOGEN_MASK 0x3
98 #define CTL_DEVTBLSEGEN_SHIFT 34
99 #define CTL_DEVTBLSEGEN_MASK 0x7
100 #define CTL_PRIVABRTEN_SHIFT 37
101 #define CTL_PRIVABRTEN_MASK 0x3
102 #define CTL_PPRAUTORSPEN (1LL << 39)
103 #define CTL_MARCEN (1LL << 40)
104 #define CTL_BLKSTOPMRKEN (1LL << 41)
105 #define CTL_PPRAUTOSPAON (1LL << 42)
106 #define CTL_DOMAINIDPNE (1LL << 43)
107
108 #define CMD_BASE_MASK 0x000FFFFFFFFFF000LL
109 #define CMD_TBL_SIZE 4096
110 #define CMD_TBL_LEN_4K (8LL << 56)
111 #define CMD_TBL_LEN_8K (9lL << 56)
112
113 #define EVT_BASE_MASK 0x000FFFFFFFFFF000LL
114 #define EVT_TBL_SIZE 4096
115 #define EVT_TBL_LEN_4K (8LL << 56)
116 #define EVT_TBL_LEN_8K (9LL << 56)
117
118 /*========================
119 * DEVICE TABLE ENTRY
120 * Contains mapping of bus-device-function
121 *
122 * 0 Valid (V)
123 * 1 Translation Valid (TV)
124 * 7:8 Host Address Dirty (HAD)
125 * 9:11 Page Table Depth (usually 4)
126 * 12:51 Page Table Physical Address
127 * 52 PPR Enable
128 * 53 GPRP
129 * 54 Guest I/O Protection Valid (GIoV)
130 * 55 Guest Translation Valid (GV)
131 * 56:57 Guest Levels translated (GLX)
132 * 58:60 Guest CR3 bits 12:14 (GCR3TRP)
133 * 61 I/O Read Permission (IR)
134 * 62 I/O Write Permission (IW)
135 * 64:79 Domain ID
136 * 80:95 Guest CR3 bits 15:30 (GCR3TRP)
137 * 96 IOTLB Enable (I)
138 * 97 Suppress multiple I/O page faults (I)
139 * 98 Suppress all I/O page faults (SA)
140 * 99:100 Port I/O Control (IoCTL)
141 * 101 Cache IOTLB Hint
142 * 102 Snoop Disable (SD)
143 * 103 Allow Exclusion (EX)
144 * 104:105 System Management Message (SysMgt)
145 * 107:127 Guest CR3 bits 31:51 (GCR3TRP)
146 * 128 Interrupt Map Valid (IV)
147 * 129:132 Interrupt Table Length (IntTabLen)
148 *========================*/
149 struct ivhd_dte {
150 uint32_t dw0;
151 uint32_t dw1;
152 uint32_t dw2;
153 uint32_t dw3;
154 uint32_t dw4;
155 uint32_t dw5;
156 uint32_t dw6;
157 uint32_t dw7;
158 } __packed;
159
160 #define HWDTE_SIZE (65536 * sizeof(struct ivhd_dte))
161
162 #define DTE_V (1L << 0) /* dw0 */
163 #define DTE_TV (1L << 1) /* dw0 */
164 #define DTE_LEVEL_SHIFT 9 /* dw0 */
165 #define DTE_LEVEL_MASK 0x7 /* dw0 */
166 #define DTE_HPTRP_MASK 0x000FFFFFFFFFF000LL /* dw0,1 */
167
168 #define DTE_PPR (1L << 20) /* dw1 */
169 #define DTE_GPRP (1L << 21) /* dw1 */
170 #define DTE_GIOV (1L << 22) /* dw1 */
171 #define DTE_GV (1L << 23) /* dw1 */
172 #define DTE_IR (1L << 29) /* dw1 */
173 #define DTE_IW (1L << 30) /* dw1 */
174
175 #define DTE_DID_MASK 0xFFFF /* dw2 */
176
177 #define DTE_IV (1L << 0) /* dw3 */
178 #define DTE_SE (1L << 1)
179 #define DTE_SA (1L << 2)
180 #define DTE_INTTABLEN_SHIFT 1
181 #define DTE_INTTABLEN_MASK 0xF
182 #define DTE_IRTP_MASK 0x000FFFFFFFFFFFC0LL
183
184 #define PTE_LVL5 48
185 #define PTE_LVL4 39
186 #define PTE_LVL3 30
187 #define PTE_LVL2 21
188 #define PTE_LVL1 12
189
190 #define PTE_NXTLVL(x) (((x) & 0x7) << 9)
191 #define PTE_PADDR_MASK 0x000FFFFFFFFFF000LL
192 #define PTE_IR (1LL << 61)
193 #define PTE_IW (1LL << 62)
194
195 #define DTE_GCR312_MASK 0x3
196 #define DTE_GCR312_SHIFT 24
197
198 #define DTE_GCR315_MASK 0xFFFF
199 #define DTE_GCR315_SHIFT 16
200
201 #define DTE_GCR331_MASK 0xFFFFF
202 #define DTE_GCR331_SHIFT 12
203
204 #define _get64(x) *(uint64_t *)(x)
205 #define _put64(x,v) *(uint64_t *)(x) = (v)
206
207 /* Set Guest CR3 address */
208 static inline void
dte_set_guest_cr3(struct ivhd_dte * dte,paddr_t paddr)209 dte_set_guest_cr3(struct ivhd_dte *dte, paddr_t paddr)
210 {
211 iommu_rmw32(&dte->dw1, DTE_GCR312_MASK, DTE_GCR312_SHIFT, paddr >> 12);
212 iommu_rmw32(&dte->dw2, DTE_GCR315_MASK, DTE_GCR315_SHIFT, paddr >> 15);
213 iommu_rmw32(&dte->dw3, DTE_GCR331_MASK, DTE_GCR331_SHIFT, paddr >> 31);
214 }
215
216 /* Set Interrupt Remapping Root Pointer */
217 static inline void
dte_set_interrupt_table_root_ptr(struct ivhd_dte * dte,paddr_t paddr)218 dte_set_interrupt_table_root_ptr(struct ivhd_dte *dte, paddr_t paddr)
219 {
220 uint64_t ov = _get64(&dte->dw4);
221 _put64(&dte->dw4, (ov & ~DTE_IRTP_MASK) | (paddr & DTE_IRTP_MASK));
222 }
223
224 /* Set Interrupt Remapping Table length */
225 static inline void
dte_set_interrupt_table_length(struct ivhd_dte * dte,int nEnt)226 dte_set_interrupt_table_length(struct ivhd_dte *dte, int nEnt)
227 {
228 iommu_rmw32(&dte->dw4, DTE_INTTABLEN_MASK, DTE_INTTABLEN_SHIFT, nEnt);
229 }
230
231 /* Set Interrupt Remapping Valid */
232 static inline void
dte_set_interrupt_valid(struct ivhd_dte * dte)233 dte_set_interrupt_valid(struct ivhd_dte *dte)
234 {
235 dte->dw4 |= DTE_IV;
236 }
237
238 /* Set Domain ID in Device Table Entry */
239 static inline void
dte_set_domain(struct ivhd_dte * dte,uint16_t did)240 dte_set_domain(struct ivhd_dte *dte, uint16_t did)
241 {
242 dte->dw2 = (dte->dw2 & ~DTE_DID_MASK) | (did & DTE_DID_MASK);
243 }
244
245 /* Set Page Table Pointer for device */
246 static inline void
dte_set_host_page_table_root_ptr(struct ivhd_dte * dte,paddr_t paddr)247 dte_set_host_page_table_root_ptr(struct ivhd_dte *dte, paddr_t paddr)
248 {
249 uint64_t ov;
250
251 ov = _get64(&dte->dw0) & ~DTE_HPTRP_MASK;
252 ov |= (paddr & DTE_HPTRP_MASK) | PTE_IW | PTE_IR;
253
254 _put64(&dte->dw0, ov);
255 }
256
257 /* Set Page Table Levels Mask */
258 static inline void
dte_set_mode(struct ivhd_dte * dte,int mode)259 dte_set_mode(struct ivhd_dte *dte, int mode)
260 {
261 iommu_rmw32(&dte->dw0, DTE_LEVEL_MASK, DTE_LEVEL_SHIFT, mode);
262 }
263
264 static inline void
dte_set_tv(struct ivhd_dte * dte)265 dte_set_tv(struct ivhd_dte *dte)
266 {
267 dte->dw0 |= DTE_TV;
268 }
269
270 /* Set Device Table Entry valid.
271 * Domain/Level/Mode/PageTable should already be set
272 */
273 static inline void
dte_set_valid(struct ivhd_dte * dte)274 dte_set_valid(struct ivhd_dte *dte)
275 {
276 dte->dw0 |= DTE_V;
277 }
278
279 /* Check if Device Table Entry is valid */
280 static inline int
dte_is_valid(struct ivhd_dte * dte)281 dte_is_valid(struct ivhd_dte *dte)
282 {
283 return (dte->dw0 & DTE_V);
284 }
285
286 /*=========================================
287 * COMMAND
288 *=========================================*/
289 struct ivhd_command {
290 uint32_t dw0;
291 uint32_t dw1;
292 uint32_t dw2;
293 uint32_t dw3;
294 } __packed;
295
296 #define CMD_SHIFT 28
297
298 enum {
299 COMPLETION_WAIT = 0x01,
300 INVALIDATE_DEVTAB_ENTRY = 0x02,
301 INVALIDATE_IOMMU_PAGES = 0x03,
302 INVALIDATE_IOTLB_PAGES = 0x04,
303 INVALIDATE_INTERRUPT_TABLE = 0x05,
304 PREFETCH_IOMMU_PAGES = 0x06,
305 COMPLETE_PPR_REQUEST = 0x07,
306 INVALIDATE_IOMMU_ALL = 0x08,
307 };
308
309 /*=========================================
310 * EVENT
311 *=========================================*/
312 struct ivhd_event {
313 uint32_t dw0;
314 uint32_t dw1;
315 uint32_t dw2;
316 uint32_t dw3;
317 } __packed;
318
319 #define EVT_TYPE_SHIFT 28
320 #define EVT_TYPE_MASK 0xF
321 #define EVT_SID_SHIFT 0
322 #define EVT_SID_MASK 0xFFFF
323 #define EVT_DID_SHIFT 0
324 #define EVT_DID_MASK 0xFFFF
325 #define EVT_FLAG_SHIFT 16
326 #define EVT_FLAG_MASK 0xFFF
327
328 /* IOMMU Fault reasons */
329 enum {
330 ILLEGAL_DEV_TABLE_ENTRY = 0x1,
331 IO_PAGE_FAULT = 0x2,
332 DEV_TAB_HARDWARE_ERROR = 0x3,
333 PAGE_TAB_HARDWARE_ERROR = 0x4,
334 ILLEGAL_COMMAND_ERROR = 0x5,
335 COMMAND_HARDWARE_ERROR = 0x6,
336 IOTLB_INV_TIMEOUT = 0x7,
337 INVALID_DEVICE_REQUEST = 0x8,
338 };
339
340 #define EVT_GN (1L << 16)
341 #define EVT_NX (1L << 17)
342 #define EVT_US (1L << 18)
343 #define EVT_I (1L << 19)
344 #define EVT_PR (1L << 20)
345 #define EVT_RW (1L << 21)
346 #define EVT_PE (1L << 22)
347 #define EVT_RZ (1L << 23)
348 #define EVT_TR (1L << 24)
349
350 struct iommu_softc;
351
352 int ivhd_flush_devtab(struct iommu_softc *, int);
353 int ivhd_invalidate_iommu_all(struct iommu_softc *);
354 int ivhd_invalidate_interrupt_table(struct iommu_softc *, int);
355 int ivhd_issue_command(struct iommu_softc *, const struct ivhd_command *, int);
356 int ivhd_invalidate_domain(struct iommu_softc *, int);
357
358 void _dumppte(struct pte_entry *, int, vaddr_t);
359
360 #endif
361