1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019-2020 Ruslan Bukin <br@bsdpad.com>
5 *
6 * This software was developed by SRI International and the University of
7 * Cambridge Computer Laboratory (Department of Computer Science and
8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
9 * DARPA SSITH research programme.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 /*
34 * Hardware overview.
35 *
36 * An incoming transaction from a peripheral device has an address, size,
37 * attributes and StreamID.
38 *
39 * In case of PCI-based devices, StreamID is a PCI rid.
40 *
41 * The StreamID is used to select a Stream Table Entry (STE) in a Stream table,
42 * which contains per-device configuration.
43 *
44 * Stream table is a linear or 2-level walk table (this driver supports both).
45 * Note that a linear table could occupy 1GB or more of memory depending on
46 * sid_bits value.
47 *
48 * STE is used to locate a Context Descriptor, which is a struct in memory
49 * that describes stages of translation, translation table type, pointer to
50 * level 0 of page tables, ASID, etc.
51 *
52 * Hardware supports two stages of translation: Stage1 (S1) and Stage2 (S2):
53 * o S1 is used for the host machine traffic translation
54 * o S2 is for a hypervisor
55 *
56 * This driver enables S1 stage with standard AArch64 page tables.
57 *
58 * Note that SMMU does not share TLB with a main CPU.
59 * Command queue is used by this driver to Invalidate SMMU TLB, STE cache.
60 *
61 * An arm64 SoC could have more than one SMMU instance.
62 * ACPI IORT table describes which SMMU unit is assigned for a particular
63 * peripheral device.
64 *
65 * Queues.
66 *
67 * Register interface and Memory-based circular buffer queues are used
68 * to interface SMMU.
69 *
70 * These are a Command queue for commands to send to the SMMU and an Event
71 * queue for event/fault reports from the SMMU. Optionally PRI queue is
72 * designed for PCIe page requests reception.
73 *
74 * Note that not every hardware supports PRI services. For instance they were
75 * not found in Neoverse N1 SDP machine.
76 * (This drivers does not implement PRI queue.)
77 *
78 * All SMMU queues are arranged as circular buffers in memory. They are used
79 * in a producer-consumer fashion so that an output queue contains data
80 * produced by the SMMU and consumed by software.
81 * An input queue contains data produced by software, consumed by the SMMU.
82 *
83 * Interrupts.
84 *
85 * Interrupts are not required by this driver for normal operation.
86 * The standard wired interrupt is only triggered when an event comes from
87 * the SMMU, which is only in a case of errors (e.g. translation fault).
88 */
89
90 #include "opt_platform.h"
91 #include "opt_acpi.h"
92
93 #include <sys/param.h>
94 #include <sys/bitstring.h>
95 #include <sys/bus.h>
96 #include <sys/kernel.h>
97 #include <sys/malloc.h>
98 #include <sys/mutex.h>
99 #include <sys/rman.h>
100 #include <sys/lock.h>
101 #include <sys/sysctl.h>
102 #include <sys/tree.h>
103 #include <sys/taskqueue.h>
104 #include <vm/vm.h>
105 #include <vm/vm_page.h>
106 #ifdef DEV_ACPI
107 #include <contrib/dev/acpica/include/acpi.h>
108 #include <dev/acpica/acpivar.h>
109 #endif
110 #include <dev/pci/pcireg.h>
111 #include <dev/pci/pcivar.h>
112 #include <dev/iommu/iommu.h>
113 #include <arm64/iommu/iommu_pmap.h>
114
115 #include <machine/bus.h>
116
117 #ifdef FDT
118 #include <dev/fdt/fdt_common.h>
119 #include <dev/ofw/ofw_bus.h>
120 #include <dev/ofw/ofw_bus_subr.h>
121 #endif
122
123 #include "iommu.h"
124 #include "iommu_if.h"
125
126 #include "smmureg.h"
127 #include "smmuvar.h"
128
129 #define STRTAB_L1_SZ_SHIFT 20
130 #define STRTAB_SPLIT 8
131
132 #define STRTAB_L1_DESC_L2PTR_M (0x3fffffffffff << 6)
133 #define STRTAB_L1_DESC_DWORDS 1
134
135 #define STRTAB_STE_DWORDS 8
136
137 #define CMDQ_ENTRY_DWORDS 2
138 #define EVTQ_ENTRY_DWORDS 4
139 #define PRIQ_ENTRY_DWORDS 2
140
141 #define CD_DWORDS 8
142
143 #define Q_WRP(q, p) ((p) & (1 << (q)->size_log2))
144 #define Q_IDX(q, p) ((p) & ((1 << (q)->size_log2) - 1))
145 #define Q_OVF(p) ((p) & (1 << 31)) /* Event queue overflowed */
146
147 #define SMMU_Q_ALIGN (64 * 1024)
148
149 #define MAXADDR_48BIT 0xFFFFFFFFFFFFUL
150 #define MAXADDR_52BIT 0xFFFFFFFFFFFFFUL
151
152 static struct resource_spec smmu_spec[] = {
153 { SYS_RES_MEMORY, 0, RF_ACTIVE },
154 { SYS_RES_IRQ, 0, RF_ACTIVE },
155 { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL },
156 { SYS_RES_IRQ, 2, RF_ACTIVE },
157 { SYS_RES_IRQ, 3, RF_ACTIVE },
158 RESOURCE_SPEC_END
159 };
160
161 MALLOC_DEFINE(M_SMMU, "SMMU", SMMU_DEVSTR);
162
163 #define dprintf(fmt, ...)
164
165 struct smmu_event {
166 int ident;
167 char *str;
168 char *msg;
169 };
170
171 static struct smmu_event events[] = {
172 { 0x01, "F_UUT",
173 "Unsupported Upstream Transaction."},
174 { 0x02, "C_BAD_STREAMID",
175 "Transaction StreamID out of range."},
176 { 0x03, "F_STE_FETCH",
177 "Fetch of STE caused external abort."},
178 { 0x04, "C_BAD_STE",
179 "Used STE invalid."},
180 { 0x05, "F_BAD_ATS_TREQ",
181 "Address Translation Request disallowed for a StreamID "
182 "and a PCIe ATS Translation Request received."},
183 { 0x06, "F_STREAM_DISABLED",
184 "The STE of a transaction marks non-substream transactions "
185 "disabled."},
186 { 0x07, "F_TRANSL_FORBIDDEN",
187 "An incoming PCIe transaction is marked Translated but "
188 "SMMU bypass is disallowed for this StreamID."},
189 { 0x08, "C_BAD_SUBSTREAMID",
190 "Incoming SubstreamID present, but configuration is invalid."},
191 { 0x09, "F_CD_FETCH",
192 "Fetch of CD caused external abort."},
193 { 0x0a, "C_BAD_CD",
194 "Fetched CD invalid."},
195 { 0x0b, "F_WALK_EABT",
196 "An external abort occurred fetching (or updating) "
197 "a translation table descriptor."},
198 { 0x10, "F_TRANSLATION",
199 "Translation fault."},
200 { 0x11, "F_ADDR_SIZE",
201 "Address Size fault."},
202 { 0x12, "F_ACCESS",
203 "Access flag fault due to AF == 0 in a page or block TTD."},
204 { 0x13, "F_PERMISSION",
205 "Permission fault occurred on page access."},
206 { 0x20, "F_TLB_CONFLICT",
207 "A TLB conflict occurred because of the transaction."},
208 { 0x21, "F_CFG_CONFLICT",
209 "A configuration cache conflict occurred due to "
210 "the transaction."},
211 { 0x24, "E_PAGE_REQUEST",
212 "Speculative page request hint."},
213 { 0x25, "F_VMS_FETCH",
214 "Fetch of VMS caused external abort."},
215 { 0, NULL, NULL },
216 };
217
218 static int
smmu_q_has_space(struct smmu_queue * q)219 smmu_q_has_space(struct smmu_queue *q)
220 {
221
222 /*
223 * See 6.3.27 SMMU_CMDQ_PROD
224 *
225 * There is space in the queue for additional commands if:
226 * SMMU_CMDQ_CONS.RD != SMMU_CMDQ_PROD.WR ||
227 * SMMU_CMDQ_CONS.RD_WRAP == SMMU_CMDQ_PROD.WR_WRAP
228 */
229
230 if (Q_IDX(q, q->lc.cons) != Q_IDX(q, q->lc.prod) ||
231 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
232 return (1);
233
234 return (0);
235 }
236
237 static int
smmu_q_empty(struct smmu_queue * q)238 smmu_q_empty(struct smmu_queue *q)
239 {
240
241 if (Q_IDX(q, q->lc.cons) == Q_IDX(q, q->lc.prod) &&
242 Q_WRP(q, q->lc.cons) == Q_WRP(q, q->lc.prod))
243 return (1);
244
245 return (0);
246 }
247
248 static int __unused
smmu_q_consumed(struct smmu_queue * q,uint32_t prod)249 smmu_q_consumed(struct smmu_queue *q, uint32_t prod)
250 {
251
252 if ((Q_WRP(q, q->lc.cons) == Q_WRP(q, prod)) &&
253 (Q_IDX(q, q->lc.cons) >= Q_IDX(q, prod)))
254 return (1);
255
256 if ((Q_WRP(q, q->lc.cons) != Q_WRP(q, prod)) &&
257 (Q_IDX(q, q->lc.cons) <= Q_IDX(q, prod)))
258 return (1);
259
260 return (0);
261 }
262
263 static uint32_t
smmu_q_inc_cons(struct smmu_queue * q)264 smmu_q_inc_cons(struct smmu_queue *q)
265 {
266 uint32_t cons;
267 uint32_t val;
268
269 cons = (Q_WRP(q, q->lc.cons) | Q_IDX(q, q->lc.cons)) + 1;
270 val = (Q_OVF(q->lc.cons) | Q_WRP(q, cons) | Q_IDX(q, cons));
271
272 return (val);
273 }
274
275 static uint32_t
smmu_q_inc_prod(struct smmu_queue * q)276 smmu_q_inc_prod(struct smmu_queue *q)
277 {
278 uint32_t prod;
279 uint32_t val;
280
281 prod = (Q_WRP(q, q->lc.prod) | Q_IDX(q, q->lc.prod)) + 1;
282 val = (Q_OVF(q->lc.prod) | Q_WRP(q, prod) | Q_IDX(q, prod));
283
284 return (val);
285 }
286
287 static int
smmu_write_ack(struct smmu_softc * sc,uint32_t reg,uint32_t reg_ack,uint32_t val)288 smmu_write_ack(struct smmu_softc *sc, uint32_t reg,
289 uint32_t reg_ack, uint32_t val)
290 {
291 uint32_t v;
292 int timeout;
293
294 timeout = 100000;
295
296 bus_write_4(sc->res[0], reg, val);
297
298 do {
299 v = bus_read_4(sc->res[0], reg_ack);
300 if (v == val)
301 break;
302 } while (timeout--);
303
304 if (timeout <= 0) {
305 device_printf(sc->dev, "Failed to write reg.\n");
306 return (-1);
307 }
308
309 return (0);
310 }
311
312 static int
smmu_init_queue(struct smmu_softc * sc,struct smmu_queue * q,uint32_t prod_off,uint32_t cons_off,uint32_t dwords)313 smmu_init_queue(struct smmu_softc *sc, struct smmu_queue *q,
314 uint32_t prod_off, uint32_t cons_off, uint32_t dwords)
315 {
316 int sz;
317
318 sz = (1 << q->size_log2) * dwords * 8;
319
320 /* Set up the command circular buffer */
321 q->vaddr = contigmalloc(sz, M_SMMU,
322 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, SMMU_Q_ALIGN, 0);
323 if (q->vaddr == NULL) {
324 device_printf(sc->dev, "failed to allocate %d bytes\n", sz);
325 return (-1);
326 }
327
328 q->prod_off = prod_off;
329 q->cons_off = cons_off;
330 q->paddr = vtophys(q->vaddr);
331
332 q->base = CMDQ_BASE_RA | EVENTQ_BASE_WA | PRIQ_BASE_WA;
333 q->base |= q->paddr & Q_BASE_ADDR_M;
334 q->base |= q->size_log2 << Q_LOG2SIZE_S;
335
336 return (0);
337 }
338
339 static int
smmu_init_queues(struct smmu_softc * sc)340 smmu_init_queues(struct smmu_softc *sc)
341 {
342 int err;
343
344 /* Command queue. */
345 err = smmu_init_queue(sc, &sc->cmdq,
346 SMMU_CMDQ_PROD, SMMU_CMDQ_CONS, CMDQ_ENTRY_DWORDS);
347 if (err)
348 return (ENXIO);
349
350 /* Event queue. */
351 err = smmu_init_queue(sc, &sc->evtq,
352 SMMU_EVENTQ_PROD, SMMU_EVENTQ_CONS, EVTQ_ENTRY_DWORDS);
353 if (err)
354 return (ENXIO);
355
356 if (!(sc->features & SMMU_FEATURE_PRI))
357 return (0);
358
359 /* PRI queue. */
360 err = smmu_init_queue(sc, &sc->priq,
361 SMMU_PRIQ_PROD, SMMU_PRIQ_CONS, PRIQ_ENTRY_DWORDS);
362 if (err)
363 return (ENXIO);
364
365 return (0);
366 }
367
368 /*
369 * Dump 2LVL or linear STE.
370 */
371 static void
smmu_dump_ste(struct smmu_softc * sc,int sid)372 smmu_dump_ste(struct smmu_softc *sc, int sid)
373 {
374 struct smmu_strtab *strtab;
375 struct l1_desc *l1_desc;
376 uint64_t *ste, *l1;
377 int i;
378
379 strtab = &sc->strtab;
380
381 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
382 i = sid >> STRTAB_SPLIT;
383 l1 = (void *)((uint64_t)strtab->vaddr +
384 STRTAB_L1_DESC_DWORDS * 8 * i);
385 device_printf(sc->dev, "L1 ste == %lx\n", l1[0]);
386
387 l1_desc = &strtab->l1[i];
388 ste = l1_desc->va;
389 if (ste == NULL) /* L2 is not initialized */
390 return;
391 } else {
392 ste = (void *)((uint64_t)strtab->vaddr +
393 sid * (STRTAB_STE_DWORDS << 3));
394 }
395
396 /* Dump L2 or linear STE. */
397 for (i = 0; i < STRTAB_STE_DWORDS; i++)
398 device_printf(sc->dev, "ste[%d] == %lx\n", i, ste[i]);
399 }
400
401 static void __unused
smmu_dump_cd(struct smmu_softc * sc,struct smmu_cd * cd)402 smmu_dump_cd(struct smmu_softc *sc, struct smmu_cd *cd)
403 {
404 uint64_t *vaddr;
405 int i;
406
407 device_printf(sc->dev, "%s\n", __func__);
408
409 vaddr = cd->vaddr;
410 for (i = 0; i < CD_DWORDS; i++)
411 device_printf(sc->dev, "cd[%d] == %lx\n", i, vaddr[i]);
412 }
413
414 static void
smmu_evtq_dequeue(struct smmu_softc * sc,uint32_t * evt)415 smmu_evtq_dequeue(struct smmu_softc *sc, uint32_t *evt)
416 {
417 struct smmu_queue *evtq;
418 void *entry_addr;
419
420 evtq = &sc->evtq;
421
422 evtq->lc.val = bus_read_8(sc->res[0], evtq->prod_off);
423 entry_addr = (void *)((uint64_t)evtq->vaddr +
424 evtq->lc.cons * EVTQ_ENTRY_DWORDS * 8);
425 memcpy(evt, entry_addr, EVTQ_ENTRY_DWORDS * 8);
426 evtq->lc.cons = smmu_q_inc_cons(evtq);
427 bus_write_4(sc->res[0], evtq->cons_off, evtq->lc.cons);
428 }
429
430 static void
smmu_print_event(struct smmu_softc * sc,uint32_t * evt)431 smmu_print_event(struct smmu_softc *sc, uint32_t *evt)
432 {
433 struct smmu_event *ev;
434 uintptr_t input_addr;
435 uint8_t event_id;
436 device_t dev;
437 int sid;
438 int i;
439
440 dev = sc->dev;
441
442 ev = NULL;
443 event_id = evt[0] & 0xff;
444 for (i = 0; events[i].ident != 0; i++) {
445 if (events[i].ident == event_id) {
446 ev = &events[i];
447 break;
448 }
449 }
450
451 sid = evt[1];
452 input_addr = evt[5];
453 input_addr <<= 32;
454 input_addr |= evt[4];
455
456 if (smmu_quirks_check(dev, sid, event_id, input_addr)) {
457 /* The event is known. Don't print anything. */
458 return;
459 }
460
461 if (ev) {
462 device_printf(sc->dev,
463 "Event %s (%s) received.\n", ev->str, ev->msg);
464 } else
465 device_printf(sc->dev, "Event 0x%x received\n", event_id);
466
467 device_printf(sc->dev, "SID %x, Input Address: %jx\n",
468 sid, input_addr);
469
470 for (i = 0; i < 8; i++)
471 device_printf(sc->dev, "evt[%d] %x\n", i, evt[i]);
472
473 smmu_dump_ste(sc, sid);
474 }
475
476 static void
make_cmd(struct smmu_softc * sc,uint64_t * cmd,struct smmu_cmdq_entry * entry)477 make_cmd(struct smmu_softc *sc, uint64_t *cmd,
478 struct smmu_cmdq_entry *entry)
479 {
480
481 memset(cmd, 0, CMDQ_ENTRY_DWORDS * 8);
482 cmd[0] = entry->opcode << CMD_QUEUE_OPCODE_S;
483
484 switch (entry->opcode) {
485 case CMD_TLBI_NH_VA:
486 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
487 cmd[1] = entry->tlbi.addr & TLBI_1_ADDR_M;
488 if (entry->tlbi.leaf) {
489 /*
490 * Leaf flag means that only cached entries
491 * for the last level of translation table walk
492 * are required to be invalidated.
493 */
494 cmd[1] |= TLBI_1_LEAF;
495 }
496 break;
497 case CMD_TLBI_NH_ASID:
498 cmd[0] |= (uint64_t)entry->tlbi.asid << TLBI_0_ASID_S;
499 break;
500 case CMD_TLBI_NSNH_ALL:
501 case CMD_TLBI_NH_ALL:
502 case CMD_TLBI_EL2_ALL:
503 break;
504 case CMD_CFGI_CD:
505 cmd[0] |= ((uint64_t)entry->cfgi.ssid << CFGI_0_SSID_S);
506 /* FALLTROUGH */
507 case CMD_CFGI_STE:
508 cmd[0] |= ((uint64_t)entry->cfgi.sid << CFGI_0_STE_SID_S);
509 cmd[1] |= ((uint64_t)entry->cfgi.leaf << CFGI_1_LEAF_S);
510 break;
511 case CMD_CFGI_STE_RANGE:
512 cmd[1] = (31 << CFGI_1_STE_RANGE_S);
513 break;
514 case CMD_SYNC:
515 cmd[0] |= SYNC_0_MSH_IS | SYNC_0_MSIATTR_OIWB;
516 if (entry->sync.msiaddr) {
517 cmd[0] |= SYNC_0_CS_SIG_IRQ;
518 cmd[1] |= (entry->sync.msiaddr & SYNC_1_MSIADDRESS_M);
519 } else
520 cmd[0] |= SYNC_0_CS_SIG_SEV;
521 break;
522 case CMD_PREFETCH_CONFIG:
523 cmd[0] |= ((uint64_t)entry->prefetch.sid << PREFETCH_0_SID_S);
524 break;
525 };
526 }
527
528 static void
smmu_cmdq_enqueue_cmd(struct smmu_softc * sc,struct smmu_cmdq_entry * entry)529 smmu_cmdq_enqueue_cmd(struct smmu_softc *sc, struct smmu_cmdq_entry *entry)
530 {
531 uint64_t cmd[CMDQ_ENTRY_DWORDS];
532 struct smmu_queue *cmdq;
533 void *entry_addr;
534
535 cmdq = &sc->cmdq;
536
537 make_cmd(sc, cmd, entry);
538
539 SMMU_LOCK(sc);
540
541 /* Ensure that a space is available. */
542 do {
543 cmdq->lc.cons = bus_read_4(sc->res[0], cmdq->cons_off);
544 } while (smmu_q_has_space(cmdq) == 0);
545
546 /* Write the command to the current prod entry. */
547 entry_addr = (void *)((uint64_t)cmdq->vaddr +
548 Q_IDX(cmdq, cmdq->lc.prod) * CMDQ_ENTRY_DWORDS * 8);
549 memcpy(entry_addr, cmd, CMDQ_ENTRY_DWORDS * 8);
550
551 /* Increment prod index. */
552 cmdq->lc.prod = smmu_q_inc_prod(cmdq);
553 bus_write_4(sc->res[0], cmdq->prod_off, cmdq->lc.prod);
554
555 SMMU_UNLOCK(sc);
556 }
557
558 static void __unused
smmu_poll_until_consumed(struct smmu_softc * sc,struct smmu_queue * q)559 smmu_poll_until_consumed(struct smmu_softc *sc, struct smmu_queue *q)
560 {
561
562 while (1) {
563 q->lc.val = bus_read_8(sc->res[0], q->prod_off);
564 if (smmu_q_empty(q))
565 break;
566 cpu_spinwait();
567 }
568 }
569
570 static int
smmu_sync(struct smmu_softc * sc)571 smmu_sync(struct smmu_softc *sc)
572 {
573 struct smmu_cmdq_entry cmd;
574 struct smmu_queue *q;
575 uint32_t *base;
576 int timeout;
577 int prod;
578
579 q = &sc->cmdq;
580 prod = q->lc.prod;
581
582 /* Enqueue sync command. */
583 cmd.opcode = CMD_SYNC;
584 cmd.sync.msiaddr = q->paddr + Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8;
585 smmu_cmdq_enqueue_cmd(sc, &cmd);
586
587 /* Wait for the sync completion. */
588 base = (void *)((uint64_t)q->vaddr +
589 Q_IDX(q, prod) * CMDQ_ENTRY_DWORDS * 8);
590
591 /*
592 * It takes around 200 loops (6 instructions each)
593 * on Neoverse N1 to complete the sync.
594 */
595 timeout = 10000;
596
597 do {
598 if (*base == 0) {
599 /* MSI write completed. */
600 break;
601 }
602 cpu_spinwait();
603 } while (timeout--);
604
605 if (timeout < 0)
606 device_printf(sc->dev, "Failed to sync\n");
607
608 return (0);
609 }
610
611 static int
smmu_sync_cd(struct smmu_softc * sc,int sid,int ssid,bool leaf)612 smmu_sync_cd(struct smmu_softc *sc, int sid, int ssid, bool leaf)
613 {
614 struct smmu_cmdq_entry cmd;
615
616 cmd.opcode = CMD_CFGI_CD;
617 cmd.cfgi.sid = sid;
618 cmd.cfgi.ssid = ssid;
619 cmd.cfgi.leaf = leaf;
620 smmu_cmdq_enqueue_cmd(sc, &cmd);
621
622 return (0);
623 }
624
625 static void
smmu_invalidate_all_sid(struct smmu_softc * sc)626 smmu_invalidate_all_sid(struct smmu_softc *sc)
627 {
628 struct smmu_cmdq_entry cmd;
629
630 /* Invalidate cached config */
631 cmd.opcode = CMD_CFGI_STE_RANGE;
632 smmu_cmdq_enqueue_cmd(sc, &cmd);
633 smmu_sync(sc);
634 }
635
636 static void
smmu_tlbi_all(struct smmu_softc * sc)637 smmu_tlbi_all(struct smmu_softc *sc)
638 {
639 struct smmu_cmdq_entry cmd;
640
641 /* Invalidate entire TLB */
642 cmd.opcode = CMD_TLBI_NSNH_ALL;
643 smmu_cmdq_enqueue_cmd(sc, &cmd);
644 smmu_sync(sc);
645 }
646
647 static void
smmu_tlbi_asid(struct smmu_softc * sc,uint16_t asid)648 smmu_tlbi_asid(struct smmu_softc *sc, uint16_t asid)
649 {
650 struct smmu_cmdq_entry cmd;
651
652 /* Invalidate TLB for an ASID. */
653 cmd.opcode = CMD_TLBI_NH_ASID;
654 cmd.tlbi.asid = asid;
655 smmu_cmdq_enqueue_cmd(sc, &cmd);
656 smmu_sync(sc);
657 }
658
659 static void
smmu_tlbi_va(struct smmu_softc * sc,vm_offset_t va,uint16_t asid)660 smmu_tlbi_va(struct smmu_softc *sc, vm_offset_t va, uint16_t asid)
661 {
662 struct smmu_cmdq_entry cmd;
663
664 /* Invalidate specific range */
665 cmd.opcode = CMD_TLBI_NH_VA;
666 cmd.tlbi.asid = asid;
667 cmd.tlbi.vmid = 0;
668 cmd.tlbi.leaf = true; /* We change only L3. */
669 cmd.tlbi.addr = va;
670 smmu_cmdq_enqueue_cmd(sc, &cmd);
671 }
672
673 static void
smmu_invalidate_sid(struct smmu_softc * sc,uint32_t sid)674 smmu_invalidate_sid(struct smmu_softc *sc, uint32_t sid)
675 {
676 struct smmu_cmdq_entry cmd;
677
678 /* Invalidate cached config */
679 cmd.opcode = CMD_CFGI_STE;
680 cmd.cfgi.sid = sid;
681 smmu_cmdq_enqueue_cmd(sc, &cmd);
682 smmu_sync(sc);
683 }
684
685 static void
smmu_prefetch_sid(struct smmu_softc * sc,uint32_t sid)686 smmu_prefetch_sid(struct smmu_softc *sc, uint32_t sid)
687 {
688 struct smmu_cmdq_entry cmd;
689
690 cmd.opcode = CMD_PREFETCH_CONFIG;
691 cmd.prefetch.sid = sid;
692 smmu_cmdq_enqueue_cmd(sc, &cmd);
693 smmu_sync(sc);
694 }
695
696 /*
697 * Init STE in bypass mode. Traffic is not translated for the sid.
698 */
699 static void
smmu_init_ste_bypass(struct smmu_softc * sc,uint32_t sid,uint64_t * ste)700 smmu_init_ste_bypass(struct smmu_softc *sc, uint32_t sid, uint64_t *ste)
701 {
702 uint64_t val;
703
704 val = STE0_VALID | STE0_CONFIG_BYPASS;
705
706 ste[1] = STE1_SHCFG_INCOMING | STE1_EATS_FULLATS;
707 ste[2] = 0;
708 ste[3] = 0;
709 ste[4] = 0;
710 ste[5] = 0;
711 ste[6] = 0;
712 ste[7] = 0;
713
714 smmu_invalidate_sid(sc, sid);
715 ste[0] = val;
716 dsb(sy);
717 smmu_invalidate_sid(sc, sid);
718
719 smmu_prefetch_sid(sc, sid);
720 }
721
722 /*
723 * Enable Stage1 (S1) translation for the sid.
724 */
725 static int
smmu_init_ste_s1(struct smmu_softc * sc,struct smmu_cd * cd,uint32_t sid,uint64_t * ste)726 smmu_init_ste_s1(struct smmu_softc *sc, struct smmu_cd *cd,
727 uint32_t sid, uint64_t *ste)
728 {
729 uint64_t val;
730
731 val = STE0_VALID;
732
733 /* S1 */
734 ste[1] = STE1_EATS_FULLATS |
735 STE1_S1CSH_IS |
736 STE1_S1CIR_WBRA |
737 STE1_S1COR_WBRA |
738 STE1_STRW_NS_EL1;
739 ste[2] = 0;
740 ste[3] = 0;
741 ste[4] = 0;
742 ste[5] = 0;
743 ste[6] = 0;
744 ste[7] = 0;
745
746 if (sc->features & SMMU_FEATURE_STALL &&
747 ((sc->features & SMMU_FEATURE_STALL_FORCE) == 0))
748 ste[1] |= STE1_S1STALLD;
749
750 /* Configure STE */
751 val |= (cd->paddr & STE0_S1CONTEXTPTR_M);
752 val |= STE0_CONFIG_S1_TRANS;
753
754 smmu_invalidate_sid(sc, sid);
755
756 /* The STE[0] has to be written in a single blast, last of all. */
757 ste[0] = val;
758 dsb(sy);
759
760 smmu_invalidate_sid(sc, sid);
761 smmu_sync_cd(sc, sid, 0, true);
762 smmu_invalidate_sid(sc, sid);
763
764 /* The sid will be used soon most likely. */
765 smmu_prefetch_sid(sc, sid);
766
767 return (0);
768 }
769
770 static uint64_t *
smmu_get_ste_addr(struct smmu_softc * sc,int sid)771 smmu_get_ste_addr(struct smmu_softc *sc, int sid)
772 {
773 struct smmu_strtab *strtab;
774 struct l1_desc *l1_desc;
775 uint64_t *addr;
776
777 strtab = &sc->strtab;
778
779 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
780 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
781 addr = l1_desc->va;
782 addr += (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
783 } else {
784 addr = (void *)((uint64_t)strtab->vaddr +
785 STRTAB_STE_DWORDS * 8 * sid);
786 };
787
788 return (addr);
789 }
790
791 static int
smmu_init_ste(struct smmu_softc * sc,struct smmu_cd * cd,int sid,bool bypass)792 smmu_init_ste(struct smmu_softc *sc, struct smmu_cd *cd, int sid, bool bypass)
793 {
794 uint64_t *addr;
795
796 addr = smmu_get_ste_addr(sc, sid);
797
798 if (bypass)
799 smmu_init_ste_bypass(sc, sid, addr);
800 else
801 smmu_init_ste_s1(sc, cd, sid, addr);
802
803 smmu_sync(sc);
804
805 return (0);
806 }
807
808 static void
smmu_deinit_ste(struct smmu_softc * sc,int sid)809 smmu_deinit_ste(struct smmu_softc *sc, int sid)
810 {
811 uint64_t *ste;
812
813 ste = smmu_get_ste_addr(sc, sid);
814 ste[0] = 0;
815
816 smmu_invalidate_sid(sc, sid);
817 smmu_sync_cd(sc, sid, 0, true);
818 smmu_invalidate_sid(sc, sid);
819
820 smmu_sync(sc);
821 }
822
823 static int
smmu_init_cd(struct smmu_softc * sc,struct smmu_domain * domain)824 smmu_init_cd(struct smmu_softc *sc, struct smmu_domain *domain)
825 {
826 vm_paddr_t paddr;
827 uint64_t *ptr;
828 uint64_t val;
829 vm_size_t size;
830 struct smmu_cd *cd;
831 struct smmu_pmap *p;
832
833 size = 1 * (CD_DWORDS << 3);
834
835 p = &domain->p;
836 cd = domain->cd = malloc(sizeof(struct smmu_cd),
837 M_SMMU, M_WAITOK | M_ZERO);
838
839 cd->vaddr = contigmalloc(size, M_SMMU,
840 M_WAITOK | M_ZERO, /* flags */
841 0, /* low */
842 (1ul << 40) - 1, /* high */
843 size, /* alignment */
844 0); /* boundary */
845 if (cd->vaddr == NULL) {
846 device_printf(sc->dev, "Failed to allocate CD\n");
847 return (ENXIO);
848 }
849
850 cd->size = size;
851 cd->paddr = vtophys(cd->vaddr);
852
853 ptr = cd->vaddr;
854
855 val = CD0_VALID;
856 val |= CD0_AA64;
857 val |= CD0_R;
858 val |= CD0_A;
859 val |= CD0_ASET;
860 val |= (uint64_t)domain->asid << CD0_ASID_S;
861 val |= CD0_TG0_4KB;
862 val |= CD0_EPD1; /* Disable TT1 */
863 val |= ((64 - sc->ias) << CD0_T0SZ_S);
864 val |= CD0_IPS_48BITS;
865
866 paddr = p->sp_l0_paddr & CD1_TTB0_M;
867 KASSERT(paddr == p->sp_l0_paddr, ("bad allocation 1"));
868
869 ptr[1] = paddr;
870 ptr[2] = 0;
871 ptr[3] = MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) |
872 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) |
873 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) |
874 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH);
875
876 /* Install the CD. */
877 ptr[0] = val;
878
879 return (0);
880 }
881
882 static int
smmu_init_strtab_linear(struct smmu_softc * sc)883 smmu_init_strtab_linear(struct smmu_softc *sc)
884 {
885 struct smmu_strtab *strtab;
886 vm_paddr_t base;
887 uint32_t size;
888 uint64_t reg;
889
890 strtab = &sc->strtab;
891 strtab->num_l1_entries = (1 << sc->sid_bits);
892
893 size = strtab->num_l1_entries * (STRTAB_STE_DWORDS << 3);
894
895 if (bootverbose)
896 device_printf(sc->dev,
897 "%s: linear strtab size %d, num_l1_entries %d\n",
898 __func__, size, strtab->num_l1_entries);
899
900 strtab->vaddr = contigmalloc(size, M_SMMU,
901 M_WAITOK | M_ZERO, /* flags */
902 0, /* low */
903 (1ul << 48) - 1, /* high */
904 size, /* alignment */
905 0); /* boundary */
906 if (strtab->vaddr == NULL) {
907 device_printf(sc->dev, "failed to allocate strtab\n");
908 return (ENXIO);
909 }
910
911 reg = STRTAB_BASE_CFG_FMT_LINEAR;
912 reg |= sc->sid_bits << STRTAB_BASE_CFG_LOG2SIZE_S;
913 strtab->base_cfg = (uint32_t)reg;
914
915 base = vtophys(strtab->vaddr);
916
917 reg = base & STRTAB_BASE_ADDR_M;
918 KASSERT(reg == base, ("bad allocation 2"));
919 reg |= STRTAB_BASE_RA;
920 strtab->base = reg;
921
922 return (0);
923 }
924
925 static int
smmu_init_strtab_2lvl(struct smmu_softc * sc)926 smmu_init_strtab_2lvl(struct smmu_softc *sc)
927 {
928 struct smmu_strtab *strtab;
929 vm_paddr_t base;
930 uint64_t reg_base;
931 uint32_t l1size;
932 uint32_t size;
933 uint32_t reg;
934 int sz;
935
936 strtab = &sc->strtab;
937
938 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
939 size = min(size, sc->sid_bits - STRTAB_SPLIT);
940 strtab->num_l1_entries = (1 << size);
941 size += STRTAB_SPLIT;
942
943 l1size = strtab->num_l1_entries * (STRTAB_L1_DESC_DWORDS << 3);
944
945 if (bootverbose)
946 device_printf(sc->dev,
947 "%s: size %d, l1 entries %d, l1size %d\n",
948 __func__, size, strtab->num_l1_entries, l1size);
949
950 strtab->vaddr = contigmalloc(l1size, M_SMMU,
951 M_WAITOK | M_ZERO, /* flags */
952 0, /* low */
953 (1ul << 48) - 1, /* high */
954 l1size, /* alignment */
955 0); /* boundary */
956 if (strtab->vaddr == NULL) {
957 device_printf(sc->dev, "Failed to allocate 2lvl strtab.\n");
958 return (ENOMEM);
959 }
960
961 sz = strtab->num_l1_entries * sizeof(struct l1_desc);
962
963 strtab->l1 = malloc(sz, M_SMMU, M_WAITOK | M_ZERO);
964 if (strtab->l1 == NULL) {
965 contigfree(strtab->vaddr, l1size, M_SMMU);
966 return (ENOMEM);
967 }
968
969 reg = STRTAB_BASE_CFG_FMT_2LVL;
970 reg |= size << STRTAB_BASE_CFG_LOG2SIZE_S;
971 reg |= STRTAB_SPLIT << STRTAB_BASE_CFG_SPLIT_S;
972 strtab->base_cfg = (uint32_t)reg;
973
974 base = vtophys(strtab->vaddr);
975
976 reg_base = base & STRTAB_BASE_ADDR_M;
977 KASSERT(reg_base == base, ("bad allocation 3"));
978 reg_base |= STRTAB_BASE_RA;
979 strtab->base = reg_base;
980
981 return (0);
982 }
983
984 static int
smmu_init_strtab(struct smmu_softc * sc)985 smmu_init_strtab(struct smmu_softc *sc)
986 {
987 int error;
988
989 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE)
990 error = smmu_init_strtab_2lvl(sc);
991 else
992 error = smmu_init_strtab_linear(sc);
993
994 return (error);
995 }
996
997 static int
smmu_init_l1_entry(struct smmu_softc * sc,int sid)998 smmu_init_l1_entry(struct smmu_softc *sc, int sid)
999 {
1000 struct smmu_strtab *strtab;
1001 struct l1_desc *l1_desc;
1002 uint64_t *addr;
1003 uint64_t val;
1004 size_t size;
1005 int i;
1006
1007 strtab = &sc->strtab;
1008 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
1009 if (l1_desc->va) {
1010 /* Already allocated. */
1011 return (0);
1012 }
1013
1014 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1015
1016 l1_desc->span = STRTAB_SPLIT + 1;
1017 l1_desc->size = size;
1018 l1_desc->va = contigmalloc(size, M_SMMU,
1019 M_WAITOK | M_ZERO, /* flags */
1020 0, /* low */
1021 (1ul << 48) - 1, /* high */
1022 size, /* alignment */
1023 0); /* boundary */
1024 if (l1_desc->va == NULL) {
1025 device_printf(sc->dev, "failed to allocate l2 entry\n");
1026 return (ENXIO);
1027 }
1028
1029 l1_desc->pa = vtophys(l1_desc->va);
1030
1031 i = sid >> STRTAB_SPLIT;
1032 addr = (void *)((uint64_t)strtab->vaddr +
1033 STRTAB_L1_DESC_DWORDS * 8 * i);
1034
1035 /* Install the L1 entry. */
1036 val = l1_desc->pa & STRTAB_L1_DESC_L2PTR_M;
1037 KASSERT(val == l1_desc->pa, ("bad allocation 4"));
1038 val |= l1_desc->span;
1039 *addr = val;
1040
1041 return (0);
1042 }
1043
1044 static void __unused
smmu_deinit_l1_entry(struct smmu_softc * sc,int sid)1045 smmu_deinit_l1_entry(struct smmu_softc *sc, int sid)
1046 {
1047 struct smmu_strtab *strtab;
1048 struct l1_desc *l1_desc;
1049 uint64_t *addr;
1050 int i;
1051
1052 strtab = &sc->strtab;
1053
1054 i = sid >> STRTAB_SPLIT;
1055 addr = (void *)((uint64_t)strtab->vaddr +
1056 STRTAB_L1_DESC_DWORDS * 8 * i);
1057 *addr = 0;
1058
1059 l1_desc = &strtab->l1[sid >> STRTAB_SPLIT];
1060 contigfree(l1_desc->va, l1_desc->size, M_SMMU);
1061 }
1062
1063 static int
smmu_disable(struct smmu_softc * sc)1064 smmu_disable(struct smmu_softc *sc)
1065 {
1066 uint32_t reg;
1067 int error;
1068
1069 /* Disable SMMU */
1070 reg = bus_read_4(sc->res[0], SMMU_CR0);
1071 reg &= ~CR0_SMMUEN;
1072 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1073 if (error)
1074 device_printf(sc->dev, "Could not disable SMMU.\n");
1075
1076 return (0);
1077 }
1078
1079 static int
smmu_event_intr(void * arg)1080 smmu_event_intr(void *arg)
1081 {
1082 uint32_t evt[EVTQ_ENTRY_DWORDS * 2];
1083 struct smmu_softc *sc;
1084
1085 sc = arg;
1086
1087 do {
1088 smmu_evtq_dequeue(sc, evt);
1089 smmu_print_event(sc, evt);
1090 } while (!smmu_q_empty(&sc->evtq));
1091
1092 return (FILTER_HANDLED);
1093 }
1094
1095 static int __unused
smmu_sync_intr(void * arg)1096 smmu_sync_intr(void *arg)
1097 {
1098 struct smmu_softc *sc;
1099
1100 sc = arg;
1101
1102 device_printf(sc->dev, "%s\n", __func__);
1103
1104 return (FILTER_HANDLED);
1105 }
1106
1107 static int
smmu_gerr_intr(void * arg)1108 smmu_gerr_intr(void *arg)
1109 {
1110 struct smmu_softc *sc;
1111
1112 sc = arg;
1113
1114 device_printf(sc->dev, "SMMU Global Error\n");
1115
1116 return (FILTER_HANDLED);
1117 }
1118
1119 static int
smmu_enable_interrupts(struct smmu_softc * sc)1120 smmu_enable_interrupts(struct smmu_softc *sc)
1121 {
1122 uint32_t reg;
1123 int error;
1124
1125 /* Disable MSI. */
1126 bus_write_8(sc->res[0], SMMU_GERROR_IRQ_CFG0, 0);
1127 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG1, 0);
1128 bus_write_4(sc->res[0], SMMU_GERROR_IRQ_CFG2, 0);
1129
1130 bus_write_8(sc->res[0], SMMU_EVENTQ_IRQ_CFG0, 0);
1131 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG1, 0);
1132 bus_write_4(sc->res[0], SMMU_EVENTQ_IRQ_CFG2, 0);
1133
1134 if (sc->features & CR0_PRIQEN) {
1135 bus_write_8(sc->res[0], SMMU_PRIQ_IRQ_CFG0, 0);
1136 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG1, 0);
1137 bus_write_4(sc->res[0], SMMU_PRIQ_IRQ_CFG2, 0);
1138 }
1139
1140 /* Disable any interrupts. */
1141 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, 0);
1142 if (error) {
1143 device_printf(sc->dev, "Could not disable interrupts.\n");
1144 return (ENXIO);
1145 }
1146
1147 /* Enable interrupts. */
1148 reg = IRQ_CTRL_EVENTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
1149 if (sc->features & SMMU_FEATURE_PRI)
1150 reg |= IRQ_CTRL_PRIQ_IRQEN;
1151
1152 error = smmu_write_ack(sc, SMMU_IRQ_CTRL, SMMU_IRQ_CTRLACK, reg);
1153 if (error) {
1154 device_printf(sc->dev, "Could not enable interrupts.\n");
1155 return (ENXIO);
1156 }
1157
1158 return (0);
1159 }
1160
1161 #ifdef DEV_ACPI
1162 static void
smmu_configure_intr(struct smmu_softc * sc,struct resource * res)1163 smmu_configure_intr(struct smmu_softc *sc, struct resource *res)
1164 {
1165 struct intr_map_data_acpi *ad;
1166 struct intr_map_data *data;
1167
1168 data = rman_get_virtual(res);
1169 KASSERT(data != NULL, ("data is NULL"));
1170
1171 if (data->type == INTR_MAP_DATA_ACPI) {
1172 ad = (struct intr_map_data_acpi *)data;
1173 ad->trig = INTR_TRIGGER_EDGE;
1174 ad->pol = INTR_POLARITY_HIGH;
1175 }
1176 }
1177 #endif
1178
1179 static int
smmu_setup_interrupts(struct smmu_softc * sc)1180 smmu_setup_interrupts(struct smmu_softc *sc)
1181 {
1182 device_t dev;
1183 int error;
1184
1185 dev = sc->dev;
1186
1187 #ifdef DEV_ACPI
1188 /*
1189 * Configure SMMU interrupts as EDGE triggered manually
1190 * as ACPI tables carries no information for that.
1191 */
1192 smmu_configure_intr(sc, sc->res[1]);
1193 /* PRIQ is not in use. */
1194 smmu_configure_intr(sc, sc->res[3]);
1195 smmu_configure_intr(sc, sc->res[4]);
1196 #endif
1197
1198 error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC,
1199 smmu_event_intr, NULL, sc, &sc->intr_cookie[0]);
1200 if (error) {
1201 device_printf(dev, "Couldn't setup Event interrupt handler\n");
1202 return (ENXIO);
1203 }
1204
1205 error = bus_setup_intr(dev, sc->res[4], INTR_TYPE_MISC,
1206 smmu_gerr_intr, NULL, sc, &sc->intr_cookie[2]);
1207 if (error) {
1208 device_printf(dev, "Couldn't setup Gerr interrupt handler\n");
1209 return (ENXIO);
1210 }
1211
1212 return (0);
1213 }
1214
1215 static int
smmu_reset(struct smmu_softc * sc)1216 smmu_reset(struct smmu_softc *sc)
1217 {
1218 struct smmu_cmdq_entry cmd;
1219 struct smmu_strtab *strtab;
1220 int error;
1221 int reg;
1222
1223 reg = bus_read_4(sc->res[0], SMMU_CR0);
1224
1225 if (reg & CR0_SMMUEN)
1226 device_printf(sc->dev,
1227 "%s: Warning: SMMU is enabled\n", __func__);
1228
1229 error = smmu_disable(sc);
1230 if (error)
1231 device_printf(sc->dev,
1232 "%s: Could not disable SMMU.\n", __func__);
1233
1234 if (smmu_enable_interrupts(sc) != 0) {
1235 device_printf(sc->dev, "Could not enable interrupts.\n");
1236 return (ENXIO);
1237 }
1238
1239 reg = CR1_TABLE_SH_IS |
1240 CR1_TABLE_OC_WBC |
1241 CR1_TABLE_IC_WBC |
1242 CR1_QUEUE_SH_IS |
1243 CR1_QUEUE_OC_WBC |
1244 CR1_QUEUE_IC_WBC;
1245 bus_write_4(sc->res[0], SMMU_CR1, reg);
1246
1247 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
1248 bus_write_4(sc->res[0], SMMU_CR2, reg);
1249
1250 /* Stream table. */
1251 strtab = &sc->strtab;
1252 bus_write_8(sc->res[0], SMMU_STRTAB_BASE, strtab->base);
1253 bus_write_4(sc->res[0], SMMU_STRTAB_BASE_CFG, strtab->base_cfg);
1254
1255 /* Command queue. */
1256 bus_write_8(sc->res[0], SMMU_CMDQ_BASE, sc->cmdq.base);
1257 bus_write_4(sc->res[0], SMMU_CMDQ_PROD, sc->cmdq.lc.prod);
1258 bus_write_4(sc->res[0], SMMU_CMDQ_CONS, sc->cmdq.lc.cons);
1259
1260 reg = CR0_CMDQEN;
1261 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1262 if (error) {
1263 device_printf(sc->dev, "Could not enable command queue\n");
1264 return (ENXIO);
1265 }
1266
1267 /* Invalidate cached configuration. */
1268 smmu_invalidate_all_sid(sc);
1269
1270 if (sc->features & SMMU_FEATURE_HYP) {
1271 cmd.opcode = CMD_TLBI_EL2_ALL;
1272 smmu_cmdq_enqueue_cmd(sc, &cmd);
1273 };
1274
1275 /* Invalidate TLB. */
1276 smmu_tlbi_all(sc);
1277
1278 /* Event queue */
1279 bus_write_8(sc->res[0], SMMU_EVENTQ_BASE, sc->evtq.base);
1280 bus_write_4(sc->res[0], SMMU_EVENTQ_PROD, sc->evtq.lc.prod);
1281 bus_write_4(sc->res[0], SMMU_EVENTQ_CONS, sc->evtq.lc.cons);
1282
1283 reg |= CR0_EVENTQEN;
1284 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1285 if (error) {
1286 device_printf(sc->dev, "Could not enable event queue\n");
1287 return (ENXIO);
1288 }
1289
1290 if (sc->features & SMMU_FEATURE_PRI) {
1291 /* PRI queue */
1292 bus_write_8(sc->res[0], SMMU_PRIQ_BASE, sc->priq.base);
1293 bus_write_4(sc->res[0], SMMU_PRIQ_PROD, sc->priq.lc.prod);
1294 bus_write_4(sc->res[0], SMMU_PRIQ_CONS, sc->priq.lc.cons);
1295
1296 reg |= CR0_PRIQEN;
1297 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1298 if (error) {
1299 device_printf(sc->dev, "Could not enable PRI queue\n");
1300 return (ENXIO);
1301 }
1302 }
1303
1304 if (sc->features & SMMU_FEATURE_ATS) {
1305 reg |= CR0_ATSCHK;
1306 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1307 if (error) {
1308 device_printf(sc->dev, "Could not enable ATS check.\n");
1309 return (ENXIO);
1310 }
1311 }
1312
1313 reg |= CR0_SMMUEN;
1314 error = smmu_write_ack(sc, SMMU_CR0, SMMU_CR0ACK, reg);
1315 if (error) {
1316 device_printf(sc->dev, "Could not enable SMMU.\n");
1317 return (ENXIO);
1318 }
1319
1320 return (0);
1321 }
1322
1323 static int
smmu_check_features(struct smmu_softc * sc)1324 smmu_check_features(struct smmu_softc *sc)
1325 {
1326 uint32_t reg;
1327 uint32_t val;
1328
1329 sc->features = 0;
1330
1331 reg = bus_read_4(sc->res[0], SMMU_IDR0);
1332
1333 if (reg & IDR0_ST_LVL_2) {
1334 if (bootverbose)
1335 device_printf(sc->dev,
1336 "2-level stream table supported.\n");
1337 sc->features |= SMMU_FEATURE_2_LVL_STREAM_TABLE;
1338 }
1339
1340 if (reg & IDR0_CD2L) {
1341 if (bootverbose)
1342 device_printf(sc->dev,
1343 "2-level CD table supported.\n");
1344 sc->features |= SMMU_FEATURE_2_LVL_CD;
1345 }
1346
1347 switch (reg & IDR0_TTENDIAN_M) {
1348 case IDR0_TTENDIAN_MIXED:
1349 if (bootverbose)
1350 device_printf(sc->dev, "Mixed endianness supported.\n");
1351 sc->features |= SMMU_FEATURE_TT_LE;
1352 sc->features |= SMMU_FEATURE_TT_BE;
1353 break;
1354 case IDR0_TTENDIAN_LITTLE:
1355 if (bootverbose)
1356 device_printf(sc->dev,
1357 "Little endian supported only.\n");
1358 sc->features |= SMMU_FEATURE_TT_LE;
1359 break;
1360 case IDR0_TTENDIAN_BIG:
1361 if (bootverbose)
1362 device_printf(sc->dev, "Big endian supported only.\n");
1363 sc->features |= SMMU_FEATURE_TT_BE;
1364 break;
1365 default:
1366 device_printf(sc->dev, "Unsupported endianness.\n");
1367 return (ENXIO);
1368 }
1369
1370 if (reg & IDR0_SEV)
1371 sc->features |= SMMU_FEATURE_SEV;
1372
1373 if (reg & IDR0_MSI) {
1374 if (bootverbose)
1375 device_printf(sc->dev, "MSI feature present.\n");
1376 sc->features |= SMMU_FEATURE_MSI;
1377 }
1378
1379 if (reg & IDR0_HYP) {
1380 if (bootverbose)
1381 device_printf(sc->dev, "HYP feature present.\n");
1382 sc->features |= SMMU_FEATURE_HYP;
1383 }
1384
1385 if (reg & IDR0_ATS)
1386 sc->features |= SMMU_FEATURE_ATS;
1387
1388 if (reg & IDR0_PRI)
1389 sc->features |= SMMU_FEATURE_PRI;
1390
1391 switch (reg & IDR0_STALL_MODEL_M) {
1392 case IDR0_STALL_MODEL_FORCE:
1393 /* Stall is forced. */
1394 sc->features |= SMMU_FEATURE_STALL_FORCE;
1395 /* FALLTHROUGH */
1396 case IDR0_STALL_MODEL_STALL:
1397 sc->features |= SMMU_FEATURE_STALL;
1398 break;
1399 }
1400
1401 /* Grab translation stages supported. */
1402 if (reg & IDR0_S1P) {
1403 if (bootverbose)
1404 device_printf(sc->dev,
1405 "Stage 1 translation supported.\n");
1406 sc->features |= SMMU_FEATURE_S1P;
1407 }
1408 if (reg & IDR0_S2P) {
1409 if (bootverbose)
1410 device_printf(sc->dev,
1411 "Stage 2 translation supported.\n");
1412 sc->features |= SMMU_FEATURE_S2P;
1413 }
1414
1415 switch (reg & IDR0_TTF_M) {
1416 case IDR0_TTF_ALL:
1417 case IDR0_TTF_AA64:
1418 sc->ias = 40;
1419 break;
1420 default:
1421 device_printf(sc->dev, "No AArch64 table format support.\n");
1422 return (ENXIO);
1423 }
1424
1425 if (reg & IDR0_ASID16)
1426 sc->asid_bits = 16;
1427 else
1428 sc->asid_bits = 8;
1429
1430 if (bootverbose)
1431 device_printf(sc->dev, "ASID bits %d\n", sc->asid_bits);
1432
1433 if (reg & IDR0_VMID16)
1434 sc->vmid_bits = 16;
1435 else
1436 sc->vmid_bits = 8;
1437
1438 reg = bus_read_4(sc->res[0], SMMU_IDR1);
1439
1440 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
1441 device_printf(sc->dev,
1442 "Embedded implementations not supported by this driver.\n");
1443 return (ENXIO);
1444 }
1445
1446 val = (reg & IDR1_CMDQS_M) >> IDR1_CMDQS_S;
1447 sc->cmdq.size_log2 = val;
1448 if (bootverbose)
1449 device_printf(sc->dev, "CMD queue bits %d\n", val);
1450
1451 val = (reg & IDR1_EVENTQS_M) >> IDR1_EVENTQS_S;
1452 sc->evtq.size_log2 = val;
1453 if (bootverbose)
1454 device_printf(sc->dev, "EVENT queue bits %d\n", val);
1455
1456 if (sc->features & SMMU_FEATURE_PRI) {
1457 val = (reg & IDR1_PRIQS_M) >> IDR1_PRIQS_S;
1458 sc->priq.size_log2 = val;
1459 if (bootverbose)
1460 device_printf(sc->dev, "PRI queue bits %d\n", val);
1461 }
1462
1463 sc->ssid_bits = (reg & IDR1_SSIDSIZE_M) >> IDR1_SSIDSIZE_S;
1464 sc->sid_bits = (reg & IDR1_SIDSIZE_M) >> IDR1_SIDSIZE_S;
1465
1466 if (sc->sid_bits <= STRTAB_SPLIT)
1467 sc->features &= ~SMMU_FEATURE_2_LVL_STREAM_TABLE;
1468
1469 if (bootverbose) {
1470 device_printf(sc->dev, "SSID bits %d\n", sc->ssid_bits);
1471 device_printf(sc->dev, "SID bits %d\n", sc->sid_bits);
1472 }
1473
1474 /* IDR3 */
1475 reg = bus_read_4(sc->res[0], SMMU_IDR3);
1476 if (reg & IDR3_RIL)
1477 sc->features |= SMMU_FEATURE_RANGE_INV;
1478
1479 /* IDR5 */
1480 reg = bus_read_4(sc->res[0], SMMU_IDR5);
1481
1482 switch (reg & IDR5_OAS_M) {
1483 case IDR5_OAS_32:
1484 sc->oas = 32;
1485 break;
1486 case IDR5_OAS_36:
1487 sc->oas = 36;
1488 break;
1489 case IDR5_OAS_40:
1490 sc->oas = 40;
1491 break;
1492 case IDR5_OAS_42:
1493 sc->oas = 42;
1494 break;
1495 case IDR5_OAS_44:
1496 sc->oas = 44;
1497 break;
1498 case IDR5_OAS_48:
1499 sc->oas = 48;
1500 break;
1501 case IDR5_OAS_52:
1502 sc->oas = 52;
1503 break;
1504 }
1505
1506 sc->pgsizes = 0;
1507 if (reg & IDR5_GRAN64K)
1508 sc->pgsizes |= 64 * 1024;
1509 if (reg & IDR5_GRAN16K)
1510 sc->pgsizes |= 16 * 1024;
1511 if (reg & IDR5_GRAN4K)
1512 sc->pgsizes |= 4 * 1024;
1513
1514 if ((reg & IDR5_VAX_M) == IDR5_VAX_52)
1515 sc->features |= SMMU_FEATURE_VAX;
1516
1517 return (0);
1518 }
1519
1520 static void
smmu_init_asids(struct smmu_softc * sc)1521 smmu_init_asids(struct smmu_softc *sc)
1522 {
1523
1524 sc->asid_set_size = (1 << sc->asid_bits);
1525 sc->asid_set = bit_alloc(sc->asid_set_size, M_SMMU, M_WAITOK);
1526 mtx_init(&sc->asid_set_mutex, "asid set", NULL, MTX_SPIN);
1527 }
1528
1529 static int
smmu_asid_alloc(struct smmu_softc * sc,int * new_asid)1530 smmu_asid_alloc(struct smmu_softc *sc, int *new_asid)
1531 {
1532
1533 mtx_lock_spin(&sc->asid_set_mutex);
1534 bit_ffc(sc->asid_set, sc->asid_set_size, new_asid);
1535 if (*new_asid == -1) {
1536 mtx_unlock_spin(&sc->asid_set_mutex);
1537 return (ENOMEM);
1538 }
1539 bit_set(sc->asid_set, *new_asid);
1540 mtx_unlock_spin(&sc->asid_set_mutex);
1541
1542 return (0);
1543 }
1544
1545 static void
smmu_asid_free(struct smmu_softc * sc,int asid)1546 smmu_asid_free(struct smmu_softc *sc, int asid)
1547 {
1548
1549 mtx_lock_spin(&sc->asid_set_mutex);
1550 bit_clear(sc->asid_set, asid);
1551 mtx_unlock_spin(&sc->asid_set_mutex);
1552 }
1553
1554 /*
1555 * Device interface.
1556 */
1557 int
smmu_attach(device_t dev)1558 smmu_attach(device_t dev)
1559 {
1560 struct smmu_softc *sc;
1561 int error;
1562
1563 sc = device_get_softc(dev);
1564 sc->dev = dev;
1565
1566 mtx_init(&sc->sc_mtx, device_get_nameunit(sc->dev), "smmu", MTX_DEF);
1567
1568 error = smmu_setup_interrupts(sc);
1569 if (error) {
1570 bus_release_resources(dev, smmu_spec, sc->res);
1571 return (ENXIO);
1572 }
1573
1574 error = smmu_check_features(sc);
1575 if (error) {
1576 device_printf(dev, "Some features are required "
1577 "but not supported by hardware.\n");
1578 return (ENXIO);
1579 }
1580
1581 smmu_init_asids(sc);
1582
1583 error = smmu_init_queues(sc);
1584 if (error) {
1585 device_printf(dev, "Couldn't allocate queues.\n");
1586 return (ENXIO);
1587 }
1588
1589 error = smmu_init_strtab(sc);
1590 if (error) {
1591 device_printf(dev, "Couldn't allocate strtab.\n");
1592 return (ENXIO);
1593 }
1594
1595 error = smmu_reset(sc);
1596 if (error) {
1597 device_printf(dev, "Couldn't reset SMMU.\n");
1598 return (ENXIO);
1599 }
1600
1601 return (0);
1602 }
1603
1604 int
smmu_detach(device_t dev)1605 smmu_detach(device_t dev)
1606 {
1607 struct smmu_softc *sc;
1608
1609 sc = device_get_softc(dev);
1610
1611 bus_release_resources(dev, smmu_spec, sc->res);
1612
1613 return (0);
1614 }
1615
1616 static int
smmu_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)1617 smmu_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
1618 {
1619 struct smmu_softc *sc;
1620
1621 sc = device_get_softc(dev);
1622
1623 device_printf(sc->dev, "%s\n", __func__);
1624
1625 return (ENOENT);
1626 }
1627
1628 static int
smmu_unmap(device_t dev,struct iommu_domain * iodom,vm_offset_t va,bus_size_t size)1629 smmu_unmap(device_t dev, struct iommu_domain *iodom,
1630 vm_offset_t va, bus_size_t size)
1631 {
1632 struct smmu_domain *domain;
1633 struct smmu_softc *sc;
1634 int err;
1635 int i;
1636
1637 sc = device_get_softc(dev);
1638
1639 domain = (struct smmu_domain *)iodom;
1640
1641 err = 0;
1642
1643 dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid);
1644
1645 for (i = 0; i < size; i += PAGE_SIZE) {
1646 if (smmu_pmap_remove(&domain->p, va) == 0) {
1647 /* pmap entry removed, invalidate TLB. */
1648 smmu_tlbi_va(sc, va, domain->asid);
1649 } else {
1650 err = ENOENT;
1651 break;
1652 }
1653 va += PAGE_SIZE;
1654 }
1655
1656 smmu_sync(sc);
1657
1658 return (err);
1659 }
1660
1661 static int
smmu_map(device_t dev,struct iommu_domain * iodom,vm_offset_t va,vm_page_t * ma,vm_size_t size,vm_prot_t prot)1662 smmu_map(device_t dev, struct iommu_domain *iodom,
1663 vm_offset_t va, vm_page_t *ma, vm_size_t size,
1664 vm_prot_t prot)
1665 {
1666 struct smmu_domain *domain;
1667 struct smmu_softc *sc;
1668 vm_paddr_t pa;
1669 int error;
1670 int i;
1671
1672 sc = device_get_softc(dev);
1673
1674 domain = (struct smmu_domain *)iodom;
1675
1676 dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size,
1677 domain->asid);
1678
1679 for (i = 0; size > 0; size -= PAGE_SIZE) {
1680 pa = VM_PAGE_TO_PHYS(ma[i++]);
1681 error = smmu_pmap_enter(&domain->p, va, pa, prot, 0);
1682 if (error)
1683 return (error);
1684 smmu_tlbi_va(sc, va, domain->asid);
1685 va += PAGE_SIZE;
1686 }
1687
1688 smmu_sync(sc);
1689
1690 return (0);
1691 }
1692
1693 static struct iommu_domain *
smmu_domain_alloc(device_t dev,struct iommu_unit * iommu)1694 smmu_domain_alloc(device_t dev, struct iommu_unit *iommu)
1695 {
1696 struct iommu_domain *iodom;
1697 struct smmu_domain *domain;
1698 struct smmu_unit *unit;
1699 struct smmu_softc *sc;
1700 int error;
1701 int new_asid;
1702
1703 sc = device_get_softc(dev);
1704
1705 unit = (struct smmu_unit *)iommu;
1706
1707 domain = malloc(sizeof(*domain), M_SMMU, M_WAITOK | M_ZERO);
1708
1709 error = smmu_asid_alloc(sc, &new_asid);
1710 if (error) {
1711 free(domain, M_SMMU);
1712 device_printf(sc->dev,
1713 "Could not allocate ASID for a new domain.\n");
1714 return (NULL);
1715 }
1716
1717 domain->asid = (uint16_t)new_asid;
1718
1719 smmu_pmap_pinit(&domain->p);
1720
1721 error = smmu_init_cd(sc, domain);
1722 if (error) {
1723 free(domain, M_SMMU);
1724 device_printf(sc->dev, "Could not initialize CD\n");
1725 return (NULL);
1726 }
1727
1728 smmu_tlbi_asid(sc, domain->asid);
1729
1730 LIST_INIT(&domain->ctx_list);
1731
1732 IOMMU_LOCK(iommu);
1733 LIST_INSERT_HEAD(&unit->domain_list, domain, next);
1734 IOMMU_UNLOCK(iommu);
1735
1736 iodom = &domain->iodom;
1737
1738 /*
1739 * Use 48-bit address space regardless of VAX bit
1740 * as we need 64k IOMMU_PAGE_SIZE for 52-bit space.
1741 */
1742 iodom->end = MAXADDR_48BIT;
1743
1744 return (iodom);
1745 }
1746
1747 static void
smmu_domain_free(device_t dev,struct iommu_domain * iodom)1748 smmu_domain_free(device_t dev, struct iommu_domain *iodom)
1749 {
1750 struct smmu_domain *domain;
1751 struct smmu_softc *sc;
1752 struct smmu_cd *cd;
1753
1754 sc = device_get_softc(dev);
1755
1756 domain = (struct smmu_domain *)iodom;
1757
1758 LIST_REMOVE(domain, next);
1759
1760 cd = domain->cd;
1761
1762 smmu_pmap_remove_pages(&domain->p);
1763 smmu_pmap_release(&domain->p);
1764
1765 smmu_tlbi_asid(sc, domain->asid);
1766 smmu_asid_free(sc, domain->asid);
1767
1768 contigfree(cd->vaddr, cd->size, M_SMMU);
1769 free(cd, M_SMMU);
1770
1771 free(domain, M_SMMU);
1772 }
1773
1774 static int
smmu_set_buswide(device_t dev,struct smmu_domain * domain,struct smmu_ctx * ctx)1775 smmu_set_buswide(device_t dev, struct smmu_domain *domain,
1776 struct smmu_ctx *ctx)
1777 {
1778 struct smmu_softc *sc;
1779 int i;
1780
1781 sc = device_get_softc(dev);
1782
1783 for (i = 0; i < PCI_SLOTMAX; i++)
1784 smmu_init_ste(sc, domain->cd, (ctx->sid | i), ctx->bypass);
1785
1786 return (0);
1787 }
1788
1789 static int
smmu_pci_get_sid(device_t child,u_int * xref0,u_int * sid0)1790 smmu_pci_get_sid(device_t child, u_int *xref0, u_int *sid0)
1791 {
1792 struct pci_id_ofw_iommu pi;
1793 int err;
1794
1795 err = pci_get_id(child, PCI_ID_OFW_IOMMU, (uintptr_t *)&pi);
1796 if (err == 0) {
1797 if (sid0)
1798 *sid0 = pi.id;
1799 if (xref0)
1800 *xref0 = pi.xref;
1801 }
1802
1803 return (err);
1804 }
1805
1806 static struct iommu_ctx *
smmu_ctx_alloc(device_t dev,struct iommu_domain * iodom,device_t child,bool disabled)1807 smmu_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child,
1808 bool disabled)
1809 {
1810 struct smmu_domain *domain;
1811 struct smmu_ctx *ctx;
1812
1813 domain = (struct smmu_domain *)iodom;
1814
1815 ctx = malloc(sizeof(struct smmu_ctx), M_SMMU, M_WAITOK | M_ZERO);
1816 ctx->dev = child;
1817 ctx->domain = domain;
1818 if (disabled)
1819 ctx->bypass = true;
1820
1821 IOMMU_DOMAIN_LOCK(iodom);
1822 LIST_INSERT_HEAD(&domain->ctx_list, ctx, next);
1823 IOMMU_DOMAIN_UNLOCK(iodom);
1824
1825 return (&ctx->ioctx);
1826 }
1827
1828 static int
smmu_ctx_init(device_t dev,struct iommu_ctx * ioctx)1829 smmu_ctx_init(device_t dev, struct iommu_ctx *ioctx)
1830 {
1831 struct smmu_domain *domain;
1832 struct iommu_domain *iodom;
1833 struct smmu_softc *sc;
1834 struct smmu_ctx *ctx;
1835 devclass_t pci_class;
1836 u_int sid;
1837 int err;
1838
1839 ctx = (struct smmu_ctx *)ioctx;
1840
1841 sc = device_get_softc(dev);
1842
1843 domain = ctx->domain;
1844 iodom = (struct iommu_domain *)domain;
1845
1846 pci_class = devclass_find("pci");
1847 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) {
1848 err = smmu_pci_get_sid(ctx->dev, NULL, &sid);
1849 if (err)
1850 return (err);
1851
1852 ioctx->rid = pci_get_rid(dev);
1853 ctx->sid = sid;
1854 ctx->vendor = pci_get_vendor(ctx->dev);
1855 ctx->device = pci_get_device(ctx->dev);
1856 }
1857
1858 if (sc->features & SMMU_FEATURE_2_LVL_STREAM_TABLE) {
1859 err = smmu_init_l1_entry(sc, ctx->sid);
1860 if (err)
1861 return (err);
1862 }
1863
1864 /*
1865 * Neoverse N1 SDP:
1866 * 0x800 xhci
1867 * 0x700 re
1868 * 0x600 sata
1869 */
1870
1871 smmu_init_ste(sc, domain->cd, ctx->sid, ctx->bypass);
1872
1873 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class)
1874 if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev)))
1875 smmu_set_buswide(dev, domain, ctx);
1876
1877 return (0);
1878 }
1879
1880 static void
smmu_ctx_free(device_t dev,struct iommu_ctx * ioctx)1881 smmu_ctx_free(device_t dev, struct iommu_ctx *ioctx)
1882 {
1883 struct smmu_softc *sc;
1884 struct smmu_ctx *ctx;
1885
1886 IOMMU_ASSERT_LOCKED(ioctx->domain->iommu);
1887
1888 sc = device_get_softc(dev);
1889 ctx = (struct smmu_ctx *)ioctx;
1890
1891 smmu_deinit_ste(sc, ctx->sid);
1892
1893 LIST_REMOVE(ctx, next);
1894
1895 free(ctx, M_SMMU);
1896 }
1897
1898 struct smmu_ctx *
smmu_ctx_lookup_by_sid(device_t dev,u_int sid)1899 smmu_ctx_lookup_by_sid(device_t dev, u_int sid)
1900 {
1901 struct smmu_softc *sc;
1902 struct smmu_domain *domain;
1903 struct smmu_unit *unit;
1904 struct smmu_ctx *ctx;
1905
1906 sc = device_get_softc(dev);
1907
1908 unit = &sc->unit;
1909
1910 LIST_FOREACH(domain, &unit->domain_list, next) {
1911 LIST_FOREACH(ctx, &domain->ctx_list, next) {
1912 if (ctx->sid == sid)
1913 return (ctx);
1914 }
1915 }
1916
1917 return (NULL);
1918 }
1919
1920 static struct iommu_ctx *
smmu_ctx_lookup(device_t dev,device_t child)1921 smmu_ctx_lookup(device_t dev, device_t child)
1922 {
1923 struct iommu_unit *iommu __diagused;
1924 struct smmu_softc *sc;
1925 struct smmu_domain *domain;
1926 struct smmu_unit *unit;
1927 struct smmu_ctx *ctx;
1928
1929 sc = device_get_softc(dev);
1930
1931 unit = &sc->unit;
1932 iommu = &unit->iommu;
1933
1934 IOMMU_ASSERT_LOCKED(iommu);
1935
1936 LIST_FOREACH(domain, &unit->domain_list, next) {
1937 IOMMU_DOMAIN_LOCK(&domain->iodom);
1938 LIST_FOREACH(ctx, &domain->ctx_list, next) {
1939 if (ctx->dev == child) {
1940 IOMMU_DOMAIN_UNLOCK(&domain->iodom);
1941 return (&ctx->ioctx);
1942 }
1943 }
1944 IOMMU_DOMAIN_UNLOCK(&domain->iodom);
1945 }
1946
1947 return (NULL);
1948 }
1949
1950 static int
smmu_find(device_t dev,device_t child)1951 smmu_find(device_t dev, device_t child)
1952 {
1953 struct smmu_softc *sc;
1954 u_int xref;
1955 int err;
1956
1957 sc = device_get_softc(dev);
1958
1959 err = smmu_pci_get_sid(child, &xref, NULL);
1960 if (err)
1961 return (ENOENT);
1962
1963 /* Check if xref is ours. */
1964 if (xref != sc->xref)
1965 return (EFAULT);
1966
1967 return (0);
1968 }
1969
1970 #ifdef FDT
1971 static int
smmu_ofw_md_data(device_t dev,struct iommu_ctx * ioctx,pcell_t * cells,int ncells)1972 smmu_ofw_md_data(device_t dev, struct iommu_ctx *ioctx, pcell_t *cells,
1973 int ncells)
1974 {
1975 struct smmu_ctx *ctx;
1976
1977 ctx = (struct smmu_ctx *)ioctx;
1978
1979 if (ncells != 1)
1980 return (-1);
1981
1982 ctx->sid = cells[0];
1983
1984 return (0);
1985 }
1986 #endif
1987
1988 static device_method_t smmu_methods[] = {
1989 /* Device interface */
1990 DEVMETHOD(device_detach, smmu_detach),
1991
1992 /* SMMU interface */
1993 DEVMETHOD(iommu_find, smmu_find),
1994 DEVMETHOD(iommu_map, smmu_map),
1995 DEVMETHOD(iommu_unmap, smmu_unmap),
1996 DEVMETHOD(iommu_domain_alloc, smmu_domain_alloc),
1997 DEVMETHOD(iommu_domain_free, smmu_domain_free),
1998 DEVMETHOD(iommu_ctx_alloc, smmu_ctx_alloc),
1999 DEVMETHOD(iommu_ctx_init, smmu_ctx_init),
2000 DEVMETHOD(iommu_ctx_free, smmu_ctx_free),
2001 DEVMETHOD(iommu_ctx_lookup, smmu_ctx_lookup),
2002 #ifdef FDT
2003 DEVMETHOD(iommu_ofw_md_data, smmu_ofw_md_data),
2004 #endif
2005
2006 /* Bus interface */
2007 DEVMETHOD(bus_read_ivar, smmu_read_ivar),
2008
2009 /* End */
2010 DEVMETHOD_END
2011 };
2012
2013 DEFINE_CLASS_0(smmu, smmu_driver, smmu_methods, sizeof(struct smmu_softc));
2014