14cc87010SRuslan Bukin /*-
24cc87010SRuslan Bukin * SPDX-License-Identifier: BSD-2-Clause
34cc87010SRuslan Bukin *
44cc87010SRuslan Bukin * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
54cc87010SRuslan Bukin *
64cc87010SRuslan Bukin * This software was developed by SRI International and the University of
74cc87010SRuslan Bukin * Cambridge Computer Laboratory (Department of Computer Science and
84cc87010SRuslan Bukin * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
94cc87010SRuslan Bukin * DARPA SSITH research programme.
104cc87010SRuslan Bukin *
114cc87010SRuslan Bukin * Portions of this work was supported by Innovate UK project 105694,
124cc87010SRuslan Bukin * "Digital Security by Design (DSbD) Technology Platform Prototype".
134cc87010SRuslan Bukin *
144cc87010SRuslan Bukin * Redistribution and use in source and binary forms, with or without
154cc87010SRuslan Bukin * modification, are permitted provided that the following conditions
164cc87010SRuslan Bukin * are met:
174cc87010SRuslan Bukin * 1. Redistributions of source code must retain the above copyright
184cc87010SRuslan Bukin * notice, this list of conditions and the following disclaimer.
194cc87010SRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright
204cc87010SRuslan Bukin * notice, this list of conditions and the following disclaimer in the
214cc87010SRuslan Bukin * documentation and/or other materials provided with the distribution.
224cc87010SRuslan Bukin *
234cc87010SRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
244cc87010SRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
254cc87010SRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
264cc87010SRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
274cc87010SRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
284cc87010SRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
294cc87010SRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
304cc87010SRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
314cc87010SRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
324cc87010SRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
334cc87010SRuslan Bukin * SUCH DAMAGE.
344cc87010SRuslan Bukin */
354cc87010SRuslan Bukin
364cc87010SRuslan Bukin #include "opt_platform.h"
374cc87010SRuslan Bukin
384cc87010SRuslan Bukin #include <sys/param.h>
394cc87010SRuslan Bukin #include <sys/bus.h>
404cc87010SRuslan Bukin #include <sys/kernel.h>
414cc87010SRuslan Bukin #include <sys/malloc.h>
424cc87010SRuslan Bukin #include <sys/memdesc.h>
434cc87010SRuslan Bukin #include <sys/tree.h>
444cc87010SRuslan Bukin #include <sys/taskqueue.h>
454cc87010SRuslan Bukin #include <sys/lock.h>
464cc87010SRuslan Bukin #include <sys/mutex.h>
477d0bbf43SRuslan Bukin #include <sys/sx.h>
484cc87010SRuslan Bukin #include <sys/sysctl.h>
494cc87010SRuslan Bukin #include <vm/vm.h>
504cc87010SRuslan Bukin
514cc87010SRuslan Bukin #include <dev/pci/pcireg.h>
524cc87010SRuslan Bukin #include <dev/pci/pcivar.h>
534cc87010SRuslan Bukin #include <machine/bus.h>
544cc87010SRuslan Bukin #include <dev/iommu/busdma_iommu.h>
554cc87010SRuslan Bukin #include <machine/vmparam.h>
564cc87010SRuslan Bukin
5741ce5498SRuslan Bukin #ifdef FDT
5841ce5498SRuslan Bukin #include <dev/fdt/fdt_common.h>
5941ce5498SRuslan Bukin #include <dev/ofw/ofw_bus.h>
6041ce5498SRuslan Bukin #include <dev/ofw/ofw_bus_subr.h>
6141ce5498SRuslan Bukin #endif
6241ce5498SRuslan Bukin
634cc87010SRuslan Bukin #include "iommu.h"
644cc87010SRuslan Bukin #include "iommu_if.h"
654cc87010SRuslan Bukin
664cc87010SRuslan Bukin static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework");
674cc87010SRuslan Bukin
687d0bbf43SRuslan Bukin #define IOMMU_LIST_LOCK() sx_xlock(&iommu_sx)
697d0bbf43SRuslan Bukin #define IOMMU_LIST_UNLOCK() sx_xunlock(&iommu_sx)
707d0bbf43SRuslan Bukin #define IOMMU_LIST_ASSERT_LOCKED() sx_assert(&iommu_sx, SA_XLOCKED)
714cc87010SRuslan Bukin
724cc87010SRuslan Bukin #define dprintf(fmt, ...)
734cc87010SRuslan Bukin
747d0bbf43SRuslan Bukin static struct sx iommu_sx;
754cc87010SRuslan Bukin
764cc87010SRuslan Bukin struct iommu_entry {
774cc87010SRuslan Bukin struct iommu_unit *iommu;
784cc87010SRuslan Bukin LIST_ENTRY(iommu_entry) next;
794cc87010SRuslan Bukin };
804cc87010SRuslan Bukin static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list);
814cc87010SRuslan Bukin
824cc87010SRuslan Bukin static int
iommu_domain_unmap_buf(struct iommu_domain * iodom,iommu_gaddr_t base,iommu_gaddr_t size,int flags)834cc87010SRuslan Bukin iommu_domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
844cc87010SRuslan Bukin iommu_gaddr_t size, int flags)
854cc87010SRuslan Bukin {
864cc87010SRuslan Bukin struct iommu_unit *iommu;
874cc87010SRuslan Bukin int error;
884cc87010SRuslan Bukin
894cc87010SRuslan Bukin iommu = iodom->iommu;
904cc87010SRuslan Bukin
914cc87010SRuslan Bukin error = IOMMU_UNMAP(iommu->dev, iodom, base, size);
924cc87010SRuslan Bukin
934cc87010SRuslan Bukin return (error);
944cc87010SRuslan Bukin }
954cc87010SRuslan Bukin
964cc87010SRuslan Bukin static int
iommu_domain_map_buf(struct iommu_domain * iodom,iommu_gaddr_t base,iommu_gaddr_t size,vm_page_t * ma,uint64_t eflags,int flags)974cc87010SRuslan Bukin iommu_domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
984cc87010SRuslan Bukin iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags)
994cc87010SRuslan Bukin {
1004cc87010SRuslan Bukin struct iommu_unit *iommu;
1014cc87010SRuslan Bukin vm_prot_t prot;
1024cc87010SRuslan Bukin vm_offset_t va;
1034cc87010SRuslan Bukin int error;
1044cc87010SRuslan Bukin
1054cc87010SRuslan Bukin dprintf("%s: base %lx, size %lx\n", __func__, base, size);
1064cc87010SRuslan Bukin
1074cc87010SRuslan Bukin prot = 0;
1084cc87010SRuslan Bukin if (eflags & IOMMU_MAP_ENTRY_READ)
1094cc87010SRuslan Bukin prot |= VM_PROT_READ;
1104cc87010SRuslan Bukin if (eflags & IOMMU_MAP_ENTRY_WRITE)
1114cc87010SRuslan Bukin prot |= VM_PROT_WRITE;
1124cc87010SRuslan Bukin
1134cc87010SRuslan Bukin va = base;
1144cc87010SRuslan Bukin
1154cc87010SRuslan Bukin iommu = iodom->iommu;
1164cc87010SRuslan Bukin
1174cc87010SRuslan Bukin error = IOMMU_MAP(iommu->dev, iodom, va, ma, size, prot);
1184cc87010SRuslan Bukin
1196ff00427SRuslan Bukin return (error);
1204cc87010SRuslan Bukin }
1214cc87010SRuslan Bukin
1224cc87010SRuslan Bukin static const struct iommu_domain_map_ops domain_map_ops = {
1234cc87010SRuslan Bukin .map = iommu_domain_map_buf,
1244cc87010SRuslan Bukin .unmap = iommu_domain_unmap_buf,
1254cc87010SRuslan Bukin };
1264cc87010SRuslan Bukin
1274cc87010SRuslan Bukin static struct iommu_domain *
iommu_domain_alloc(struct iommu_unit * iommu)1284cc87010SRuslan Bukin iommu_domain_alloc(struct iommu_unit *iommu)
1294cc87010SRuslan Bukin {
1304cc87010SRuslan Bukin struct iommu_domain *iodom;
1314cc87010SRuslan Bukin
1324cc87010SRuslan Bukin iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu);
1334cc87010SRuslan Bukin if (iodom == NULL)
1344cc87010SRuslan Bukin return (NULL);
1354cc87010SRuslan Bukin
1364b4e8cb5SRuslan Bukin KASSERT(iodom->end != 0, ("domain end is not set"));
1374b4e8cb5SRuslan Bukin
1384cc87010SRuslan Bukin iommu_domain_init(iommu, iodom, &domain_map_ops);
1394cc87010SRuslan Bukin iodom->iommu = iommu;
1404cc87010SRuslan Bukin iommu_gas_init_domain(iodom);
1414cc87010SRuslan Bukin
1424cc87010SRuslan Bukin return (iodom);
1434cc87010SRuslan Bukin }
1444cc87010SRuslan Bukin
1454cc87010SRuslan Bukin static int
iommu_domain_free(struct iommu_domain * iodom)1464cc87010SRuslan Bukin iommu_domain_free(struct iommu_domain *iodom)
1474cc87010SRuslan Bukin {
1484cc87010SRuslan Bukin struct iommu_unit *iommu;
1494cc87010SRuslan Bukin
1504cc87010SRuslan Bukin iommu = iodom->iommu;
1514cc87010SRuslan Bukin
1524cc87010SRuslan Bukin IOMMU_LOCK(iommu);
1534cc87010SRuslan Bukin
1544cc87010SRuslan Bukin if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
1554cc87010SRuslan Bukin IOMMU_DOMAIN_LOCK(iodom);
1564cc87010SRuslan Bukin iommu_gas_fini_domain(iodom);
1574cc87010SRuslan Bukin IOMMU_DOMAIN_UNLOCK(iodom);
1584cc87010SRuslan Bukin }
1594cc87010SRuslan Bukin
1604cc87010SRuslan Bukin iommu_domain_fini(iodom);
1614cc87010SRuslan Bukin
1624cc87010SRuslan Bukin IOMMU_DOMAIN_FREE(iommu->dev, iodom);
1634cc87010SRuslan Bukin IOMMU_UNLOCK(iommu);
1644cc87010SRuslan Bukin
1654cc87010SRuslan Bukin return (0);
1664cc87010SRuslan Bukin }
1674cc87010SRuslan Bukin
1684cc87010SRuslan Bukin static void
iommu_tag_init(struct iommu_domain * iodom,struct bus_dma_tag_iommu * t)1694b4e8cb5SRuslan Bukin iommu_tag_init(struct iommu_domain *iodom, struct bus_dma_tag_iommu *t)
1704cc87010SRuslan Bukin {
1714cc87010SRuslan Bukin bus_addr_t maxaddr;
1724cc87010SRuslan Bukin
1734b4e8cb5SRuslan Bukin maxaddr = MIN(iodom->end, BUS_SPACE_MAXADDR);
1744cc87010SRuslan Bukin
1754cc87010SRuslan Bukin t->common.ref_count = 0;
1764cc87010SRuslan Bukin t->common.impl = &bus_dma_iommu_impl;
1774cc87010SRuslan Bukin t->common.alignment = 1;
1784cc87010SRuslan Bukin t->common.boundary = 0;
1794cc87010SRuslan Bukin t->common.lowaddr = maxaddr;
1804cc87010SRuslan Bukin t->common.highaddr = maxaddr;
1814cc87010SRuslan Bukin t->common.maxsize = maxaddr;
1824cc87010SRuslan Bukin t->common.nsegments = BUS_SPACE_UNRESTRICTED;
1834cc87010SRuslan Bukin t->common.maxsegsz = maxaddr;
1844cc87010SRuslan Bukin }
1854cc87010SRuslan Bukin
1864cc87010SRuslan Bukin static struct iommu_ctx *
iommu_ctx_alloc(device_t requester,struct iommu_domain * iodom,bool disabled)18741ce5498SRuslan Bukin iommu_ctx_alloc(device_t requester, struct iommu_domain *iodom, bool disabled)
1884cc87010SRuslan Bukin {
1894cc87010SRuslan Bukin struct iommu_unit *iommu;
1904cc87010SRuslan Bukin struct iommu_ctx *ioctx;
1914cc87010SRuslan Bukin
1924cc87010SRuslan Bukin iommu = iodom->iommu;
1934cc87010SRuslan Bukin
19441ce5498SRuslan Bukin ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, requester, disabled);
1954cc87010SRuslan Bukin if (ioctx == NULL)
1964cc87010SRuslan Bukin return (NULL);
1974cc87010SRuslan Bukin
19841ce5498SRuslan Bukin ioctx->domain = iodom;
19941ce5498SRuslan Bukin
20041ce5498SRuslan Bukin return (ioctx);
20141ce5498SRuslan Bukin }
20241ce5498SRuslan Bukin
20341ce5498SRuslan Bukin static int
iommu_ctx_init(device_t requester,struct iommu_ctx * ioctx)20441ce5498SRuslan Bukin iommu_ctx_init(device_t requester, struct iommu_ctx *ioctx)
20541ce5498SRuslan Bukin {
20641ce5498SRuslan Bukin struct bus_dma_tag_iommu *tag;
20741ce5498SRuslan Bukin struct iommu_domain *iodom;
20841ce5498SRuslan Bukin struct iommu_unit *iommu;
20941ce5498SRuslan Bukin int error;
21041ce5498SRuslan Bukin
21141ce5498SRuslan Bukin iodom = ioctx->domain;
21241ce5498SRuslan Bukin iommu = iodom->iommu;
21341ce5498SRuslan Bukin
21441ce5498SRuslan Bukin error = IOMMU_CTX_INIT(iommu->dev, ioctx);
21541ce5498SRuslan Bukin if (error)
21641ce5498SRuslan Bukin return (error);
21741ce5498SRuslan Bukin
21841ce5498SRuslan Bukin tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu),
21941ce5498SRuslan Bukin M_IOMMU, M_WAITOK | M_ZERO);
22041ce5498SRuslan Bukin tag->owner = requester;
22141ce5498SRuslan Bukin tag->ctx = ioctx;
22241ce5498SRuslan Bukin tag->ctx->domain = iodom;
22341ce5498SRuslan Bukin
2244b4e8cb5SRuslan Bukin iommu_tag_init(iodom, tag);
22541ce5498SRuslan Bukin
22641ce5498SRuslan Bukin return (error);
22741ce5498SRuslan Bukin }
22841ce5498SRuslan Bukin
22941ce5498SRuslan Bukin static struct iommu_unit *
iommu_lookup(device_t dev)23041ce5498SRuslan Bukin iommu_lookup(device_t dev)
23141ce5498SRuslan Bukin {
23241ce5498SRuslan Bukin struct iommu_entry *entry;
23341ce5498SRuslan Bukin struct iommu_unit *iommu;
23441ce5498SRuslan Bukin
23541ce5498SRuslan Bukin IOMMU_LIST_LOCK();
23641ce5498SRuslan Bukin LIST_FOREACH(entry, &iommu_list, next) {
23741ce5498SRuslan Bukin iommu = entry->iommu;
23841ce5498SRuslan Bukin if (iommu->dev == dev) {
23941ce5498SRuslan Bukin IOMMU_LIST_UNLOCK();
24041ce5498SRuslan Bukin return (iommu);
24141ce5498SRuslan Bukin }
24241ce5498SRuslan Bukin }
24341ce5498SRuslan Bukin IOMMU_LIST_UNLOCK();
24441ce5498SRuslan Bukin
24541ce5498SRuslan Bukin return (NULL);
24641ce5498SRuslan Bukin }
24741ce5498SRuslan Bukin
248dc08d52dSRuslan Bukin #ifdef FDT
24941ce5498SRuslan Bukin struct iommu_ctx *
iommu_get_ctx_ofw(device_t dev,int channel)25041ce5498SRuslan Bukin iommu_get_ctx_ofw(device_t dev, int channel)
25141ce5498SRuslan Bukin {
25241ce5498SRuslan Bukin struct iommu_domain *iodom;
25341ce5498SRuslan Bukin struct iommu_unit *iommu;
25441ce5498SRuslan Bukin struct iommu_ctx *ioctx;
25541ce5498SRuslan Bukin phandle_t node, parent;
25641ce5498SRuslan Bukin device_t iommu_dev;
25741ce5498SRuslan Bukin pcell_t *cells;
25841ce5498SRuslan Bukin int niommus;
25941ce5498SRuslan Bukin int ncells;
26041ce5498SRuslan Bukin int error;
26141ce5498SRuslan Bukin
26241ce5498SRuslan Bukin node = ofw_bus_get_node(dev);
26341ce5498SRuslan Bukin if (node <= 0) {
26441ce5498SRuslan Bukin device_printf(dev,
26541ce5498SRuslan Bukin "%s called on not ofw based device.\n", __func__);
26641ce5498SRuslan Bukin return (NULL);
26741ce5498SRuslan Bukin }
26841ce5498SRuslan Bukin
26941ce5498SRuslan Bukin error = ofw_bus_parse_xref_list_get_length(node,
27041ce5498SRuslan Bukin "iommus", "#iommu-cells", &niommus);
27141ce5498SRuslan Bukin if (error) {
27241ce5498SRuslan Bukin device_printf(dev, "%s can't get iommu list.\n", __func__);
27341ce5498SRuslan Bukin return (NULL);
27441ce5498SRuslan Bukin }
27541ce5498SRuslan Bukin
27641ce5498SRuslan Bukin if (niommus == 0) {
27741ce5498SRuslan Bukin device_printf(dev, "%s iommu list is empty.\n", __func__);
27841ce5498SRuslan Bukin return (NULL);
27941ce5498SRuslan Bukin }
28041ce5498SRuslan Bukin
28141ce5498SRuslan Bukin error = ofw_bus_parse_xref_list_alloc(node, "iommus", "#iommu-cells",
28241ce5498SRuslan Bukin channel, &parent, &ncells, &cells);
28341ce5498SRuslan Bukin if (error != 0) {
28441ce5498SRuslan Bukin device_printf(dev, "%s can't get iommu device xref.\n",
28541ce5498SRuslan Bukin __func__);
28641ce5498SRuslan Bukin return (NULL);
28741ce5498SRuslan Bukin }
28841ce5498SRuslan Bukin
28941ce5498SRuslan Bukin iommu_dev = OF_device_from_xref(parent);
29041ce5498SRuslan Bukin if (iommu_dev == NULL) {
29141ce5498SRuslan Bukin device_printf(dev, "%s can't get iommu device.\n", __func__);
29241ce5498SRuslan Bukin return (NULL);
29341ce5498SRuslan Bukin }
29441ce5498SRuslan Bukin
29541ce5498SRuslan Bukin iommu = iommu_lookup(iommu_dev);
29641ce5498SRuslan Bukin if (iommu == NULL) {
29741ce5498SRuslan Bukin device_printf(dev, "%s can't lookup iommu.\n", __func__);
29841ce5498SRuslan Bukin return (NULL);
29941ce5498SRuslan Bukin }
30041ce5498SRuslan Bukin
3014cc87010SRuslan Bukin /*
30241ce5498SRuslan Bukin * In our current configuration we have a domain per each ctx,
30341ce5498SRuslan Bukin * so allocate a domain first.
3044cc87010SRuslan Bukin */
30541ce5498SRuslan Bukin iodom = iommu_domain_alloc(iommu);
30641ce5498SRuslan Bukin if (iodom == NULL) {
30741ce5498SRuslan Bukin device_printf(dev, "%s can't allocate domain.\n", __func__);
30841ce5498SRuslan Bukin return (NULL);
30941ce5498SRuslan Bukin }
31041ce5498SRuslan Bukin
31141ce5498SRuslan Bukin ioctx = iommu_ctx_alloc(dev, iodom, false);
31241ce5498SRuslan Bukin if (ioctx == NULL) {
31341ce5498SRuslan Bukin iommu_domain_free(iodom);
31441ce5498SRuslan Bukin return (NULL);
31541ce5498SRuslan Bukin }
31641ce5498SRuslan Bukin
31741ce5498SRuslan Bukin ioctx->domain = iodom;
31841ce5498SRuslan Bukin
31941ce5498SRuslan Bukin error = IOMMU_OFW_MD_DATA(iommu->dev, ioctx, cells, ncells);
32041ce5498SRuslan Bukin if (error) {
32141ce5498SRuslan Bukin device_printf(dev, "%s can't set MD data\n", __func__);
32241ce5498SRuslan Bukin return (NULL);
32341ce5498SRuslan Bukin }
32441ce5498SRuslan Bukin
32541ce5498SRuslan Bukin error = iommu_ctx_init(dev, ioctx);
32641ce5498SRuslan Bukin if (error) {
32741ce5498SRuslan Bukin IOMMU_CTX_FREE(iommu->dev, ioctx);
32841ce5498SRuslan Bukin iommu_domain_free(iodom);
32941ce5498SRuslan Bukin return (NULL);
33041ce5498SRuslan Bukin }
3314cc87010SRuslan Bukin
3324cc87010SRuslan Bukin return (ioctx);
3334cc87010SRuslan Bukin }
334dc08d52dSRuslan Bukin #endif
3354cc87010SRuslan Bukin
3364cc87010SRuslan Bukin struct iommu_ctx *
iommu_get_ctx(struct iommu_unit * iommu,device_t requester,uint16_t rid,bool disabled,bool rmrr)3374cc87010SRuslan Bukin iommu_get_ctx(struct iommu_unit *iommu, device_t requester,
3384cc87010SRuslan Bukin uint16_t rid, bool disabled, bool rmrr)
3394cc87010SRuslan Bukin {
3404cc87010SRuslan Bukin struct iommu_domain *iodom;
34141ce5498SRuslan Bukin struct iommu_ctx *ioctx;
34241ce5498SRuslan Bukin int error;
3434cc87010SRuslan Bukin
3444cc87010SRuslan Bukin IOMMU_LOCK(iommu);
3454cc87010SRuslan Bukin ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester);
3464cc87010SRuslan Bukin if (ioctx) {
3474cc87010SRuslan Bukin IOMMU_UNLOCK(iommu);
3484cc87010SRuslan Bukin return (ioctx);
3494cc87010SRuslan Bukin }
3504cc87010SRuslan Bukin IOMMU_UNLOCK(iommu);
3514cc87010SRuslan Bukin
3524cc87010SRuslan Bukin /*
3534cc87010SRuslan Bukin * In our current configuration we have a domain per each ctx.
3544cc87010SRuslan Bukin * So allocate a domain first.
3554cc87010SRuslan Bukin */
3564cc87010SRuslan Bukin iodom = iommu_domain_alloc(iommu);
3574cc87010SRuslan Bukin if (iodom == NULL)
3584cc87010SRuslan Bukin return (NULL);
3594cc87010SRuslan Bukin
3604cc87010SRuslan Bukin ioctx = iommu_ctx_alloc(requester, iodom, disabled);
3614cc87010SRuslan Bukin if (ioctx == NULL) {
3624cc87010SRuslan Bukin iommu_domain_free(iodom);
3634cc87010SRuslan Bukin return (NULL);
3644cc87010SRuslan Bukin }
3654cc87010SRuslan Bukin
36641ce5498SRuslan Bukin error = iommu_ctx_init(requester, ioctx);
36741ce5498SRuslan Bukin if (error) {
36841ce5498SRuslan Bukin IOMMU_CTX_FREE(iommu->dev, ioctx);
36941ce5498SRuslan Bukin iommu_domain_free(iodom);
37041ce5498SRuslan Bukin return (NULL);
37141ce5498SRuslan Bukin }
3724cc87010SRuslan Bukin
3734cc87010SRuslan Bukin return (ioctx);
3744cc87010SRuslan Bukin }
3754cc87010SRuslan Bukin
3764cc87010SRuslan Bukin void
iommu_free_ctx_locked(struct iommu_unit * iommu,struct iommu_ctx * ioctx)3774cc87010SRuslan Bukin iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx)
3784cc87010SRuslan Bukin {
3794cc87010SRuslan Bukin struct bus_dma_tag_iommu *tag;
3804cc87010SRuslan Bukin
3814cc87010SRuslan Bukin IOMMU_ASSERT_LOCKED(iommu);
3824cc87010SRuslan Bukin
3834cc87010SRuslan Bukin tag = ioctx->tag;
3844cc87010SRuslan Bukin
3854cc87010SRuslan Bukin IOMMU_CTX_FREE(iommu->dev, ioctx);
3864cc87010SRuslan Bukin
3874cc87010SRuslan Bukin free(tag, M_IOMMU);
3884cc87010SRuslan Bukin }
3894cc87010SRuslan Bukin
3904cc87010SRuslan Bukin void
iommu_free_ctx(struct iommu_ctx * ioctx)3914cc87010SRuslan Bukin iommu_free_ctx(struct iommu_ctx *ioctx)
3924cc87010SRuslan Bukin {
3934cc87010SRuslan Bukin struct iommu_unit *iommu;
3944cc87010SRuslan Bukin struct iommu_domain *iodom;
3954cc87010SRuslan Bukin int error;
3964cc87010SRuslan Bukin
3974cc87010SRuslan Bukin iodom = ioctx->domain;
3984cc87010SRuslan Bukin iommu = iodom->iommu;
3994cc87010SRuslan Bukin
4004cc87010SRuslan Bukin IOMMU_LOCK(iommu);
4014cc87010SRuslan Bukin iommu_free_ctx_locked(iommu, ioctx);
4024cc87010SRuslan Bukin IOMMU_UNLOCK(iommu);
4034cc87010SRuslan Bukin
4044cc87010SRuslan Bukin /* Since we have a domain per each ctx, remove the domain too. */
4054cc87010SRuslan Bukin error = iommu_domain_free(iodom);
4064cc87010SRuslan Bukin if (error)
4074cc87010SRuslan Bukin device_printf(iommu->dev, "Could not free a domain\n");
4084cc87010SRuslan Bukin }
4094cc87010SRuslan Bukin
4104cc87010SRuslan Bukin static void
iommu_domain_free_entry(struct iommu_map_entry * entry,bool free)4114cc87010SRuslan Bukin iommu_domain_free_entry(struct iommu_map_entry *entry, bool free)
4124cc87010SRuslan Bukin {
4134670f908SAlan Cox iommu_gas_free_space(entry);
4144cc87010SRuslan Bukin
4154cc87010SRuslan Bukin if (free)
4164670f908SAlan Cox iommu_gas_free_entry(entry);
4174cc87010SRuslan Bukin else
4184cc87010SRuslan Bukin entry->flags = 0;
4194cc87010SRuslan Bukin }
4204cc87010SRuslan Bukin
4214cc87010SRuslan Bukin void
iommu_domain_unload(struct iommu_domain * iodom,struct iommu_map_entries_tailq * entries,bool cansleep)4224cc87010SRuslan Bukin iommu_domain_unload(struct iommu_domain *iodom,
4234cc87010SRuslan Bukin struct iommu_map_entries_tailq *entries, bool cansleep)
4244cc87010SRuslan Bukin {
4254cc87010SRuslan Bukin struct iommu_map_entry *entry, *entry1;
4260a8e88faSRuslan Bukin int error __diagused;
4274cc87010SRuslan Bukin
4284cc87010SRuslan Bukin TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
4294cc87010SRuslan Bukin KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
4304cc87010SRuslan Bukin ("not mapped entry %p %p", iodom, entry));
4314cc87010SRuslan Bukin error = iodom->ops->unmap(iodom, entry->start, entry->end -
4324cc87010SRuslan Bukin entry->start, cansleep ? IOMMU_PGF_WAITOK : 0);
4334cc87010SRuslan Bukin KASSERT(error == 0, ("unmap %p error %d", iodom, error));
4344cc87010SRuslan Bukin TAILQ_REMOVE(entries, entry, dmamap_link);
4354cc87010SRuslan Bukin iommu_domain_free_entry(entry, true);
4364cc87010SRuslan Bukin }
4374cc87010SRuslan Bukin
4384cc87010SRuslan Bukin if (TAILQ_EMPTY(entries))
4394cc87010SRuslan Bukin return;
4404cc87010SRuslan Bukin
4414cc87010SRuslan Bukin panic("entries map is not empty");
4424cc87010SRuslan Bukin }
4434cc87010SRuslan Bukin
4444cc87010SRuslan Bukin int
iommu_register(struct iommu_unit * iommu)4454cc87010SRuslan Bukin iommu_register(struct iommu_unit *iommu)
4464cc87010SRuslan Bukin {
4474cc87010SRuslan Bukin struct iommu_entry *entry;
4484cc87010SRuslan Bukin
4494cc87010SRuslan Bukin mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF);
4504cc87010SRuslan Bukin
4514cc87010SRuslan Bukin entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO);
4524cc87010SRuslan Bukin entry->iommu = iommu;
4534cc87010SRuslan Bukin
4544cc87010SRuslan Bukin IOMMU_LIST_LOCK();
4554cc87010SRuslan Bukin LIST_INSERT_HEAD(&iommu_list, entry, next);
4564cc87010SRuslan Bukin IOMMU_LIST_UNLOCK();
4574cc87010SRuslan Bukin
4584cc87010SRuslan Bukin iommu_init_busdma(iommu);
4594cc87010SRuslan Bukin
4604cc87010SRuslan Bukin return (0);
4614cc87010SRuslan Bukin }
4624cc87010SRuslan Bukin
4634cc87010SRuslan Bukin int
iommu_unregister(struct iommu_unit * iommu)4644cc87010SRuslan Bukin iommu_unregister(struct iommu_unit *iommu)
4654cc87010SRuslan Bukin {
4664cc87010SRuslan Bukin struct iommu_entry *entry, *tmp;
4674cc87010SRuslan Bukin
4684cc87010SRuslan Bukin IOMMU_LIST_LOCK();
4694cc87010SRuslan Bukin LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) {
4704cc87010SRuslan Bukin if (entry->iommu == iommu) {
4714cc87010SRuslan Bukin LIST_REMOVE(entry, next);
4724cc87010SRuslan Bukin free(entry, M_IOMMU);
4734cc87010SRuslan Bukin }
4744cc87010SRuslan Bukin }
4754cc87010SRuslan Bukin IOMMU_LIST_UNLOCK();
4764cc87010SRuslan Bukin
4774cc87010SRuslan Bukin iommu_fini_busdma(iommu);
4784cc87010SRuslan Bukin
4794cc87010SRuslan Bukin mtx_destroy(&iommu->lock);
4804cc87010SRuslan Bukin
4814cc87010SRuslan Bukin return (0);
4824cc87010SRuslan Bukin }
4834cc87010SRuslan Bukin
4844cc87010SRuslan Bukin struct iommu_unit *
iommu_find(device_t dev,bool verbose)4854cc87010SRuslan Bukin iommu_find(device_t dev, bool verbose)
4864cc87010SRuslan Bukin {
4874cc87010SRuslan Bukin struct iommu_entry *entry;
4884cc87010SRuslan Bukin struct iommu_unit *iommu;
4894cc87010SRuslan Bukin int error;
4904cc87010SRuslan Bukin
4914cc87010SRuslan Bukin IOMMU_LIST_LOCK();
4924cc87010SRuslan Bukin LIST_FOREACH(entry, &iommu_list, next) {
4934cc87010SRuslan Bukin iommu = entry->iommu;
4944cc87010SRuslan Bukin error = IOMMU_FIND(iommu->dev, dev);
4954cc87010SRuslan Bukin if (error == 0) {
4964cc87010SRuslan Bukin IOMMU_LIST_UNLOCK();
4974cc87010SRuslan Bukin return (entry->iommu);
4984cc87010SRuslan Bukin }
4994cc87010SRuslan Bukin }
5004cc87010SRuslan Bukin IOMMU_LIST_UNLOCK();
5014cc87010SRuslan Bukin
5024cc87010SRuslan Bukin return (NULL);
5034cc87010SRuslan Bukin }
5044cc87010SRuslan Bukin
5054cc87010SRuslan Bukin void
iommu_domain_unload_entry(struct iommu_map_entry * entry,bool free,bool cansleep __unused)5068bc36738SAlan Cox iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free,
5078bc36738SAlan Cox bool cansleep __unused)
5084cc87010SRuslan Bukin {
5094cc87010SRuslan Bukin
5104cc87010SRuslan Bukin dprintf("%s\n", __func__);
5114cc87010SRuslan Bukin
5124cc87010SRuslan Bukin iommu_domain_free_entry(entry, free);
5134cc87010SRuslan Bukin }
5144cc87010SRuslan Bukin
5154cc87010SRuslan Bukin static void
iommu_init(void)5164cc87010SRuslan Bukin iommu_init(void)
5174cc87010SRuslan Bukin {
5184cc87010SRuslan Bukin
5197d0bbf43SRuslan Bukin sx_init(&iommu_sx, "IOMMU list");
5204cc87010SRuslan Bukin }
5214cc87010SRuslan Bukin
5224cc87010SRuslan Bukin SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL);
523