14cc87010SRuslan Bukin /*- 24cc87010SRuslan Bukin * SPDX-License-Identifier: BSD-2-Clause 34cc87010SRuslan Bukin * 44cc87010SRuslan Bukin * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com> 54cc87010SRuslan Bukin * 64cc87010SRuslan Bukin * This software was developed by SRI International and the University of 74cc87010SRuslan Bukin * Cambridge Computer Laboratory (Department of Computer Science and 84cc87010SRuslan Bukin * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the 94cc87010SRuslan Bukin * DARPA SSITH research programme. 104cc87010SRuslan Bukin * 114cc87010SRuslan Bukin * Portions of this work was supported by Innovate UK project 105694, 124cc87010SRuslan Bukin * "Digital Security by Design (DSbD) Technology Platform Prototype". 134cc87010SRuslan Bukin * 144cc87010SRuslan Bukin * Redistribution and use in source and binary forms, with or without 154cc87010SRuslan Bukin * modification, are permitted provided that the following conditions 164cc87010SRuslan Bukin * are met: 174cc87010SRuslan Bukin * 1. Redistributions of source code must retain the above copyright 184cc87010SRuslan Bukin * notice, this list of conditions and the following disclaimer. 194cc87010SRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright 204cc87010SRuslan Bukin * notice, this list of conditions and the following disclaimer in the 214cc87010SRuslan Bukin * documentation and/or other materials provided with the distribution. 224cc87010SRuslan Bukin * 234cc87010SRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 244cc87010SRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 254cc87010SRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 264cc87010SRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 274cc87010SRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 284cc87010SRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 294cc87010SRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 304cc87010SRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 314cc87010SRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 324cc87010SRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 334cc87010SRuslan Bukin * SUCH DAMAGE. 344cc87010SRuslan Bukin */ 354cc87010SRuslan Bukin 364cc87010SRuslan Bukin #include "opt_platform.h" 374cc87010SRuslan Bukin 384cc87010SRuslan Bukin #include <sys/cdefs.h> 394cc87010SRuslan Bukin __FBSDID("$FreeBSD$"); 404cc87010SRuslan Bukin 414cc87010SRuslan Bukin #include <sys/param.h> 424cc87010SRuslan Bukin #include <sys/bus.h> 434cc87010SRuslan Bukin #include <sys/kernel.h> 444cc87010SRuslan Bukin #include <sys/malloc.h> 454cc87010SRuslan Bukin #include <sys/memdesc.h> 464cc87010SRuslan Bukin #include <sys/tree.h> 474cc87010SRuslan Bukin #include <sys/taskqueue.h> 484cc87010SRuslan Bukin #include <sys/lock.h> 494cc87010SRuslan Bukin #include <sys/mutex.h> 507d0bbf43SRuslan Bukin #include <sys/sx.h> 514cc87010SRuslan Bukin #include <sys/sysctl.h> 524cc87010SRuslan Bukin #include <vm/vm.h> 534cc87010SRuslan Bukin 544cc87010SRuslan Bukin #include <dev/pci/pcireg.h> 554cc87010SRuslan Bukin #include <dev/pci/pcivar.h> 564cc87010SRuslan Bukin #include <machine/bus.h> 574cc87010SRuslan Bukin #include <dev/iommu/busdma_iommu.h> 584cc87010SRuslan Bukin #include <machine/vmparam.h> 594cc87010SRuslan Bukin 6041ce5498SRuslan Bukin #ifdef FDT 6141ce5498SRuslan Bukin #include <dev/fdt/fdt_common.h> 6241ce5498SRuslan Bukin #include <dev/ofw/ofw_bus.h> 6341ce5498SRuslan Bukin #include <dev/ofw/ofw_bus_subr.h> 6441ce5498SRuslan Bukin #endif 6541ce5498SRuslan Bukin 664cc87010SRuslan Bukin #include "iommu.h" 674cc87010SRuslan Bukin #include "iommu_if.h" 684cc87010SRuslan Bukin 694cc87010SRuslan Bukin static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework"); 704cc87010SRuslan Bukin 717d0bbf43SRuslan Bukin #define IOMMU_LIST_LOCK() sx_xlock(&iommu_sx) 727d0bbf43SRuslan Bukin #define IOMMU_LIST_UNLOCK() sx_xunlock(&iommu_sx) 737d0bbf43SRuslan Bukin #define IOMMU_LIST_ASSERT_LOCKED() sx_assert(&iommu_sx, SA_XLOCKED) 744cc87010SRuslan Bukin 754cc87010SRuslan Bukin #define dprintf(fmt, ...) 764cc87010SRuslan Bukin 777d0bbf43SRuslan Bukin static struct sx iommu_sx; 784cc87010SRuslan Bukin 794cc87010SRuslan Bukin struct iommu_entry { 804cc87010SRuslan Bukin struct iommu_unit *iommu; 814cc87010SRuslan Bukin LIST_ENTRY(iommu_entry) next; 824cc87010SRuslan Bukin }; 834cc87010SRuslan Bukin static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list); 844cc87010SRuslan Bukin 854cc87010SRuslan Bukin static int 864cc87010SRuslan Bukin iommu_domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base, 874cc87010SRuslan Bukin iommu_gaddr_t size, int flags) 884cc87010SRuslan Bukin { 894cc87010SRuslan Bukin struct iommu_unit *iommu; 904cc87010SRuslan Bukin int error; 914cc87010SRuslan Bukin 924cc87010SRuslan Bukin iommu = iodom->iommu; 934cc87010SRuslan Bukin 944cc87010SRuslan Bukin error = IOMMU_UNMAP(iommu->dev, iodom, base, size); 954cc87010SRuslan Bukin 964cc87010SRuslan Bukin return (error); 974cc87010SRuslan Bukin } 984cc87010SRuslan Bukin 994cc87010SRuslan Bukin static int 1004cc87010SRuslan Bukin iommu_domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base, 1014cc87010SRuslan Bukin iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags) 1024cc87010SRuslan Bukin { 1034cc87010SRuslan Bukin struct iommu_unit *iommu; 1044cc87010SRuslan Bukin vm_prot_t prot; 1054cc87010SRuslan Bukin vm_offset_t va; 1064cc87010SRuslan Bukin int error; 1074cc87010SRuslan Bukin 1084cc87010SRuslan Bukin dprintf("%s: base %lx, size %lx\n", __func__, base, size); 1094cc87010SRuslan Bukin 1104cc87010SRuslan Bukin prot = 0; 1114cc87010SRuslan Bukin if (eflags & IOMMU_MAP_ENTRY_READ) 1124cc87010SRuslan Bukin prot |= VM_PROT_READ; 1134cc87010SRuslan Bukin if (eflags & IOMMU_MAP_ENTRY_WRITE) 1144cc87010SRuslan Bukin prot |= VM_PROT_WRITE; 1154cc87010SRuslan Bukin 1164cc87010SRuslan Bukin va = base; 1174cc87010SRuslan Bukin 1184cc87010SRuslan Bukin iommu = iodom->iommu; 1194cc87010SRuslan Bukin 1204cc87010SRuslan Bukin error = IOMMU_MAP(iommu->dev, iodom, va, ma, size, prot); 1214cc87010SRuslan Bukin 1226ff00427SRuslan Bukin return (error); 1234cc87010SRuslan Bukin } 1244cc87010SRuslan Bukin 1254cc87010SRuslan Bukin static const struct iommu_domain_map_ops domain_map_ops = { 1264cc87010SRuslan Bukin .map = iommu_domain_map_buf, 1274cc87010SRuslan Bukin .unmap = iommu_domain_unmap_buf, 1284cc87010SRuslan Bukin }; 1294cc87010SRuslan Bukin 1304cc87010SRuslan Bukin static struct iommu_domain * 1314cc87010SRuslan Bukin iommu_domain_alloc(struct iommu_unit *iommu) 1324cc87010SRuslan Bukin { 1334cc87010SRuslan Bukin struct iommu_domain *iodom; 1344cc87010SRuslan Bukin 1354cc87010SRuslan Bukin iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu); 1364cc87010SRuslan Bukin if (iodom == NULL) 1374cc87010SRuslan Bukin return (NULL); 1384cc87010SRuslan Bukin 1394b4e8cb5SRuslan Bukin KASSERT(iodom->end != 0, ("domain end is not set")); 1404b4e8cb5SRuslan Bukin 1414cc87010SRuslan Bukin iommu_domain_init(iommu, iodom, &domain_map_ops); 1424cc87010SRuslan Bukin iodom->iommu = iommu; 1434cc87010SRuslan Bukin iommu_gas_init_domain(iodom); 1444cc87010SRuslan Bukin 1454cc87010SRuslan Bukin return (iodom); 1464cc87010SRuslan Bukin } 1474cc87010SRuslan Bukin 1484cc87010SRuslan Bukin static int 1494cc87010SRuslan Bukin iommu_domain_free(struct iommu_domain *iodom) 1504cc87010SRuslan Bukin { 1514cc87010SRuslan Bukin struct iommu_unit *iommu; 1524cc87010SRuslan Bukin 1534cc87010SRuslan Bukin iommu = iodom->iommu; 1544cc87010SRuslan Bukin 1554cc87010SRuslan Bukin IOMMU_LOCK(iommu); 1564cc87010SRuslan Bukin 1574cc87010SRuslan Bukin if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) { 1584cc87010SRuslan Bukin IOMMU_DOMAIN_LOCK(iodom); 1594cc87010SRuslan Bukin iommu_gas_fini_domain(iodom); 1604cc87010SRuslan Bukin IOMMU_DOMAIN_UNLOCK(iodom); 1614cc87010SRuslan Bukin } 1624cc87010SRuslan Bukin 1634cc87010SRuslan Bukin iommu_domain_fini(iodom); 1644cc87010SRuslan Bukin 1654cc87010SRuslan Bukin IOMMU_DOMAIN_FREE(iommu->dev, iodom); 1664cc87010SRuslan Bukin IOMMU_UNLOCK(iommu); 1674cc87010SRuslan Bukin 1684cc87010SRuslan Bukin return (0); 1694cc87010SRuslan Bukin } 1704cc87010SRuslan Bukin 1714cc87010SRuslan Bukin static void 1724b4e8cb5SRuslan Bukin iommu_tag_init(struct iommu_domain *iodom, struct bus_dma_tag_iommu *t) 1734cc87010SRuslan Bukin { 1744cc87010SRuslan Bukin bus_addr_t maxaddr; 1754cc87010SRuslan Bukin 1764b4e8cb5SRuslan Bukin maxaddr = MIN(iodom->end, BUS_SPACE_MAXADDR); 1774cc87010SRuslan Bukin 1784cc87010SRuslan Bukin t->common.ref_count = 0; 1794cc87010SRuslan Bukin t->common.impl = &bus_dma_iommu_impl; 1804cc87010SRuslan Bukin t->common.alignment = 1; 1814cc87010SRuslan Bukin t->common.boundary = 0; 1824cc87010SRuslan Bukin t->common.lowaddr = maxaddr; 1834cc87010SRuslan Bukin t->common.highaddr = maxaddr; 1844cc87010SRuslan Bukin t->common.maxsize = maxaddr; 1854cc87010SRuslan Bukin t->common.nsegments = BUS_SPACE_UNRESTRICTED; 1864cc87010SRuslan Bukin t->common.maxsegsz = maxaddr; 1874cc87010SRuslan Bukin } 1884cc87010SRuslan Bukin 1894cc87010SRuslan Bukin static struct iommu_ctx * 19041ce5498SRuslan Bukin iommu_ctx_alloc(device_t requester, struct iommu_domain *iodom, bool disabled) 1914cc87010SRuslan Bukin { 1924cc87010SRuslan Bukin struct iommu_unit *iommu; 1934cc87010SRuslan Bukin struct iommu_ctx *ioctx; 1944cc87010SRuslan Bukin 1954cc87010SRuslan Bukin iommu = iodom->iommu; 1964cc87010SRuslan Bukin 19741ce5498SRuslan Bukin ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, requester, disabled); 1984cc87010SRuslan Bukin if (ioctx == NULL) 1994cc87010SRuslan Bukin return (NULL); 2004cc87010SRuslan Bukin 20141ce5498SRuslan Bukin ioctx->domain = iodom; 20241ce5498SRuslan Bukin 20341ce5498SRuslan Bukin return (ioctx); 20441ce5498SRuslan Bukin } 20541ce5498SRuslan Bukin 20641ce5498SRuslan Bukin static int 20741ce5498SRuslan Bukin iommu_ctx_init(device_t requester, struct iommu_ctx *ioctx) 20841ce5498SRuslan Bukin { 20941ce5498SRuslan Bukin struct bus_dma_tag_iommu *tag; 21041ce5498SRuslan Bukin struct iommu_domain *iodom; 21141ce5498SRuslan Bukin struct iommu_unit *iommu; 21241ce5498SRuslan Bukin int error; 21341ce5498SRuslan Bukin 21441ce5498SRuslan Bukin iodom = ioctx->domain; 21541ce5498SRuslan Bukin iommu = iodom->iommu; 21641ce5498SRuslan Bukin 21741ce5498SRuslan Bukin error = IOMMU_CTX_INIT(iommu->dev, ioctx); 21841ce5498SRuslan Bukin if (error) 21941ce5498SRuslan Bukin return (error); 22041ce5498SRuslan Bukin 22141ce5498SRuslan Bukin tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu), 22241ce5498SRuslan Bukin M_IOMMU, M_WAITOK | M_ZERO); 22341ce5498SRuslan Bukin tag->owner = requester; 22441ce5498SRuslan Bukin tag->ctx = ioctx; 22541ce5498SRuslan Bukin tag->ctx->domain = iodom; 22641ce5498SRuslan Bukin 2274b4e8cb5SRuslan Bukin iommu_tag_init(iodom, tag); 22841ce5498SRuslan Bukin 22941ce5498SRuslan Bukin return (error); 23041ce5498SRuslan Bukin } 23141ce5498SRuslan Bukin 23241ce5498SRuslan Bukin static struct iommu_unit * 23341ce5498SRuslan Bukin iommu_lookup(device_t dev) 23441ce5498SRuslan Bukin { 23541ce5498SRuslan Bukin struct iommu_entry *entry; 23641ce5498SRuslan Bukin struct iommu_unit *iommu; 23741ce5498SRuslan Bukin 23841ce5498SRuslan Bukin IOMMU_LIST_LOCK(); 23941ce5498SRuslan Bukin LIST_FOREACH(entry, &iommu_list, next) { 24041ce5498SRuslan Bukin iommu = entry->iommu; 24141ce5498SRuslan Bukin if (iommu->dev == dev) { 24241ce5498SRuslan Bukin IOMMU_LIST_UNLOCK(); 24341ce5498SRuslan Bukin return (iommu); 24441ce5498SRuslan Bukin } 24541ce5498SRuslan Bukin } 24641ce5498SRuslan Bukin IOMMU_LIST_UNLOCK(); 24741ce5498SRuslan Bukin 24841ce5498SRuslan Bukin return (NULL); 24941ce5498SRuslan Bukin } 25041ce5498SRuslan Bukin 251dc08d52dSRuslan Bukin #ifdef FDT 25241ce5498SRuslan Bukin struct iommu_ctx * 25341ce5498SRuslan Bukin iommu_get_ctx_ofw(device_t dev, int channel) 25441ce5498SRuslan Bukin { 25541ce5498SRuslan Bukin struct iommu_domain *iodom; 25641ce5498SRuslan Bukin struct iommu_unit *iommu; 25741ce5498SRuslan Bukin struct iommu_ctx *ioctx; 25841ce5498SRuslan Bukin phandle_t node, parent; 25941ce5498SRuslan Bukin device_t iommu_dev; 26041ce5498SRuslan Bukin pcell_t *cells; 26141ce5498SRuslan Bukin int niommus; 26241ce5498SRuslan Bukin int ncells; 26341ce5498SRuslan Bukin int error; 26441ce5498SRuslan Bukin 26541ce5498SRuslan Bukin node = ofw_bus_get_node(dev); 26641ce5498SRuslan Bukin if (node <= 0) { 26741ce5498SRuslan Bukin device_printf(dev, 26841ce5498SRuslan Bukin "%s called on not ofw based device.\n", __func__); 26941ce5498SRuslan Bukin return (NULL); 27041ce5498SRuslan Bukin } 27141ce5498SRuslan Bukin 27241ce5498SRuslan Bukin error = ofw_bus_parse_xref_list_get_length(node, 27341ce5498SRuslan Bukin "iommus", "#iommu-cells", &niommus); 27441ce5498SRuslan Bukin if (error) { 27541ce5498SRuslan Bukin device_printf(dev, "%s can't get iommu list.\n", __func__); 27641ce5498SRuslan Bukin return (NULL); 27741ce5498SRuslan Bukin } 27841ce5498SRuslan Bukin 27941ce5498SRuslan Bukin if (niommus == 0) { 28041ce5498SRuslan Bukin device_printf(dev, "%s iommu list is empty.\n", __func__); 28141ce5498SRuslan Bukin return (NULL); 28241ce5498SRuslan Bukin } 28341ce5498SRuslan Bukin 28441ce5498SRuslan Bukin error = ofw_bus_parse_xref_list_alloc(node, "iommus", "#iommu-cells", 28541ce5498SRuslan Bukin channel, &parent, &ncells, &cells); 28641ce5498SRuslan Bukin if (error != 0) { 28741ce5498SRuslan Bukin device_printf(dev, "%s can't get iommu device xref.\n", 28841ce5498SRuslan Bukin __func__); 28941ce5498SRuslan Bukin return (NULL); 29041ce5498SRuslan Bukin } 29141ce5498SRuslan Bukin 29241ce5498SRuslan Bukin iommu_dev = OF_device_from_xref(parent); 29341ce5498SRuslan Bukin if (iommu_dev == NULL) { 29441ce5498SRuslan Bukin device_printf(dev, "%s can't get iommu device.\n", __func__); 29541ce5498SRuslan Bukin return (NULL); 29641ce5498SRuslan Bukin } 29741ce5498SRuslan Bukin 29841ce5498SRuslan Bukin iommu = iommu_lookup(iommu_dev); 29941ce5498SRuslan Bukin if (iommu == NULL) { 30041ce5498SRuslan Bukin device_printf(dev, "%s can't lookup iommu.\n", __func__); 30141ce5498SRuslan Bukin return (NULL); 30241ce5498SRuslan Bukin } 30341ce5498SRuslan Bukin 3044cc87010SRuslan Bukin /* 30541ce5498SRuslan Bukin * In our current configuration we have a domain per each ctx, 30641ce5498SRuslan Bukin * so allocate a domain first. 3074cc87010SRuslan Bukin */ 30841ce5498SRuslan Bukin iodom = iommu_domain_alloc(iommu); 30941ce5498SRuslan Bukin if (iodom == NULL) { 31041ce5498SRuslan Bukin device_printf(dev, "%s can't allocate domain.\n", __func__); 31141ce5498SRuslan Bukin return (NULL); 31241ce5498SRuslan Bukin } 31341ce5498SRuslan Bukin 31441ce5498SRuslan Bukin ioctx = iommu_ctx_alloc(dev, iodom, false); 31541ce5498SRuslan Bukin if (ioctx == NULL) { 31641ce5498SRuslan Bukin iommu_domain_free(iodom); 31741ce5498SRuslan Bukin return (NULL); 31841ce5498SRuslan Bukin } 31941ce5498SRuslan Bukin 32041ce5498SRuslan Bukin ioctx->domain = iodom; 32141ce5498SRuslan Bukin 32241ce5498SRuslan Bukin error = IOMMU_OFW_MD_DATA(iommu->dev, ioctx, cells, ncells); 32341ce5498SRuslan Bukin if (error) { 32441ce5498SRuslan Bukin device_printf(dev, "%s can't set MD data\n", __func__); 32541ce5498SRuslan Bukin return (NULL); 32641ce5498SRuslan Bukin } 32741ce5498SRuslan Bukin 32841ce5498SRuslan Bukin error = iommu_ctx_init(dev, ioctx); 32941ce5498SRuslan Bukin if (error) { 33041ce5498SRuslan Bukin IOMMU_CTX_FREE(iommu->dev, ioctx); 33141ce5498SRuslan Bukin iommu_domain_free(iodom); 33241ce5498SRuslan Bukin return (NULL); 33341ce5498SRuslan Bukin } 3344cc87010SRuslan Bukin 3354cc87010SRuslan Bukin return (ioctx); 3364cc87010SRuslan Bukin } 337dc08d52dSRuslan Bukin #endif 3384cc87010SRuslan Bukin 3394cc87010SRuslan Bukin struct iommu_ctx * 3404cc87010SRuslan Bukin iommu_get_ctx(struct iommu_unit *iommu, device_t requester, 3414cc87010SRuslan Bukin uint16_t rid, bool disabled, bool rmrr) 3424cc87010SRuslan Bukin { 3434cc87010SRuslan Bukin struct iommu_domain *iodom; 34441ce5498SRuslan Bukin struct iommu_ctx *ioctx; 34541ce5498SRuslan Bukin int error; 3464cc87010SRuslan Bukin 3474cc87010SRuslan Bukin IOMMU_LOCK(iommu); 3484cc87010SRuslan Bukin ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester); 3494cc87010SRuslan Bukin if (ioctx) { 3504cc87010SRuslan Bukin IOMMU_UNLOCK(iommu); 3514cc87010SRuslan Bukin return (ioctx); 3524cc87010SRuslan Bukin } 3534cc87010SRuslan Bukin IOMMU_UNLOCK(iommu); 3544cc87010SRuslan Bukin 3554cc87010SRuslan Bukin /* 3564cc87010SRuslan Bukin * In our current configuration we have a domain per each ctx. 3574cc87010SRuslan Bukin * So allocate a domain first. 3584cc87010SRuslan Bukin */ 3594cc87010SRuslan Bukin iodom = iommu_domain_alloc(iommu); 3604cc87010SRuslan Bukin if (iodom == NULL) 3614cc87010SRuslan Bukin return (NULL); 3624cc87010SRuslan Bukin 3634cc87010SRuslan Bukin ioctx = iommu_ctx_alloc(requester, iodom, disabled); 3644cc87010SRuslan Bukin if (ioctx == NULL) { 3654cc87010SRuslan Bukin iommu_domain_free(iodom); 3664cc87010SRuslan Bukin return (NULL); 3674cc87010SRuslan Bukin } 3684cc87010SRuslan Bukin 36941ce5498SRuslan Bukin error = iommu_ctx_init(requester, ioctx); 37041ce5498SRuslan Bukin if (error) { 37141ce5498SRuslan Bukin IOMMU_CTX_FREE(iommu->dev, ioctx); 37241ce5498SRuslan Bukin iommu_domain_free(iodom); 37341ce5498SRuslan Bukin return (NULL); 37441ce5498SRuslan Bukin } 3754cc87010SRuslan Bukin 3764cc87010SRuslan Bukin return (ioctx); 3774cc87010SRuslan Bukin } 3784cc87010SRuslan Bukin 3794cc87010SRuslan Bukin void 3804cc87010SRuslan Bukin iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx) 3814cc87010SRuslan Bukin { 3824cc87010SRuslan Bukin struct bus_dma_tag_iommu *tag; 3834cc87010SRuslan Bukin 3844cc87010SRuslan Bukin IOMMU_ASSERT_LOCKED(iommu); 3854cc87010SRuslan Bukin 3864cc87010SRuslan Bukin tag = ioctx->tag; 3874cc87010SRuslan Bukin 3884cc87010SRuslan Bukin IOMMU_CTX_FREE(iommu->dev, ioctx); 3894cc87010SRuslan Bukin 3904cc87010SRuslan Bukin free(tag, M_IOMMU); 3914cc87010SRuslan Bukin } 3924cc87010SRuslan Bukin 3934cc87010SRuslan Bukin void 3944cc87010SRuslan Bukin iommu_free_ctx(struct iommu_ctx *ioctx) 3954cc87010SRuslan Bukin { 3964cc87010SRuslan Bukin struct iommu_unit *iommu; 3974cc87010SRuslan Bukin struct iommu_domain *iodom; 3984cc87010SRuslan Bukin int error; 3994cc87010SRuslan Bukin 4004cc87010SRuslan Bukin iodom = ioctx->domain; 4014cc87010SRuslan Bukin iommu = iodom->iommu; 4024cc87010SRuslan Bukin 4034cc87010SRuslan Bukin IOMMU_LOCK(iommu); 4044cc87010SRuslan Bukin iommu_free_ctx_locked(iommu, ioctx); 4054cc87010SRuslan Bukin IOMMU_UNLOCK(iommu); 4064cc87010SRuslan Bukin 4074cc87010SRuslan Bukin /* Since we have a domain per each ctx, remove the domain too. */ 4084cc87010SRuslan Bukin error = iommu_domain_free(iodom); 4094cc87010SRuslan Bukin if (error) 4104cc87010SRuslan Bukin device_printf(iommu->dev, "Could not free a domain\n"); 4114cc87010SRuslan Bukin } 4124cc87010SRuslan Bukin 4134cc87010SRuslan Bukin static void 4144cc87010SRuslan Bukin iommu_domain_free_entry(struct iommu_map_entry *entry, bool free) 4154cc87010SRuslan Bukin { 4164670f908SAlan Cox iommu_gas_free_space(entry); 4174cc87010SRuslan Bukin 4184cc87010SRuslan Bukin if (free) 4194670f908SAlan Cox iommu_gas_free_entry(entry); 4204cc87010SRuslan Bukin else 4214cc87010SRuslan Bukin entry->flags = 0; 4224cc87010SRuslan Bukin } 4234cc87010SRuslan Bukin 4244cc87010SRuslan Bukin void 4254cc87010SRuslan Bukin iommu_domain_unload(struct iommu_domain *iodom, 4264cc87010SRuslan Bukin struct iommu_map_entries_tailq *entries, bool cansleep) 4274cc87010SRuslan Bukin { 4284cc87010SRuslan Bukin struct iommu_map_entry *entry, *entry1; 4290a8e88faSRuslan Bukin int error __diagused; 4304cc87010SRuslan Bukin 4314cc87010SRuslan Bukin TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 4324cc87010SRuslan Bukin KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, 4334cc87010SRuslan Bukin ("not mapped entry %p %p", iodom, entry)); 4344cc87010SRuslan Bukin error = iodom->ops->unmap(iodom, entry->start, entry->end - 4354cc87010SRuslan Bukin entry->start, cansleep ? IOMMU_PGF_WAITOK : 0); 4364cc87010SRuslan Bukin KASSERT(error == 0, ("unmap %p error %d", iodom, error)); 4374cc87010SRuslan Bukin TAILQ_REMOVE(entries, entry, dmamap_link); 4384cc87010SRuslan Bukin iommu_domain_free_entry(entry, true); 4394cc87010SRuslan Bukin } 4404cc87010SRuslan Bukin 4414cc87010SRuslan Bukin if (TAILQ_EMPTY(entries)) 4424cc87010SRuslan Bukin return; 4434cc87010SRuslan Bukin 4444cc87010SRuslan Bukin panic("entries map is not empty"); 4454cc87010SRuslan Bukin } 4464cc87010SRuslan Bukin 4474cc87010SRuslan Bukin int 4484cc87010SRuslan Bukin iommu_register(struct iommu_unit *iommu) 4494cc87010SRuslan Bukin { 4504cc87010SRuslan Bukin struct iommu_entry *entry; 4514cc87010SRuslan Bukin 4524cc87010SRuslan Bukin mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF); 4534cc87010SRuslan Bukin 4544cc87010SRuslan Bukin entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO); 4554cc87010SRuslan Bukin entry->iommu = iommu; 4564cc87010SRuslan Bukin 4574cc87010SRuslan Bukin IOMMU_LIST_LOCK(); 4584cc87010SRuslan Bukin LIST_INSERT_HEAD(&iommu_list, entry, next); 4594cc87010SRuslan Bukin IOMMU_LIST_UNLOCK(); 4604cc87010SRuslan Bukin 4614cc87010SRuslan Bukin iommu_init_busdma(iommu); 4624cc87010SRuslan Bukin 4634cc87010SRuslan Bukin return (0); 4644cc87010SRuslan Bukin } 4654cc87010SRuslan Bukin 4664cc87010SRuslan Bukin int 4674cc87010SRuslan Bukin iommu_unregister(struct iommu_unit *iommu) 4684cc87010SRuslan Bukin { 4694cc87010SRuslan Bukin struct iommu_entry *entry, *tmp; 4704cc87010SRuslan Bukin 4714cc87010SRuslan Bukin IOMMU_LIST_LOCK(); 4724cc87010SRuslan Bukin LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) { 4734cc87010SRuslan Bukin if (entry->iommu == iommu) { 4744cc87010SRuslan Bukin LIST_REMOVE(entry, next); 4754cc87010SRuslan Bukin free(entry, M_IOMMU); 4764cc87010SRuslan Bukin } 4774cc87010SRuslan Bukin } 4784cc87010SRuslan Bukin IOMMU_LIST_UNLOCK(); 4794cc87010SRuslan Bukin 4804cc87010SRuslan Bukin iommu_fini_busdma(iommu); 4814cc87010SRuslan Bukin 4824cc87010SRuslan Bukin mtx_destroy(&iommu->lock); 4834cc87010SRuslan Bukin 4844cc87010SRuslan Bukin return (0); 4854cc87010SRuslan Bukin } 4864cc87010SRuslan Bukin 4874cc87010SRuslan Bukin struct iommu_unit * 4884cc87010SRuslan Bukin iommu_find(device_t dev, bool verbose) 4894cc87010SRuslan Bukin { 4904cc87010SRuslan Bukin struct iommu_entry *entry; 4914cc87010SRuslan Bukin struct iommu_unit *iommu; 4924cc87010SRuslan Bukin int error; 4934cc87010SRuslan Bukin 4944cc87010SRuslan Bukin IOMMU_LIST_LOCK(); 4954cc87010SRuslan Bukin LIST_FOREACH(entry, &iommu_list, next) { 4964cc87010SRuslan Bukin iommu = entry->iommu; 4974cc87010SRuslan Bukin error = IOMMU_FIND(iommu->dev, dev); 4984cc87010SRuslan Bukin if (error == 0) { 4994cc87010SRuslan Bukin IOMMU_LIST_UNLOCK(); 5004cc87010SRuslan Bukin return (entry->iommu); 5014cc87010SRuslan Bukin } 5024cc87010SRuslan Bukin } 5034cc87010SRuslan Bukin IOMMU_LIST_UNLOCK(); 5044cc87010SRuslan Bukin 5054cc87010SRuslan Bukin return (NULL); 5064cc87010SRuslan Bukin } 5074cc87010SRuslan Bukin 5084cc87010SRuslan Bukin void 5098bc36738SAlan Cox iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free, 5108bc36738SAlan Cox bool cansleep __unused) 5114cc87010SRuslan Bukin { 5124cc87010SRuslan Bukin 5134cc87010SRuslan Bukin dprintf("%s\n", __func__); 5144cc87010SRuslan Bukin 5154cc87010SRuslan Bukin iommu_domain_free_entry(entry, free); 5164cc87010SRuslan Bukin } 5174cc87010SRuslan Bukin 5184cc87010SRuslan Bukin static void 5194cc87010SRuslan Bukin iommu_init(void) 5204cc87010SRuslan Bukin { 5214cc87010SRuslan Bukin 5227d0bbf43SRuslan Bukin sx_init(&iommu_sx, "IOMMU list"); 5234cc87010SRuslan Bukin } 5244cc87010SRuslan Bukin 5254cc87010SRuslan Bukin SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL); 526