13024e8afSRuslan Bukin /*- 23024e8afSRuslan Bukin * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 33024e8afSRuslan Bukin * 43024e8afSRuslan Bukin * Copyright (c) 2013 The FreeBSD Foundation 53024e8afSRuslan Bukin * 63024e8afSRuslan Bukin * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 73024e8afSRuslan Bukin * under sponsorship from the FreeBSD Foundation. 83024e8afSRuslan Bukin * 93024e8afSRuslan Bukin * Redistribution and use in source and binary forms, with or without 103024e8afSRuslan Bukin * modification, are permitted provided that the following conditions 113024e8afSRuslan Bukin * are met: 123024e8afSRuslan Bukin * 1. Redistributions of source code must retain the above copyright 133024e8afSRuslan Bukin * notice, this list of conditions and the following disclaimer. 143024e8afSRuslan Bukin * 2. Redistributions in binary form must reproduce the above copyright 153024e8afSRuslan Bukin * notice, this list of conditions and the following disclaimer in the 163024e8afSRuslan Bukin * documentation and/or other materials provided with the distribution. 173024e8afSRuslan Bukin * 183024e8afSRuslan Bukin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 193024e8afSRuslan Bukin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 203024e8afSRuslan Bukin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 213024e8afSRuslan Bukin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 223024e8afSRuslan Bukin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 233024e8afSRuslan Bukin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 243024e8afSRuslan Bukin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 253024e8afSRuslan Bukin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 263024e8afSRuslan Bukin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 273024e8afSRuslan Bukin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 283024e8afSRuslan Bukin * SUCH DAMAGE. 293024e8afSRuslan Bukin */ 303024e8afSRuslan Bukin 313024e8afSRuslan Bukin #include <sys/cdefs.h> 323024e8afSRuslan Bukin __FBSDID("$FreeBSD$"); 333024e8afSRuslan Bukin 343024e8afSRuslan Bukin #define RB_AUGMENT(entry) iommu_gas_augment_entry(entry) 353024e8afSRuslan Bukin 363024e8afSRuslan Bukin #include <sys/param.h> 373024e8afSRuslan Bukin #include <sys/systm.h> 383024e8afSRuslan Bukin #include <sys/malloc.h> 393024e8afSRuslan Bukin #include <sys/bus.h> 403024e8afSRuslan Bukin #include <sys/interrupt.h> 413024e8afSRuslan Bukin #include <sys/kernel.h> 423024e8afSRuslan Bukin #include <sys/ktr.h> 433024e8afSRuslan Bukin #include <sys/lock.h> 443024e8afSRuslan Bukin #include <sys/proc.h> 453024e8afSRuslan Bukin #include <sys/rwlock.h> 463024e8afSRuslan Bukin #include <sys/memdesc.h> 473024e8afSRuslan Bukin #include <sys/mutex.h> 483024e8afSRuslan Bukin #include <sys/sysctl.h> 493024e8afSRuslan Bukin #include <sys/rman.h> 503024e8afSRuslan Bukin #include <sys/taskqueue.h> 513024e8afSRuslan Bukin #include <sys/tree.h> 523024e8afSRuslan Bukin #include <sys/uio.h> 533024e8afSRuslan Bukin #include <sys/vmem.h> 543024e8afSRuslan Bukin #include <vm/vm.h> 553024e8afSRuslan Bukin #include <vm/vm_extern.h> 563024e8afSRuslan Bukin #include <vm/vm_kern.h> 573024e8afSRuslan Bukin #include <vm/vm_object.h> 583024e8afSRuslan Bukin #include <vm/vm_page.h> 593024e8afSRuslan Bukin #include <vm/vm_map.h> 603024e8afSRuslan Bukin #include <vm/uma.h> 61c8597a1fSRuslan Bukin #include <dev/pci/pcireg.h> 62c8597a1fSRuslan Bukin #include <dev/pci/pcivar.h> 63c8597a1fSRuslan Bukin #include <dev/iommu/iommu.h> 64f23f7d3aSRuslan Bukin #include <dev/iommu/iommu_gas.h> 65e707c8beSRuslan Bukin #include <dev/iommu/iommu_msi.h> 663024e8afSRuslan Bukin #include <machine/atomic.h> 673024e8afSRuslan Bukin #include <machine/bus.h> 683024e8afSRuslan Bukin #include <machine/md_var.h> 69c4cd6990SRuslan Bukin #include <machine/iommu.h> 70c8597a1fSRuslan Bukin #include <dev/iommu/busdma_iommu.h> 713024e8afSRuslan Bukin 723024e8afSRuslan Bukin /* 733024e8afSRuslan Bukin * Guest Address Space management. 743024e8afSRuslan Bukin */ 753024e8afSRuslan Bukin 763024e8afSRuslan Bukin static uma_zone_t iommu_map_entry_zone; 773024e8afSRuslan Bukin 789c843a40SRuslan Bukin #ifdef INVARIANTS 799c843a40SRuslan Bukin static int iommu_check_free; 809c843a40SRuslan Bukin #endif 819c843a40SRuslan Bukin 823024e8afSRuslan Bukin static void 833024e8afSRuslan Bukin intel_gas_init(void) 843024e8afSRuslan Bukin { 853024e8afSRuslan Bukin 863024e8afSRuslan Bukin iommu_map_entry_zone = uma_zcreate("IOMMU_MAP_ENTRY", 873024e8afSRuslan Bukin sizeof(struct iommu_map_entry), NULL, NULL, 883024e8afSRuslan Bukin NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NODUMP); 893024e8afSRuslan Bukin } 903024e8afSRuslan Bukin SYSINIT(intel_gas, SI_SUB_DRIVERS, SI_ORDER_FIRST, intel_gas_init, NULL); 913024e8afSRuslan Bukin 923024e8afSRuslan Bukin struct iommu_map_entry * 933024e8afSRuslan Bukin iommu_gas_alloc_entry(struct iommu_domain *domain, u_int flags) 943024e8afSRuslan Bukin { 953024e8afSRuslan Bukin struct iommu_map_entry *res; 963024e8afSRuslan Bukin 9715f6baf4SRuslan Bukin KASSERT((flags & ~(IOMMU_PGF_WAITOK)) == 0, 983024e8afSRuslan Bukin ("unsupported flags %x", flags)); 993024e8afSRuslan Bukin 10015f6baf4SRuslan Bukin res = uma_zalloc(iommu_map_entry_zone, ((flags & IOMMU_PGF_WAITOK) != 1013024e8afSRuslan Bukin 0 ? M_WAITOK : M_NOWAIT) | M_ZERO); 1023024e8afSRuslan Bukin if (res != NULL) { 1033024e8afSRuslan Bukin res->domain = domain; 1043024e8afSRuslan Bukin atomic_add_int(&domain->entries_cnt, 1); 1053024e8afSRuslan Bukin } 1063024e8afSRuslan Bukin return (res); 1073024e8afSRuslan Bukin } 1083024e8afSRuslan Bukin 1093024e8afSRuslan Bukin void 1103024e8afSRuslan Bukin iommu_gas_free_entry(struct iommu_domain *domain, struct iommu_map_entry *entry) 1113024e8afSRuslan Bukin { 1123024e8afSRuslan Bukin 113b64dca2bSRuslan Bukin KASSERT(domain == entry->domain, 1143024e8afSRuslan Bukin ("mismatched free domain %p entry %p entry->domain %p", domain, 1153024e8afSRuslan Bukin entry, entry->domain)); 1163024e8afSRuslan Bukin atomic_subtract_int(&domain->entries_cnt, 1); 1173024e8afSRuslan Bukin uma_zfree(iommu_map_entry_zone, entry); 1183024e8afSRuslan Bukin } 1193024e8afSRuslan Bukin 1203024e8afSRuslan Bukin static int 1213024e8afSRuslan Bukin iommu_gas_cmp_entries(struct iommu_map_entry *a, struct iommu_map_entry *b) 1223024e8afSRuslan Bukin { 1233024e8afSRuslan Bukin 1243024e8afSRuslan Bukin /* Last entry have zero size, so <= */ 1253024e8afSRuslan Bukin KASSERT(a->start <= a->end, ("inverted entry %p (%jx, %jx)", 1263024e8afSRuslan Bukin a, (uintmax_t)a->start, (uintmax_t)a->end)); 1273024e8afSRuslan Bukin KASSERT(b->start <= b->end, ("inverted entry %p (%jx, %jx)", 1283024e8afSRuslan Bukin b, (uintmax_t)b->start, (uintmax_t)b->end)); 1293024e8afSRuslan Bukin KASSERT(a->end <= b->start || b->end <= a->start || 1303024e8afSRuslan Bukin a->end == a->start || b->end == b->start, 1313024e8afSRuslan Bukin ("overlapping entries %p (%jx, %jx) %p (%jx, %jx)", 1323024e8afSRuslan Bukin a, (uintmax_t)a->start, (uintmax_t)a->end, 1333024e8afSRuslan Bukin b, (uintmax_t)b->start, (uintmax_t)b->end)); 1343024e8afSRuslan Bukin 1353024e8afSRuslan Bukin if (a->end < b->end) 1363024e8afSRuslan Bukin return (-1); 1373024e8afSRuslan Bukin else if (b->end < a->end) 1383024e8afSRuslan Bukin return (1); 1393024e8afSRuslan Bukin return (0); 1403024e8afSRuslan Bukin } 1413024e8afSRuslan Bukin 1423024e8afSRuslan Bukin static void 1433024e8afSRuslan Bukin iommu_gas_augment_entry(struct iommu_map_entry *entry) 1443024e8afSRuslan Bukin { 1453024e8afSRuslan Bukin struct iommu_map_entry *child; 1463024e8afSRuslan Bukin iommu_gaddr_t free_down; 1473024e8afSRuslan Bukin 1483024e8afSRuslan Bukin free_down = 0; 1493024e8afSRuslan Bukin if ((child = RB_LEFT(entry, rb_entry)) != NULL) { 1503024e8afSRuslan Bukin free_down = MAX(free_down, child->free_down); 1513024e8afSRuslan Bukin free_down = MAX(free_down, entry->start - child->last); 1523024e8afSRuslan Bukin entry->first = child->first; 1533024e8afSRuslan Bukin } else 1543024e8afSRuslan Bukin entry->first = entry->start; 1553024e8afSRuslan Bukin 1563024e8afSRuslan Bukin if ((child = RB_RIGHT(entry, rb_entry)) != NULL) { 1573024e8afSRuslan Bukin free_down = MAX(free_down, child->free_down); 1583024e8afSRuslan Bukin free_down = MAX(free_down, child->first - entry->end); 1593024e8afSRuslan Bukin entry->last = child->last; 1603024e8afSRuslan Bukin } else 1613024e8afSRuslan Bukin entry->last = entry->end; 1623024e8afSRuslan Bukin entry->free_down = free_down; 1633024e8afSRuslan Bukin } 1643024e8afSRuslan Bukin 1653024e8afSRuslan Bukin RB_GENERATE(iommu_gas_entries_tree, iommu_map_entry, rb_entry, 1663024e8afSRuslan Bukin iommu_gas_cmp_entries); 1673024e8afSRuslan Bukin 1683024e8afSRuslan Bukin #ifdef INVARIANTS 1693024e8afSRuslan Bukin static void 1703024e8afSRuslan Bukin iommu_gas_check_free(struct iommu_domain *domain) 1713024e8afSRuslan Bukin { 1723024e8afSRuslan Bukin struct iommu_map_entry *entry, *l, *r; 1733024e8afSRuslan Bukin iommu_gaddr_t v; 1743024e8afSRuslan Bukin 1753024e8afSRuslan Bukin RB_FOREACH(entry, iommu_gas_entries_tree, &domain->rb_root) { 176b64dca2bSRuslan Bukin KASSERT(domain == entry->domain, 1773024e8afSRuslan Bukin ("mismatched free domain %p entry %p entry->domain %p", 1783024e8afSRuslan Bukin domain, entry, entry->domain)); 1793024e8afSRuslan Bukin l = RB_LEFT(entry, rb_entry); 1803024e8afSRuslan Bukin r = RB_RIGHT(entry, rb_entry); 1813024e8afSRuslan Bukin v = 0; 1823024e8afSRuslan Bukin if (l != NULL) { 1833024e8afSRuslan Bukin v = MAX(v, l->free_down); 1843024e8afSRuslan Bukin v = MAX(v, entry->start - l->last); 1853024e8afSRuslan Bukin } 1863024e8afSRuslan Bukin if (r != NULL) { 1873024e8afSRuslan Bukin v = MAX(v, r->free_down); 1883024e8afSRuslan Bukin v = MAX(v, r->first - entry->end); 1893024e8afSRuslan Bukin } 1903024e8afSRuslan Bukin MPASS(entry->free_down == v); 1913024e8afSRuslan Bukin } 1923024e8afSRuslan Bukin } 1933024e8afSRuslan Bukin #endif 1943024e8afSRuslan Bukin 1953024e8afSRuslan Bukin static bool 1963024e8afSRuslan Bukin iommu_gas_rb_insert(struct iommu_domain *domain, struct iommu_map_entry *entry) 1973024e8afSRuslan Bukin { 1983024e8afSRuslan Bukin struct iommu_map_entry *found; 1993024e8afSRuslan Bukin 2003024e8afSRuslan Bukin found = RB_INSERT(iommu_gas_entries_tree, 2013024e8afSRuslan Bukin &domain->rb_root, entry); 2023024e8afSRuslan Bukin return (found == NULL); 2033024e8afSRuslan Bukin } 2043024e8afSRuslan Bukin 2053024e8afSRuslan Bukin static void 2063024e8afSRuslan Bukin iommu_gas_rb_remove(struct iommu_domain *domain, struct iommu_map_entry *entry) 2073024e8afSRuslan Bukin { 2083024e8afSRuslan Bukin 2093024e8afSRuslan Bukin RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); 2103024e8afSRuslan Bukin } 2113024e8afSRuslan Bukin 212f5cafae1SRuslan Bukin struct iommu_domain * 213f5cafae1SRuslan Bukin iommu_get_ctx_domain(struct iommu_ctx *ctx) 214f5cafae1SRuslan Bukin { 215f5cafae1SRuslan Bukin 216f5cafae1SRuslan Bukin return (ctx->domain); 217f5cafae1SRuslan Bukin } 218f5cafae1SRuslan Bukin 2193024e8afSRuslan Bukin void 2203024e8afSRuslan Bukin iommu_gas_init_domain(struct iommu_domain *domain) 2213024e8afSRuslan Bukin { 2223024e8afSRuslan Bukin struct iommu_map_entry *begin, *end; 2233024e8afSRuslan Bukin 22415f6baf4SRuslan Bukin begin = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); 22515f6baf4SRuslan Bukin end = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); 2263024e8afSRuslan Bukin 2273024e8afSRuslan Bukin IOMMU_DOMAIN_LOCK(domain); 2283024e8afSRuslan Bukin KASSERT(domain->entries_cnt == 2, ("dirty domain %p", domain)); 2293024e8afSRuslan Bukin KASSERT(RB_EMPTY(&domain->rb_root), 2303024e8afSRuslan Bukin ("non-empty entries %p", domain)); 2313024e8afSRuslan Bukin 2323024e8afSRuslan Bukin begin->start = 0; 2333024e8afSRuslan Bukin begin->end = IOMMU_PAGE_SIZE; 2343024e8afSRuslan Bukin begin->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; 2353024e8afSRuslan Bukin iommu_gas_rb_insert(domain, begin); 2363024e8afSRuslan Bukin 2373024e8afSRuslan Bukin end->start = domain->end; 2383024e8afSRuslan Bukin end->end = domain->end; 2393024e8afSRuslan Bukin end->flags = IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED; 2403024e8afSRuslan Bukin iommu_gas_rb_insert(domain, end); 2413024e8afSRuslan Bukin 2423024e8afSRuslan Bukin domain->first_place = begin; 2433024e8afSRuslan Bukin domain->last_place = end; 24415f6baf4SRuslan Bukin domain->flags |= IOMMU_DOMAIN_GAS_INITED; 2453024e8afSRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 2463024e8afSRuslan Bukin } 2473024e8afSRuslan Bukin 2483024e8afSRuslan Bukin void 2493024e8afSRuslan Bukin iommu_gas_fini_domain(struct iommu_domain *domain) 2503024e8afSRuslan Bukin { 2513024e8afSRuslan Bukin struct iommu_map_entry *entry, *entry1; 2523024e8afSRuslan Bukin 2533024e8afSRuslan Bukin IOMMU_DOMAIN_ASSERT_LOCKED(domain); 2543024e8afSRuslan Bukin KASSERT(domain->entries_cnt == 2, 2553024e8afSRuslan Bukin ("domain still in use %p", domain)); 2563024e8afSRuslan Bukin 2573024e8afSRuslan Bukin entry = RB_MIN(iommu_gas_entries_tree, &domain->rb_root); 2583024e8afSRuslan Bukin KASSERT(entry->start == 0, ("start entry start %p", domain)); 2593024e8afSRuslan Bukin KASSERT(entry->end == IOMMU_PAGE_SIZE, ("start entry end %p", domain)); 260dea8594fSRuslan Bukin KASSERT(entry->flags == 261dea8594fSRuslan Bukin (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED), 2623024e8afSRuslan Bukin ("start entry flags %p", domain)); 2633024e8afSRuslan Bukin RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); 2643024e8afSRuslan Bukin iommu_gas_free_entry(domain, entry); 2653024e8afSRuslan Bukin 2663024e8afSRuslan Bukin entry = RB_MAX(iommu_gas_entries_tree, &domain->rb_root); 2673024e8afSRuslan Bukin KASSERT(entry->start == domain->end, ("end entry start %p", domain)); 2683024e8afSRuslan Bukin KASSERT(entry->end == domain->end, ("end entry end %p", domain)); 269dea8594fSRuslan Bukin KASSERT(entry->flags == 270dea8594fSRuslan Bukin (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_UNMAPPED), 2713024e8afSRuslan Bukin ("end entry flags %p", domain)); 2723024e8afSRuslan Bukin RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, entry); 2733024e8afSRuslan Bukin iommu_gas_free_entry(domain, entry); 2743024e8afSRuslan Bukin 2753024e8afSRuslan Bukin RB_FOREACH_SAFE(entry, iommu_gas_entries_tree, &domain->rb_root, 2763024e8afSRuslan Bukin entry1) { 2773024e8afSRuslan Bukin KASSERT((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0, 2783024e8afSRuslan Bukin ("non-RMRR entry left %p", domain)); 2793024e8afSRuslan Bukin RB_REMOVE(iommu_gas_entries_tree, &domain->rb_root, 2803024e8afSRuslan Bukin entry); 2813024e8afSRuslan Bukin iommu_gas_free_entry(domain, entry); 2823024e8afSRuslan Bukin } 2833024e8afSRuslan Bukin } 2843024e8afSRuslan Bukin 2853024e8afSRuslan Bukin struct iommu_gas_match_args { 2863024e8afSRuslan Bukin struct iommu_domain *domain; 2873024e8afSRuslan Bukin iommu_gaddr_t size; 2883024e8afSRuslan Bukin int offset; 2893024e8afSRuslan Bukin const struct bus_dma_tag_common *common; 2903024e8afSRuslan Bukin u_int gas_flags; 2913024e8afSRuslan Bukin struct iommu_map_entry *entry; 2923024e8afSRuslan Bukin }; 2933024e8afSRuslan Bukin 2943024e8afSRuslan Bukin /* 2953024e8afSRuslan Bukin * The interval [beg, end) is a free interval between two iommu_map_entries. 2963024e8afSRuslan Bukin * maxaddr is an upper bound on addresses that can be allocated. Try to 2973024e8afSRuslan Bukin * allocate space in the free interval, subject to the conditions expressed 2983024e8afSRuslan Bukin * by a, and return 'true' if and only if the allocation attempt succeeds. 2993024e8afSRuslan Bukin */ 3003024e8afSRuslan Bukin static bool 3013024e8afSRuslan Bukin iommu_gas_match_one(struct iommu_gas_match_args *a, iommu_gaddr_t beg, 3023024e8afSRuslan Bukin iommu_gaddr_t end, iommu_gaddr_t maxaddr) 3033024e8afSRuslan Bukin { 3043024e8afSRuslan Bukin iommu_gaddr_t bs, start; 3053024e8afSRuslan Bukin 3063024e8afSRuslan Bukin a->entry->start = roundup2(beg + IOMMU_PAGE_SIZE, 3073024e8afSRuslan Bukin a->common->alignment); 30811fced21SDoug Moore if (a->entry->start + a->offset + a->size > maxaddr) 3093024e8afSRuslan Bukin return (false); 3103024e8afSRuslan Bukin 3113024e8afSRuslan Bukin /* IOMMU_PAGE_SIZE to create gap after new entry. */ 3123024e8afSRuslan Bukin if (a->entry->start < beg + IOMMU_PAGE_SIZE || 3133024e8afSRuslan Bukin a->entry->start + a->size + a->offset + IOMMU_PAGE_SIZE > end) 3143024e8afSRuslan Bukin return (false); 3153024e8afSRuslan Bukin 3163024e8afSRuslan Bukin /* No boundary crossing. */ 317c606ab59SDoug Moore if (vm_addr_bound_ok(a->entry->start + a->offset, a->size, 3183024e8afSRuslan Bukin a->common->boundary)) 3193024e8afSRuslan Bukin return (true); 3203024e8afSRuslan Bukin 3213024e8afSRuslan Bukin /* 3223024e8afSRuslan Bukin * The start + offset to start + offset + size region crosses 3233024e8afSRuslan Bukin * the boundary. Check if there is enough space after the 3243024e8afSRuslan Bukin * next boundary after the beg. 3253024e8afSRuslan Bukin */ 3263024e8afSRuslan Bukin bs = rounddown2(a->entry->start + a->offset + a->common->boundary, 3273024e8afSRuslan Bukin a->common->boundary); 3283024e8afSRuslan Bukin start = roundup2(bs, a->common->alignment); 3293024e8afSRuslan Bukin /* IOMMU_PAGE_SIZE to create gap after new entry. */ 3303024e8afSRuslan Bukin if (start + a->offset + a->size + IOMMU_PAGE_SIZE <= end && 3313024e8afSRuslan Bukin start + a->offset + a->size <= maxaddr && 332cfb2aa3fSDoug Moore vm_addr_bound_ok(start + a->offset, a->size, 3333024e8afSRuslan Bukin a->common->boundary)) { 3343024e8afSRuslan Bukin a->entry->start = start; 3353024e8afSRuslan Bukin return (true); 3363024e8afSRuslan Bukin } 3373024e8afSRuslan Bukin 3383024e8afSRuslan Bukin /* 3393024e8afSRuslan Bukin * Not enough space to align at the requested boundary, or 3403024e8afSRuslan Bukin * boundary is smaller than the size, but allowed to split. 3413024e8afSRuslan Bukin * We already checked that start + size does not overlap maxaddr. 3423024e8afSRuslan Bukin * 3433024e8afSRuslan Bukin * XXXKIB. It is possible that bs is exactly at the start of 3443024e8afSRuslan Bukin * the next entry, then we do not have gap. Ignore for now. 3453024e8afSRuslan Bukin */ 3463024e8afSRuslan Bukin if ((a->gas_flags & IOMMU_MF_CANSPLIT) != 0) { 3473024e8afSRuslan Bukin a->size = bs - a->entry->start; 3483024e8afSRuslan Bukin return (true); 3493024e8afSRuslan Bukin } 3503024e8afSRuslan Bukin 3513024e8afSRuslan Bukin return (false); 3523024e8afSRuslan Bukin } 3533024e8afSRuslan Bukin 3543024e8afSRuslan Bukin static void 3553024e8afSRuslan Bukin iommu_gas_match_insert(struct iommu_gas_match_args *a) 3563024e8afSRuslan Bukin { 357661bd70bSKonstantin Belousov bool found __diagused; 3583024e8afSRuslan Bukin 3593024e8afSRuslan Bukin /* 3603024e8afSRuslan Bukin * The prev->end is always aligned on the page size, which 3613024e8afSRuslan Bukin * causes page alignment for the entry->start too. The size 3623024e8afSRuslan Bukin * is checked to be multiple of the page size. 3633024e8afSRuslan Bukin * 3643024e8afSRuslan Bukin * The page sized gap is created between consequent 3653024e8afSRuslan Bukin * allocations to ensure that out-of-bounds accesses fault. 3663024e8afSRuslan Bukin */ 3673024e8afSRuslan Bukin a->entry->end = a->entry->start + a->size; 3683024e8afSRuslan Bukin 3693024e8afSRuslan Bukin found = iommu_gas_rb_insert(a->domain, a->entry); 3703024e8afSRuslan Bukin KASSERT(found, ("found dup %p start %jx size %jx", 3713024e8afSRuslan Bukin a->domain, (uintmax_t)a->entry->start, (uintmax_t)a->size)); 3723024e8afSRuslan Bukin a->entry->flags = IOMMU_MAP_ENTRY_MAP; 3733024e8afSRuslan Bukin } 3743024e8afSRuslan Bukin 3753024e8afSRuslan Bukin static int 3763024e8afSRuslan Bukin iommu_gas_lowermatch(struct iommu_gas_match_args *a, struct iommu_map_entry *entry) 3773024e8afSRuslan Bukin { 3783024e8afSRuslan Bukin struct iommu_map_entry *child; 3793024e8afSRuslan Bukin 380b831865fSDoug Moore /* 381b831865fSDoug Moore * If the subtree doesn't have free space for the requested allocation 382b831865fSDoug Moore * plus two guard pages, give up. 383b831865fSDoug Moore */ 384b831865fSDoug Moore if (entry->free_down < a->size + a->offset + 2 * IOMMU_PAGE_SIZE) 3853024e8afSRuslan Bukin return (ENOMEM); 3863024e8afSRuslan Bukin if (entry->first >= a->common->lowaddr) 3873024e8afSRuslan Bukin return (ENOMEM); 38830031172SDoug Moore child = RB_RIGHT(entry, rb_entry); 3893024e8afSRuslan Bukin if (child != NULL && 0 == iommu_gas_lowermatch(a, child)) 3903024e8afSRuslan Bukin return (0); 39130031172SDoug Moore if (child != NULL && entry->end < a->common->lowaddr && 39230031172SDoug Moore iommu_gas_match_one(a, entry->end, child->first, 39330031172SDoug Moore a->common->lowaddr)) { 39430031172SDoug Moore iommu_gas_match_insert(a); 39530031172SDoug Moore return (0); 39630031172SDoug Moore } 39730031172SDoug Moore child = RB_LEFT(entry, rb_entry); 3983024e8afSRuslan Bukin if (child != NULL && child->last < a->common->lowaddr && 3993024e8afSRuslan Bukin iommu_gas_match_one(a, child->last, entry->start, 4003024e8afSRuslan Bukin a->common->lowaddr)) { 4013024e8afSRuslan Bukin iommu_gas_match_insert(a); 4023024e8afSRuslan Bukin return (0); 4033024e8afSRuslan Bukin } 4043024e8afSRuslan Bukin if (child != NULL && 0 == iommu_gas_lowermatch(a, child)) 4053024e8afSRuslan Bukin return (0); 4063024e8afSRuslan Bukin return (ENOMEM); 4073024e8afSRuslan Bukin } 4083024e8afSRuslan Bukin 4093024e8afSRuslan Bukin static int 4103024e8afSRuslan Bukin iommu_gas_uppermatch(struct iommu_gas_match_args *a, struct iommu_map_entry *entry) 4113024e8afSRuslan Bukin { 4123024e8afSRuslan Bukin struct iommu_map_entry *child; 4133024e8afSRuslan Bukin 414b831865fSDoug Moore /* 415b831865fSDoug Moore * If the subtree doesn't have free space for the requested allocation 416b831865fSDoug Moore * plus two guard pages, give up. 417b831865fSDoug Moore */ 418b831865fSDoug Moore if (entry->free_down < a->size + a->offset + 2 * IOMMU_PAGE_SIZE) 4193024e8afSRuslan Bukin return (ENOMEM); 4203024e8afSRuslan Bukin if (entry->last < a->common->highaddr) 4213024e8afSRuslan Bukin return (ENOMEM); 4223024e8afSRuslan Bukin child = RB_LEFT(entry, rb_entry); 4233024e8afSRuslan Bukin if (child != NULL && 0 == iommu_gas_uppermatch(a, child)) 4243024e8afSRuslan Bukin return (0); 4253024e8afSRuslan Bukin if (child != NULL && child->last >= a->common->highaddr && 4263024e8afSRuslan Bukin iommu_gas_match_one(a, child->last, entry->start, 4273024e8afSRuslan Bukin a->domain->end)) { 4283024e8afSRuslan Bukin iommu_gas_match_insert(a); 4293024e8afSRuslan Bukin return (0); 4303024e8afSRuslan Bukin } 4313024e8afSRuslan Bukin child = RB_RIGHT(entry, rb_entry); 4323024e8afSRuslan Bukin if (child != NULL && entry->end >= a->common->highaddr && 4333024e8afSRuslan Bukin iommu_gas_match_one(a, entry->end, child->first, 4343024e8afSRuslan Bukin a->domain->end)) { 4353024e8afSRuslan Bukin iommu_gas_match_insert(a); 4363024e8afSRuslan Bukin return (0); 4373024e8afSRuslan Bukin } 4383024e8afSRuslan Bukin if (child != NULL && 0 == iommu_gas_uppermatch(a, child)) 4393024e8afSRuslan Bukin return (0); 4403024e8afSRuslan Bukin return (ENOMEM); 4413024e8afSRuslan Bukin } 4423024e8afSRuslan Bukin 4433024e8afSRuslan Bukin static int 4443024e8afSRuslan Bukin iommu_gas_find_space(struct iommu_domain *domain, 4453024e8afSRuslan Bukin const struct bus_dma_tag_common *common, iommu_gaddr_t size, 4463024e8afSRuslan Bukin int offset, u_int flags, struct iommu_map_entry *entry) 4473024e8afSRuslan Bukin { 4483024e8afSRuslan Bukin struct iommu_gas_match_args a; 4493024e8afSRuslan Bukin int error; 4503024e8afSRuslan Bukin 4513024e8afSRuslan Bukin IOMMU_DOMAIN_ASSERT_LOCKED(domain); 4523024e8afSRuslan Bukin KASSERT(entry->flags == 0, ("dirty entry %p %p", domain, entry)); 4533024e8afSRuslan Bukin KASSERT((size & IOMMU_PAGE_MASK) == 0, ("size %jx", (uintmax_t)size)); 4543024e8afSRuslan Bukin 4553024e8afSRuslan Bukin a.domain = domain; 4563024e8afSRuslan Bukin a.size = size; 4573024e8afSRuslan Bukin a.offset = offset; 4583024e8afSRuslan Bukin a.common = common; 4593024e8afSRuslan Bukin a.gas_flags = flags; 4603024e8afSRuslan Bukin a.entry = entry; 4613024e8afSRuslan Bukin 4623024e8afSRuslan Bukin /* Handle lower region. */ 4633024e8afSRuslan Bukin if (common->lowaddr > 0) { 4643024e8afSRuslan Bukin error = iommu_gas_lowermatch(&a, 4653024e8afSRuslan Bukin RB_ROOT(&domain->rb_root)); 4663024e8afSRuslan Bukin if (error == 0) 4673024e8afSRuslan Bukin return (0); 4683024e8afSRuslan Bukin KASSERT(error == ENOMEM, 4693024e8afSRuslan Bukin ("error %d from iommu_gas_lowermatch", error)); 4703024e8afSRuslan Bukin } 4713024e8afSRuslan Bukin /* Handle upper region. */ 4723024e8afSRuslan Bukin if (common->highaddr >= domain->end) 4733024e8afSRuslan Bukin return (ENOMEM); 4743024e8afSRuslan Bukin error = iommu_gas_uppermatch(&a, RB_ROOT(&domain->rb_root)); 4753024e8afSRuslan Bukin KASSERT(error == ENOMEM, 4763024e8afSRuslan Bukin ("error %d from iommu_gas_uppermatch", error)); 4773024e8afSRuslan Bukin return (error); 4783024e8afSRuslan Bukin } 4793024e8afSRuslan Bukin 4803024e8afSRuslan Bukin static int 4813024e8afSRuslan Bukin iommu_gas_alloc_region(struct iommu_domain *domain, struct iommu_map_entry *entry, 4823024e8afSRuslan Bukin u_int flags) 4833024e8afSRuslan Bukin { 4843024e8afSRuslan Bukin struct iommu_map_entry *next, *prev; 485661bd70bSKonstantin Belousov bool found __diagused; 4863024e8afSRuslan Bukin 4873024e8afSRuslan Bukin IOMMU_DOMAIN_ASSERT_LOCKED(domain); 4883024e8afSRuslan Bukin 4893024e8afSRuslan Bukin if ((entry->start & IOMMU_PAGE_MASK) != 0 || 4903024e8afSRuslan Bukin (entry->end & IOMMU_PAGE_MASK) != 0) 4913024e8afSRuslan Bukin return (EINVAL); 4923024e8afSRuslan Bukin if (entry->start >= entry->end) 4933024e8afSRuslan Bukin return (EINVAL); 4943024e8afSRuslan Bukin if (entry->end >= domain->end) 4953024e8afSRuslan Bukin return (EINVAL); 4963024e8afSRuslan Bukin 4973024e8afSRuslan Bukin next = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, entry); 4983024e8afSRuslan Bukin KASSERT(next != NULL, ("next must be non-null %p %jx", domain, 4993024e8afSRuslan Bukin (uintmax_t)entry->start)); 5003024e8afSRuslan Bukin prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, next); 5013024e8afSRuslan Bukin /* prev could be NULL */ 5023024e8afSRuslan Bukin 5033024e8afSRuslan Bukin /* 5043024e8afSRuslan Bukin * Adapt to broken BIOSes which specify overlapping RMRR 5053024e8afSRuslan Bukin * entries. 5063024e8afSRuslan Bukin * 5073024e8afSRuslan Bukin * XXXKIB: this does not handle a case when prev or next 5083024e8afSRuslan Bukin * entries are completely covered by the current one, which 5093024e8afSRuslan Bukin * extends both ways. 5103024e8afSRuslan Bukin */ 5113024e8afSRuslan Bukin if (prev != NULL && prev->end > entry->start && 5123024e8afSRuslan Bukin (prev->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { 5133024e8afSRuslan Bukin if ((flags & IOMMU_MF_RMRR) == 0 || 5143024e8afSRuslan Bukin (prev->flags & IOMMU_MAP_ENTRY_RMRR) == 0) 5153024e8afSRuslan Bukin return (EBUSY); 5163024e8afSRuslan Bukin entry->start = prev->end; 5173024e8afSRuslan Bukin } 5183024e8afSRuslan Bukin if (next->start < entry->end && 5193024e8afSRuslan Bukin (next->flags & IOMMU_MAP_ENTRY_PLACE) == 0) { 5203024e8afSRuslan Bukin if ((flags & IOMMU_MF_RMRR) == 0 || 5213024e8afSRuslan Bukin (next->flags & IOMMU_MAP_ENTRY_RMRR) == 0) 5223024e8afSRuslan Bukin return (EBUSY); 5233024e8afSRuslan Bukin entry->end = next->start; 5243024e8afSRuslan Bukin } 5253024e8afSRuslan Bukin if (entry->end == entry->start) 5263024e8afSRuslan Bukin return (0); 5273024e8afSRuslan Bukin 5283024e8afSRuslan Bukin if (prev != NULL && prev->end > entry->start) { 5293024e8afSRuslan Bukin /* This assumes that prev is the placeholder entry. */ 5303024e8afSRuslan Bukin iommu_gas_rb_remove(domain, prev); 5313024e8afSRuslan Bukin prev = NULL; 5323024e8afSRuslan Bukin } 5333024e8afSRuslan Bukin if (next->start < entry->end) { 5343024e8afSRuslan Bukin iommu_gas_rb_remove(domain, next); 5353024e8afSRuslan Bukin next = NULL; 5363024e8afSRuslan Bukin } 5373024e8afSRuslan Bukin 5383024e8afSRuslan Bukin found = iommu_gas_rb_insert(domain, entry); 5393024e8afSRuslan Bukin KASSERT(found, ("found RMRR dup %p start %jx end %jx", 5403024e8afSRuslan Bukin domain, (uintmax_t)entry->start, (uintmax_t)entry->end)); 5413024e8afSRuslan Bukin if ((flags & IOMMU_MF_RMRR) != 0) 5423024e8afSRuslan Bukin entry->flags = IOMMU_MAP_ENTRY_RMRR; 5433024e8afSRuslan Bukin 5443024e8afSRuslan Bukin #ifdef INVARIANTS 5453024e8afSRuslan Bukin struct iommu_map_entry *ip, *in; 5463024e8afSRuslan Bukin ip = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry); 5473024e8afSRuslan Bukin in = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry); 5483024e8afSRuslan Bukin KASSERT(prev == NULL || ip == prev, 5493024e8afSRuslan Bukin ("RMRR %p (%jx %jx) prev %p (%jx %jx) ins prev %p (%jx %jx)", 5503024e8afSRuslan Bukin entry, entry->start, entry->end, prev, 5513024e8afSRuslan Bukin prev == NULL ? 0 : prev->start, prev == NULL ? 0 : prev->end, 5523024e8afSRuslan Bukin ip, ip == NULL ? 0 : ip->start, ip == NULL ? 0 : ip->end)); 5533024e8afSRuslan Bukin KASSERT(next == NULL || in == next, 5543024e8afSRuslan Bukin ("RMRR %p (%jx %jx) next %p (%jx %jx) ins next %p (%jx %jx)", 5553024e8afSRuslan Bukin entry, entry->start, entry->end, next, 5563024e8afSRuslan Bukin next == NULL ? 0 : next->start, next == NULL ? 0 : next->end, 5573024e8afSRuslan Bukin in, in == NULL ? 0 : in->start, in == NULL ? 0 : in->end)); 5583024e8afSRuslan Bukin #endif 5593024e8afSRuslan Bukin 5603024e8afSRuslan Bukin return (0); 5613024e8afSRuslan Bukin } 5623024e8afSRuslan Bukin 5633024e8afSRuslan Bukin void 5643024e8afSRuslan Bukin iommu_gas_free_space(struct iommu_domain *domain, struct iommu_map_entry *entry) 5653024e8afSRuslan Bukin { 5663024e8afSRuslan Bukin 5673024e8afSRuslan Bukin IOMMU_DOMAIN_ASSERT_LOCKED(domain); 5683024e8afSRuslan Bukin KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | 5693024e8afSRuslan Bukin IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_MAP, 5703024e8afSRuslan Bukin ("permanent entry %p %p", domain, entry)); 5713024e8afSRuslan Bukin 5723024e8afSRuslan Bukin iommu_gas_rb_remove(domain, entry); 5733024e8afSRuslan Bukin entry->flags &= ~IOMMU_MAP_ENTRY_MAP; 5743024e8afSRuslan Bukin #ifdef INVARIANTS 5753024e8afSRuslan Bukin if (iommu_check_free) 5763024e8afSRuslan Bukin iommu_gas_check_free(domain); 5773024e8afSRuslan Bukin #endif 5783024e8afSRuslan Bukin } 5793024e8afSRuslan Bukin 5803024e8afSRuslan Bukin void 5813024e8afSRuslan Bukin iommu_gas_free_region(struct iommu_domain *domain, struct iommu_map_entry *entry) 5823024e8afSRuslan Bukin { 5833024e8afSRuslan Bukin struct iommu_map_entry *next, *prev; 5843024e8afSRuslan Bukin 5853024e8afSRuslan Bukin IOMMU_DOMAIN_ASSERT_LOCKED(domain); 5863024e8afSRuslan Bukin KASSERT((entry->flags & (IOMMU_MAP_ENTRY_PLACE | IOMMU_MAP_ENTRY_RMRR | 5873024e8afSRuslan Bukin IOMMU_MAP_ENTRY_MAP)) == IOMMU_MAP_ENTRY_RMRR, 5883024e8afSRuslan Bukin ("non-RMRR entry %p %p", domain, entry)); 5893024e8afSRuslan Bukin 5903024e8afSRuslan Bukin prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, entry); 5913024e8afSRuslan Bukin next = RB_NEXT(iommu_gas_entries_tree, &domain->rb_root, entry); 5923024e8afSRuslan Bukin iommu_gas_rb_remove(domain, entry); 5933024e8afSRuslan Bukin entry->flags &= ~IOMMU_MAP_ENTRY_RMRR; 5943024e8afSRuslan Bukin 5953024e8afSRuslan Bukin if (prev == NULL) 5963024e8afSRuslan Bukin iommu_gas_rb_insert(domain, domain->first_place); 5973024e8afSRuslan Bukin if (next == NULL) 5983024e8afSRuslan Bukin iommu_gas_rb_insert(domain, domain->last_place); 5993024e8afSRuslan Bukin } 6003024e8afSRuslan Bukin 6013024e8afSRuslan Bukin int 6023024e8afSRuslan Bukin iommu_gas_map(struct iommu_domain *domain, 6033024e8afSRuslan Bukin const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset, 6043024e8afSRuslan Bukin u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res) 6053024e8afSRuslan Bukin { 6063024e8afSRuslan Bukin struct iommu_map_entry *entry; 6073024e8afSRuslan Bukin int error; 6083024e8afSRuslan Bukin 6093024e8afSRuslan Bukin KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_CANSPLIT)) == 0, 6103024e8afSRuslan Bukin ("invalid flags 0x%x", flags)); 6113024e8afSRuslan Bukin 6123024e8afSRuslan Bukin entry = iommu_gas_alloc_entry(domain, 61315f6baf4SRuslan Bukin (flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0); 6143024e8afSRuslan Bukin if (entry == NULL) 6153024e8afSRuslan Bukin return (ENOMEM); 6163024e8afSRuslan Bukin IOMMU_DOMAIN_LOCK(domain); 6173024e8afSRuslan Bukin error = iommu_gas_find_space(domain, common, size, offset, flags, 6183024e8afSRuslan Bukin entry); 6193024e8afSRuslan Bukin if (error == ENOMEM) { 6203024e8afSRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 6213024e8afSRuslan Bukin iommu_gas_free_entry(domain, entry); 6223024e8afSRuslan Bukin return (error); 6233024e8afSRuslan Bukin } 6243024e8afSRuslan Bukin #ifdef INVARIANTS 6253024e8afSRuslan Bukin if (iommu_check_free) 6263024e8afSRuslan Bukin iommu_gas_check_free(domain); 6273024e8afSRuslan Bukin #endif 6283024e8afSRuslan Bukin KASSERT(error == 0, 6293024e8afSRuslan Bukin ("unexpected error %d from iommu_gas_find_entry", error)); 6303024e8afSRuslan Bukin KASSERT(entry->end < domain->end, ("allocated GPA %jx, max GPA %jx", 6313024e8afSRuslan Bukin (uintmax_t)entry->end, (uintmax_t)domain->end)); 6323024e8afSRuslan Bukin entry->flags |= eflags; 6333024e8afSRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 6343024e8afSRuslan Bukin 6350eed04c8SRuslan Bukin error = domain->ops->map(domain, entry->start, 6360eed04c8SRuslan Bukin entry->end - entry->start, ma, eflags, 63715f6baf4SRuslan Bukin ((flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0)); 6383024e8afSRuslan Bukin if (error == ENOMEM) { 6393024e8afSRuslan Bukin iommu_domain_unload_entry(entry, true); 6403024e8afSRuslan Bukin return (error); 6413024e8afSRuslan Bukin } 6423024e8afSRuslan Bukin KASSERT(error == 0, 6433024e8afSRuslan Bukin ("unexpected error %d from domain_map_buf", error)); 6443024e8afSRuslan Bukin 6453024e8afSRuslan Bukin *res = entry; 6463024e8afSRuslan Bukin return (0); 6473024e8afSRuslan Bukin } 6483024e8afSRuslan Bukin 6493024e8afSRuslan Bukin int 6503024e8afSRuslan Bukin iommu_gas_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, 6513024e8afSRuslan Bukin u_int eflags, u_int flags, vm_page_t *ma) 6523024e8afSRuslan Bukin { 6533024e8afSRuslan Bukin iommu_gaddr_t start; 6543024e8afSRuslan Bukin int error; 6553024e8afSRuslan Bukin 6563024e8afSRuslan Bukin KASSERT(entry->flags == 0, ("used RMRR entry %p %p %x", domain, 6573024e8afSRuslan Bukin entry, entry->flags)); 6583024e8afSRuslan Bukin KASSERT((flags & ~(IOMMU_MF_CANWAIT | IOMMU_MF_RMRR)) == 0, 6593024e8afSRuslan Bukin ("invalid flags 0x%x", flags)); 6603024e8afSRuslan Bukin 6613024e8afSRuslan Bukin start = entry->start; 6623024e8afSRuslan Bukin IOMMU_DOMAIN_LOCK(domain); 6633024e8afSRuslan Bukin error = iommu_gas_alloc_region(domain, entry, flags); 6643024e8afSRuslan Bukin if (error != 0) { 6653024e8afSRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 6663024e8afSRuslan Bukin return (error); 6673024e8afSRuslan Bukin } 6683024e8afSRuslan Bukin entry->flags |= eflags; 6693024e8afSRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 6703024e8afSRuslan Bukin if (entry->end == entry->start) 6713024e8afSRuslan Bukin return (0); 6723024e8afSRuslan Bukin 6730eed04c8SRuslan Bukin error = domain->ops->map(domain, entry->start, 6740eed04c8SRuslan Bukin entry->end - entry->start, ma + OFF_TO_IDX(start - entry->start), 6750eed04c8SRuslan Bukin eflags, ((flags & IOMMU_MF_CANWAIT) != 0 ? IOMMU_PGF_WAITOK : 0)); 6763024e8afSRuslan Bukin if (error == ENOMEM) { 6773024e8afSRuslan Bukin iommu_domain_unload_entry(entry, false); 6783024e8afSRuslan Bukin return (error); 6793024e8afSRuslan Bukin } 6803024e8afSRuslan Bukin KASSERT(error == 0, 6813024e8afSRuslan Bukin ("unexpected error %d from domain_map_buf", error)); 6823024e8afSRuslan Bukin 6833024e8afSRuslan Bukin return (0); 6843024e8afSRuslan Bukin } 6853024e8afSRuslan Bukin 686ee47a12aSRyan Libby static int 687ee47a12aSRyan Libby iommu_gas_reserve_region_locked(struct iommu_domain *domain, 688ee47a12aSRyan Libby iommu_gaddr_t start, iommu_gaddr_t end, struct iommu_map_entry *entry) 689ee47a12aSRyan Libby { 690ee47a12aSRyan Libby int error; 691ee47a12aSRyan Libby 692ee47a12aSRyan Libby IOMMU_DOMAIN_ASSERT_LOCKED(domain); 693ee47a12aSRyan Libby 694ee47a12aSRyan Libby entry->start = start; 695ee47a12aSRyan Libby entry->end = end; 696ee47a12aSRyan Libby error = iommu_gas_alloc_region(domain, entry, IOMMU_MF_CANWAIT); 697ee47a12aSRyan Libby if (error == 0) 698ee47a12aSRyan Libby entry->flags |= IOMMU_MAP_ENTRY_UNMAPPED; 699ee47a12aSRyan Libby return (error); 700ee47a12aSRyan Libby } 701ee47a12aSRyan Libby 7023024e8afSRuslan Bukin int 7033024e8afSRuslan Bukin iommu_gas_reserve_region(struct iommu_domain *domain, iommu_gaddr_t start, 70494dfb28eSRuslan Bukin iommu_gaddr_t end, struct iommu_map_entry **entry0) 7053024e8afSRuslan Bukin { 7063024e8afSRuslan Bukin struct iommu_map_entry *entry; 7073024e8afSRuslan Bukin int error; 7083024e8afSRuslan Bukin 70915f6baf4SRuslan Bukin entry = iommu_gas_alloc_entry(domain, IOMMU_PGF_WAITOK); 7103024e8afSRuslan Bukin IOMMU_DOMAIN_LOCK(domain); 711ee47a12aSRyan Libby error = iommu_gas_reserve_region_locked(domain, start, end, entry); 7123024e8afSRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 7133024e8afSRuslan Bukin if (error != 0) 7143024e8afSRuslan Bukin iommu_gas_free_entry(domain, entry); 71594dfb28eSRuslan Bukin else if (entry0 != NULL) 71694dfb28eSRuslan Bukin *entry0 = entry; 7173024e8afSRuslan Bukin return (error); 7183024e8afSRuslan Bukin } 7193024e8afSRuslan Bukin 720ee47a12aSRyan Libby /* 721ee47a12aSRyan Libby * As in iommu_gas_reserve_region, reserve [start, end), but allow for existing 722ee47a12aSRyan Libby * entries. 723ee47a12aSRyan Libby */ 724ee47a12aSRyan Libby int 725ee47a12aSRyan Libby iommu_gas_reserve_region_extend(struct iommu_domain *domain, 726ee47a12aSRyan Libby iommu_gaddr_t start, iommu_gaddr_t end) 727ee47a12aSRyan Libby { 728ee47a12aSRyan Libby struct iommu_map_entry *entry, *next, *prev, key = {}; 729ee47a12aSRyan Libby iommu_gaddr_t entry_start, entry_end; 730ee47a12aSRyan Libby int error; 731ee47a12aSRyan Libby 732ee47a12aSRyan Libby error = 0; 733ee47a12aSRyan Libby entry = NULL; 734ee47a12aSRyan Libby end = ummin(end, domain->end); 735ee47a12aSRyan Libby while (start < end) { 736ee47a12aSRyan Libby /* Preallocate an entry. */ 737ee47a12aSRyan Libby if (entry == NULL) 738ee47a12aSRyan Libby entry = iommu_gas_alloc_entry(domain, 739ee47a12aSRyan Libby IOMMU_PGF_WAITOK); 740ee47a12aSRyan Libby /* Calculate the free region from here to the next entry. */ 741ee47a12aSRyan Libby key.start = key.end = start; 742ee47a12aSRyan Libby IOMMU_DOMAIN_LOCK(domain); 743ee47a12aSRyan Libby next = RB_NFIND(iommu_gas_entries_tree, &domain->rb_root, &key); 744ee47a12aSRyan Libby KASSERT(next != NULL, ("domain %p with end %#jx has no entry " 745ee47a12aSRyan Libby "after %#jx", domain, (uintmax_t)domain->end, 746ee47a12aSRyan Libby (uintmax_t)start)); 747ee47a12aSRyan Libby entry_end = ummin(end, next->start); 748ee47a12aSRyan Libby prev = RB_PREV(iommu_gas_entries_tree, &domain->rb_root, next); 749ee47a12aSRyan Libby if (prev != NULL) 750ee47a12aSRyan Libby entry_start = ummax(start, prev->end); 751ee47a12aSRyan Libby else 752ee47a12aSRyan Libby entry_start = start; 753ee47a12aSRyan Libby start = next->end; 754ee47a12aSRyan Libby /* Reserve the region if non-empty. */ 755ee47a12aSRyan Libby if (entry_start != entry_end) { 756ee47a12aSRyan Libby error = iommu_gas_reserve_region_locked(domain, 757ee47a12aSRyan Libby entry_start, entry_end, entry); 758ee47a12aSRyan Libby if (error != 0) 759ee47a12aSRyan Libby break; 760ee47a12aSRyan Libby entry = NULL; 761ee47a12aSRyan Libby } 762ee47a12aSRyan Libby IOMMU_DOMAIN_UNLOCK(domain); 763ee47a12aSRyan Libby } 764ee47a12aSRyan Libby /* Release a preallocated entry if it was not used. */ 765ee47a12aSRyan Libby if (entry != NULL) 766ee47a12aSRyan Libby iommu_gas_free_entry(domain, entry); 767ee47a12aSRyan Libby return (error); 768ee47a12aSRyan Libby } 769ee47a12aSRyan Libby 7703024e8afSRuslan Bukin struct iommu_map_entry * 7713024e8afSRuslan Bukin iommu_map_alloc_entry(struct iommu_domain *domain, u_int flags) 7723024e8afSRuslan Bukin { 7733024e8afSRuslan Bukin struct iommu_map_entry *res; 7743024e8afSRuslan Bukin 7753024e8afSRuslan Bukin res = iommu_gas_alloc_entry(domain, flags); 7763024e8afSRuslan Bukin 7773024e8afSRuslan Bukin return (res); 7783024e8afSRuslan Bukin } 7793024e8afSRuslan Bukin 7803024e8afSRuslan Bukin void 7813024e8afSRuslan Bukin iommu_map_free_entry(struct iommu_domain *domain, struct iommu_map_entry *entry) 7823024e8afSRuslan Bukin { 7833024e8afSRuslan Bukin 7843024e8afSRuslan Bukin iommu_gas_free_entry(domain, entry); 7853024e8afSRuslan Bukin } 7863024e8afSRuslan Bukin 7873024e8afSRuslan Bukin int 7883024e8afSRuslan Bukin iommu_map(struct iommu_domain *domain, 7893024e8afSRuslan Bukin const struct bus_dma_tag_common *common, iommu_gaddr_t size, int offset, 7903024e8afSRuslan Bukin u_int eflags, u_int flags, vm_page_t *ma, struct iommu_map_entry **res) 7913024e8afSRuslan Bukin { 7923024e8afSRuslan Bukin int error; 7933024e8afSRuslan Bukin 7943024e8afSRuslan Bukin error = iommu_gas_map(domain, common, size, offset, eflags, flags, 7953024e8afSRuslan Bukin ma, res); 7963024e8afSRuslan Bukin 7973024e8afSRuslan Bukin return (error); 7983024e8afSRuslan Bukin } 7993024e8afSRuslan Bukin 800f32f0095SRuslan Bukin void 801f32f0095SRuslan Bukin iommu_unmap_msi(struct iommu_ctx *ctx) 802f32f0095SRuslan Bukin { 803f32f0095SRuslan Bukin struct iommu_map_entry *entry; 804f32f0095SRuslan Bukin struct iommu_domain *domain; 805f32f0095SRuslan Bukin 806f32f0095SRuslan Bukin domain = ctx->domain; 807f32f0095SRuslan Bukin entry = domain->msi_entry; 808f32f0095SRuslan Bukin if (entry == NULL) 809f32f0095SRuslan Bukin return; 810f32f0095SRuslan Bukin 811f32f0095SRuslan Bukin domain->ops->unmap(domain, entry->start, entry->end - 812f32f0095SRuslan Bukin entry->start, IOMMU_PGF_WAITOK); 813f32f0095SRuslan Bukin 814f32f0095SRuslan Bukin IOMMU_DOMAIN_LOCK(domain); 815f32f0095SRuslan Bukin iommu_gas_free_space(domain, entry); 816f32f0095SRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 817f32f0095SRuslan Bukin 818f32f0095SRuslan Bukin iommu_gas_free_entry(domain, entry); 819f32f0095SRuslan Bukin 820f32f0095SRuslan Bukin domain->msi_entry = NULL; 821f32f0095SRuslan Bukin domain->msi_base = 0; 822f32f0095SRuslan Bukin domain->msi_phys = 0; 823f32f0095SRuslan Bukin } 824f32f0095SRuslan Bukin 8253024e8afSRuslan Bukin int 826e707c8beSRuslan Bukin iommu_map_msi(struct iommu_ctx *ctx, iommu_gaddr_t size, int offset, 827e707c8beSRuslan Bukin u_int eflags, u_int flags, vm_page_t *ma) 828e707c8beSRuslan Bukin { 829e707c8beSRuslan Bukin struct iommu_domain *domain; 830e707c8beSRuslan Bukin struct iommu_map_entry *entry; 831e707c8beSRuslan Bukin int error; 832e707c8beSRuslan Bukin 833e707c8beSRuslan Bukin error = 0; 834e707c8beSRuslan Bukin domain = ctx->domain; 835e707c8beSRuslan Bukin 836e707c8beSRuslan Bukin /* Check if there is already an MSI page allocated */ 837e707c8beSRuslan Bukin IOMMU_DOMAIN_LOCK(domain); 838e707c8beSRuslan Bukin entry = domain->msi_entry; 839e707c8beSRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 840e707c8beSRuslan Bukin 841e707c8beSRuslan Bukin if (entry == NULL) { 842e707c8beSRuslan Bukin error = iommu_gas_map(domain, &ctx->tag->common, size, offset, 843e707c8beSRuslan Bukin eflags, flags, ma, &entry); 844e707c8beSRuslan Bukin IOMMU_DOMAIN_LOCK(domain); 845e707c8beSRuslan Bukin if (error == 0) { 846e707c8beSRuslan Bukin if (domain->msi_entry == NULL) { 847e707c8beSRuslan Bukin MPASS(domain->msi_base == 0); 848e707c8beSRuslan Bukin MPASS(domain->msi_phys == 0); 849e707c8beSRuslan Bukin 850e707c8beSRuslan Bukin domain->msi_entry = entry; 851e707c8beSRuslan Bukin domain->msi_base = entry->start; 852e707c8beSRuslan Bukin domain->msi_phys = VM_PAGE_TO_PHYS(ma[0]); 853e707c8beSRuslan Bukin } else { 854e707c8beSRuslan Bukin /* 855e707c8beSRuslan Bukin * We lost the race and already have an 856e707c8beSRuslan Bukin * MSI page allocated. Free the unneeded entry. 857e707c8beSRuslan Bukin */ 858e707c8beSRuslan Bukin iommu_gas_free_entry(domain, entry); 859e707c8beSRuslan Bukin } 860e707c8beSRuslan Bukin } else if (domain->msi_entry != NULL) { 861e707c8beSRuslan Bukin /* 862e707c8beSRuslan Bukin * The allocation failed, but another succeeded. 863e707c8beSRuslan Bukin * Return success as there is a valid MSI page. 864e707c8beSRuslan Bukin */ 865e707c8beSRuslan Bukin error = 0; 866e707c8beSRuslan Bukin } 867e707c8beSRuslan Bukin IOMMU_DOMAIN_UNLOCK(domain); 868e707c8beSRuslan Bukin } 869e707c8beSRuslan Bukin 870e707c8beSRuslan Bukin return (error); 871e707c8beSRuslan Bukin } 872e707c8beSRuslan Bukin 873e707c8beSRuslan Bukin void 874e707c8beSRuslan Bukin iommu_translate_msi(struct iommu_domain *domain, uint64_t *addr) 875e707c8beSRuslan Bukin { 876e707c8beSRuslan Bukin 877e707c8beSRuslan Bukin *addr = (*addr - domain->msi_phys) + domain->msi_base; 878e707c8beSRuslan Bukin 879e707c8beSRuslan Bukin KASSERT(*addr >= domain->msi_entry->start, 880e707c8beSRuslan Bukin ("%s: Address is below the MSI entry start address (%jx < %jx)", 881e707c8beSRuslan Bukin __func__, (uintmax_t)*addr, (uintmax_t)domain->msi_entry->start)); 882e707c8beSRuslan Bukin 883e707c8beSRuslan Bukin KASSERT(*addr + sizeof(*addr) <= domain->msi_entry->end, 884e707c8beSRuslan Bukin ("%s: Address is above the MSI entry end address (%jx < %jx)", 885e707c8beSRuslan Bukin __func__, (uintmax_t)*addr, (uintmax_t)domain->msi_entry->end)); 886e707c8beSRuslan Bukin } 887e707c8beSRuslan Bukin 888e707c8beSRuslan Bukin int 8893024e8afSRuslan Bukin iommu_map_region(struct iommu_domain *domain, struct iommu_map_entry *entry, 8903024e8afSRuslan Bukin u_int eflags, u_int flags, vm_page_t *ma) 8913024e8afSRuslan Bukin { 8923024e8afSRuslan Bukin int error; 8933024e8afSRuslan Bukin 8943024e8afSRuslan Bukin error = iommu_gas_map_region(domain, entry, eflags, flags, ma); 8953024e8afSRuslan Bukin 8963024e8afSRuslan Bukin return (error); 8973024e8afSRuslan Bukin } 8989c843a40SRuslan Bukin 899357149f0SRuslan Bukin SYSCTL_NODE(_hw, OID_AUTO, iommu, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, ""); 900357149f0SRuslan Bukin 9019c843a40SRuslan Bukin #ifdef INVARIANTS 9029c843a40SRuslan Bukin SYSCTL_INT(_hw_iommu, OID_AUTO, check_free, CTLFLAG_RWTUN, 9039c843a40SRuslan Bukin &iommu_check_free, 0, 9049c843a40SRuslan Bukin "Check the GPA RBtree for free_down and free_after validity"); 9059c843a40SRuslan Bukin #endif 906