16db7f8e5SKonstantin Belousov /*- 26db7f8e5SKonstantin Belousov * Copyright (c) 2017, 2018 The FreeBSD Foundation 36db7f8e5SKonstantin Belousov * All rights reserved. 46db7f8e5SKonstantin Belousov * 56db7f8e5SKonstantin Belousov * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 66db7f8e5SKonstantin Belousov * under sponsorship from the FreeBSD Foundation. 76db7f8e5SKonstantin Belousov * 86db7f8e5SKonstantin Belousov * Redistribution and use in source and binary forms, with or without 96db7f8e5SKonstantin Belousov * modification, are permitted provided that the following conditions 106db7f8e5SKonstantin Belousov * are met: 116db7f8e5SKonstantin Belousov * 1. Redistributions of source code must retain the above copyright 126db7f8e5SKonstantin Belousov * notice, this list of conditions and the following disclaimer. 136db7f8e5SKonstantin Belousov * 2. Redistributions in binary form must reproduce the above copyright 146db7f8e5SKonstantin Belousov * notice, this list of conditions and the following disclaimer in the 156db7f8e5SKonstantin Belousov * documentation and/or other materials provided with the distribution. 166db7f8e5SKonstantin Belousov * 176db7f8e5SKonstantin Belousov * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 186db7f8e5SKonstantin Belousov * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 196db7f8e5SKonstantin Belousov * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 206db7f8e5SKonstantin Belousov * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 216db7f8e5SKonstantin Belousov * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 226db7f8e5SKonstantin Belousov * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 236db7f8e5SKonstantin Belousov * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 246db7f8e5SKonstantin Belousov * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 256db7f8e5SKonstantin Belousov * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 266db7f8e5SKonstantin Belousov * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 276db7f8e5SKonstantin Belousov * SUCH DAMAGE. 286db7f8e5SKonstantin Belousov */ 296db7f8e5SKonstantin Belousov 306db7f8e5SKonstantin Belousov #include <sys/cdefs.h> 316db7f8e5SKonstantin Belousov __FBSDID("$FreeBSD$"); 326db7f8e5SKonstantin Belousov 336db7f8e5SKonstantin Belousov #include "opt_acpi.h" 346db7f8e5SKonstantin Belousov #include "opt_ddb.h" 356db7f8e5SKonstantin Belousov 366db7f8e5SKonstantin Belousov #include <sys/param.h> 376db7f8e5SKonstantin Belousov #include <sys/systm.h> 386db7f8e5SKonstantin Belousov #include <sys/bio.h> 396db7f8e5SKonstantin Belousov #include <sys/bus.h> 406db7f8e5SKonstantin Belousov #include <sys/conf.h> 416db7f8e5SKonstantin Belousov #include <sys/devicestat.h> 426db7f8e5SKonstantin Belousov #include <sys/disk.h> 436db7f8e5SKonstantin Belousov #include <sys/efi.h> 446db7f8e5SKonstantin Belousov #include <sys/kernel.h> 456db7f8e5SKonstantin Belousov #include <sys/kthread.h> 466db7f8e5SKonstantin Belousov #include <sys/limits.h> 476db7f8e5SKonstantin Belousov #include <sys/lock.h> 486db7f8e5SKonstantin Belousov #include <sys/malloc.h> 496db7f8e5SKonstantin Belousov #include <sys/module.h> 506db7f8e5SKonstantin Belousov #include <sys/rwlock.h> 516db7f8e5SKonstantin Belousov #include <sys/sglist.h> 526db7f8e5SKonstantin Belousov #include <sys/uio.h> 536db7f8e5SKonstantin Belousov #include <sys/uuid.h> 546db7f8e5SKonstantin Belousov #include <geom/geom.h> 556db7f8e5SKonstantin Belousov #include <geom/geom_int.h> 566db7f8e5SKonstantin Belousov #include <machine/vmparam.h> 576db7f8e5SKonstantin Belousov #include <vm/vm.h> 586db7f8e5SKonstantin Belousov #include <vm/vm_object.h> 596db7f8e5SKonstantin Belousov #include <vm/vm_page.h> 606db7f8e5SKonstantin Belousov #include <vm/vm_pager.h> 616db7f8e5SKonstantin Belousov #include <contrib/dev/acpica/include/acpi.h> 626db7f8e5SKonstantin Belousov #include <contrib/dev/acpica/include/accommon.h> 636db7f8e5SKonstantin Belousov #include <contrib/dev/acpica/include/acuuid.h> 646db7f8e5SKonstantin Belousov #include <dev/acpica/acpivar.h> 656db7f8e5SKonstantin Belousov #include <dev/nvdimm/nvdimm_var.h> 666db7f8e5SKonstantin Belousov 676db7f8e5SKonstantin Belousov struct SPA_mapping *spa_mappings; 686db7f8e5SKonstantin Belousov int spa_mappings_cnt; 696db7f8e5SKonstantin Belousov 706db7f8e5SKonstantin Belousov static int 716db7f8e5SKonstantin Belousov nvdimm_spa_count(void *nfitsubtbl __unused, void *arg) 726db7f8e5SKonstantin Belousov { 736db7f8e5SKonstantin Belousov int *cnt; 746db7f8e5SKonstantin Belousov 756db7f8e5SKonstantin Belousov cnt = arg; 766db7f8e5SKonstantin Belousov (*cnt)++; 776db7f8e5SKonstantin Belousov return (0); 786db7f8e5SKonstantin Belousov } 796db7f8e5SKonstantin Belousov 806db7f8e5SKonstantin Belousov static struct nvdimm_SPA_uuid_list_elm { 816db7f8e5SKonstantin Belousov const char *u_name; 826db7f8e5SKonstantin Belousov const char *u_id_str; 836db7f8e5SKonstantin Belousov struct uuid u_id; 846db7f8e5SKonstantin Belousov const bool u_usr_acc; 856db7f8e5SKonstantin Belousov } nvdimm_SPA_uuid_list[] = { 866db7f8e5SKonstantin Belousov [SPA_TYPE_VOLATILE_MEMORY] = { 876db7f8e5SKonstantin Belousov .u_name = "VOLA MEM ", 886db7f8e5SKonstantin Belousov .u_id_str = UUID_VOLATILE_MEMORY, 896db7f8e5SKonstantin Belousov .u_usr_acc = true, 906db7f8e5SKonstantin Belousov }, 916db7f8e5SKonstantin Belousov [SPA_TYPE_PERSISTENT_MEMORY] = { 926db7f8e5SKonstantin Belousov .u_name = "PERS MEM", 936db7f8e5SKonstantin Belousov .u_id_str = UUID_PERSISTENT_MEMORY, 946db7f8e5SKonstantin Belousov .u_usr_acc = true, 956db7f8e5SKonstantin Belousov }, 966db7f8e5SKonstantin Belousov [SPA_TYPE_CONTROL_REGION] = { 976db7f8e5SKonstantin Belousov .u_name = "CTRL RG ", 986db7f8e5SKonstantin Belousov .u_id_str = UUID_CONTROL_REGION, 996db7f8e5SKonstantin Belousov .u_usr_acc = false, 1006db7f8e5SKonstantin Belousov }, 1016db7f8e5SKonstantin Belousov [SPA_TYPE_DATA_REGION] = { 1026db7f8e5SKonstantin Belousov .u_name = "DATA RG ", 1036db7f8e5SKonstantin Belousov .u_id_str = UUID_DATA_REGION, 1046db7f8e5SKonstantin Belousov .u_usr_acc = true, 1056db7f8e5SKonstantin Belousov }, 1066db7f8e5SKonstantin Belousov [SPA_TYPE_VOLATILE_VIRTUAL_DISK] = { 1076db7f8e5SKonstantin Belousov .u_name = "VIRT DSK", 1086db7f8e5SKonstantin Belousov .u_id_str = UUID_VOLATILE_VIRTUAL_DISK, 1096db7f8e5SKonstantin Belousov .u_usr_acc = true, 1106db7f8e5SKonstantin Belousov }, 1116db7f8e5SKonstantin Belousov [SPA_TYPE_VOLATILE_VIRTUAL_CD] = { 1126db7f8e5SKonstantin Belousov .u_name = "VIRT CD ", 1136db7f8e5SKonstantin Belousov .u_id_str = UUID_VOLATILE_VIRTUAL_CD, 1146db7f8e5SKonstantin Belousov .u_usr_acc = true, 1156db7f8e5SKonstantin Belousov }, 1166db7f8e5SKonstantin Belousov [SPA_TYPE_PERSISTENT_VIRTUAL_DISK] = { 1176db7f8e5SKonstantin Belousov .u_name = "PV DSK ", 1186db7f8e5SKonstantin Belousov .u_id_str = UUID_PERSISTENT_VIRTUAL_DISK, 1196db7f8e5SKonstantin Belousov .u_usr_acc = true, 1206db7f8e5SKonstantin Belousov }, 1216db7f8e5SKonstantin Belousov [SPA_TYPE_PERSISTENT_VIRTUAL_CD] = { 1226db7f8e5SKonstantin Belousov .u_name = "PV CD ", 1236db7f8e5SKonstantin Belousov .u_id_str = UUID_PERSISTENT_VIRTUAL_CD, 1246db7f8e5SKonstantin Belousov .u_usr_acc = true, 1256db7f8e5SKonstantin Belousov }, 1266db7f8e5SKonstantin Belousov }; 1276db7f8e5SKonstantin Belousov 1286db7f8e5SKonstantin Belousov static vm_memattr_t 1296db7f8e5SKonstantin Belousov nvdimm_spa_memattr(struct SPA_mapping *spa) 1306db7f8e5SKonstantin Belousov { 1316db7f8e5SKonstantin Belousov vm_memattr_t mode; 1326db7f8e5SKonstantin Belousov 1336db7f8e5SKonstantin Belousov if ((spa->spa_efi_mem_flags & EFI_MD_ATTR_WB) != 0) 1346db7f8e5SKonstantin Belousov mode = VM_MEMATTR_WRITE_BACK; 1356db7f8e5SKonstantin Belousov else if ((spa->spa_efi_mem_flags & EFI_MD_ATTR_WT) != 0) 1366db7f8e5SKonstantin Belousov mode = VM_MEMATTR_WRITE_THROUGH; 1376db7f8e5SKonstantin Belousov else if ((spa->spa_efi_mem_flags & EFI_MD_ATTR_WC) != 0) 1386db7f8e5SKonstantin Belousov mode = VM_MEMATTR_WRITE_COMBINING; 1396db7f8e5SKonstantin Belousov else if ((spa->spa_efi_mem_flags & EFI_MD_ATTR_WP) != 0) 1406db7f8e5SKonstantin Belousov mode = VM_MEMATTR_WRITE_PROTECTED; 1416db7f8e5SKonstantin Belousov else if ((spa->spa_efi_mem_flags & EFI_MD_ATTR_UC) != 0) 1426db7f8e5SKonstantin Belousov mode = VM_MEMATTR_UNCACHEABLE; 1436db7f8e5SKonstantin Belousov else { 1446db7f8e5SKonstantin Belousov if (bootverbose) 1456db7f8e5SKonstantin Belousov printf("SPA%d mapping attr unsupported\n", 1466db7f8e5SKonstantin Belousov spa->spa_nfit_idx); 1476db7f8e5SKonstantin Belousov mode = VM_MEMATTR_UNCACHEABLE; 1486db7f8e5SKonstantin Belousov } 1496db7f8e5SKonstantin Belousov return (mode); 1506db7f8e5SKonstantin Belousov } 1516db7f8e5SKonstantin Belousov 1526db7f8e5SKonstantin Belousov static int 1536db7f8e5SKonstantin Belousov nvdimm_spa_uio(struct SPA_mapping *spa, struct uio *uio) 1546db7f8e5SKonstantin Belousov { 1556db7f8e5SKonstantin Belousov struct vm_page m, *ma; 1566db7f8e5SKonstantin Belousov off_t off; 1576db7f8e5SKonstantin Belousov vm_memattr_t mattr; 1586db7f8e5SKonstantin Belousov int error, n; 1596db7f8e5SKonstantin Belousov 16090a38351SKonstantin Belousov error = 0; 1616db7f8e5SKonstantin Belousov if (spa->spa_kva == NULL) { 1626db7f8e5SKonstantin Belousov mattr = nvdimm_spa_memattr(spa); 1636db7f8e5SKonstantin Belousov vm_page_initfake(&m, 0, mattr); 1646db7f8e5SKonstantin Belousov ma = &m; 1656db7f8e5SKonstantin Belousov while (uio->uio_resid > 0) { 1666db7f8e5SKonstantin Belousov if (uio->uio_offset >= spa->spa_len) 1676db7f8e5SKonstantin Belousov break; 1686db7f8e5SKonstantin Belousov off = spa->spa_phys_base + uio->uio_offset; 1696db7f8e5SKonstantin Belousov vm_page_updatefake(&m, trunc_page(off), mattr); 1706db7f8e5SKonstantin Belousov n = PAGE_SIZE; 1716db7f8e5SKonstantin Belousov if (n > uio->uio_resid) 1726db7f8e5SKonstantin Belousov n = uio->uio_resid; 1736db7f8e5SKonstantin Belousov error = uiomove_fromphys(&ma, off & PAGE_MASK, n, uio); 1746db7f8e5SKonstantin Belousov if (error != 0) 1756db7f8e5SKonstantin Belousov break; 1766db7f8e5SKonstantin Belousov } 1776db7f8e5SKonstantin Belousov } else { 1786db7f8e5SKonstantin Belousov while (uio->uio_resid > 0) { 1796db7f8e5SKonstantin Belousov if (uio->uio_offset >= spa->spa_len) 1806db7f8e5SKonstantin Belousov break; 1816db7f8e5SKonstantin Belousov n = INT_MAX; 1826db7f8e5SKonstantin Belousov if (n > uio->uio_resid) 1836db7f8e5SKonstantin Belousov n = uio->uio_resid; 1846db7f8e5SKonstantin Belousov if (uio->uio_offset + n > spa->spa_len) 1856db7f8e5SKonstantin Belousov n = spa->spa_len - uio->uio_offset; 1866db7f8e5SKonstantin Belousov error = uiomove((char *)spa->spa_kva + uio->uio_offset, 1876db7f8e5SKonstantin Belousov n, uio); 1886db7f8e5SKonstantin Belousov if (error != 0) 1896db7f8e5SKonstantin Belousov break; 1906db7f8e5SKonstantin Belousov } 1916db7f8e5SKonstantin Belousov } 1926db7f8e5SKonstantin Belousov return (error); 1936db7f8e5SKonstantin Belousov } 1946db7f8e5SKonstantin Belousov 1956db7f8e5SKonstantin Belousov static int 1966db7f8e5SKonstantin Belousov nvdimm_spa_rw(struct cdev *dev, struct uio *uio, int ioflag) 1976db7f8e5SKonstantin Belousov { 1986db7f8e5SKonstantin Belousov 1996db7f8e5SKonstantin Belousov return (nvdimm_spa_uio(dev->si_drv1, uio)); 2006db7f8e5SKonstantin Belousov } 2016db7f8e5SKonstantin Belousov 2026db7f8e5SKonstantin Belousov static int 2036db7f8e5SKonstantin Belousov nvdimm_spa_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, 2046db7f8e5SKonstantin Belousov struct thread *td) 2056db7f8e5SKonstantin Belousov { 2066db7f8e5SKonstantin Belousov struct SPA_mapping *spa; 2076db7f8e5SKonstantin Belousov int error; 2086db7f8e5SKonstantin Belousov 2096db7f8e5SKonstantin Belousov spa = dev->si_drv1; 2106db7f8e5SKonstantin Belousov error = 0; 2116db7f8e5SKonstantin Belousov switch (cmd) { 2126db7f8e5SKonstantin Belousov case DIOCGSECTORSIZE: 2136db7f8e5SKonstantin Belousov *(u_int *)data = DEV_BSIZE; 2146db7f8e5SKonstantin Belousov break; 2156db7f8e5SKonstantin Belousov case DIOCGMEDIASIZE: 2166db7f8e5SKonstantin Belousov *(off_t *)data = spa->spa_len; 2176db7f8e5SKonstantin Belousov break; 2186db7f8e5SKonstantin Belousov default: 2196db7f8e5SKonstantin Belousov error = ENOTTY; 2206db7f8e5SKonstantin Belousov break; 2216db7f8e5SKonstantin Belousov } 2226db7f8e5SKonstantin Belousov return (error); 2236db7f8e5SKonstantin Belousov } 2246db7f8e5SKonstantin Belousov 2256db7f8e5SKonstantin Belousov static int 2266db7f8e5SKonstantin Belousov nvdimm_spa_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size, 2276db7f8e5SKonstantin Belousov vm_object_t *objp, int nprot) 2286db7f8e5SKonstantin Belousov { 2296db7f8e5SKonstantin Belousov struct SPA_mapping *spa; 2306db7f8e5SKonstantin Belousov 2316db7f8e5SKonstantin Belousov spa = dev->si_drv1; 2326db7f8e5SKonstantin Belousov if (spa->spa_obj == NULL) 2336db7f8e5SKonstantin Belousov return (ENXIO); 2346db7f8e5SKonstantin Belousov if (*offset >= spa->spa_len || *offset + size < *offset || 2356db7f8e5SKonstantin Belousov *offset + size > spa->spa_len) 2366db7f8e5SKonstantin Belousov return (EINVAL); 2376db7f8e5SKonstantin Belousov vm_object_reference(spa->spa_obj); 2386db7f8e5SKonstantin Belousov *objp = spa->spa_obj; 2396db7f8e5SKonstantin Belousov return (0); 2406db7f8e5SKonstantin Belousov } 2416db7f8e5SKonstantin Belousov 2426db7f8e5SKonstantin Belousov static struct cdevsw spa_cdevsw = { 2436db7f8e5SKonstantin Belousov .d_version = D_VERSION, 2446db7f8e5SKonstantin Belousov .d_flags = D_DISK, 2456db7f8e5SKonstantin Belousov .d_name = "nvdimm_spa", 2466db7f8e5SKonstantin Belousov .d_read = nvdimm_spa_rw, 2476db7f8e5SKonstantin Belousov .d_write = nvdimm_spa_rw, 2486db7f8e5SKonstantin Belousov .d_ioctl = nvdimm_spa_ioctl, 2496db7f8e5SKonstantin Belousov .d_mmap_single = nvdimm_spa_mmap_single, 2506db7f8e5SKonstantin Belousov }; 2516db7f8e5SKonstantin Belousov 2526db7f8e5SKonstantin Belousov static void 2536db7f8e5SKonstantin Belousov nvdimm_spa_g_all_unmapped(struct SPA_mapping *spa, struct bio *bp, 2546db7f8e5SKonstantin Belousov int rw) 2556db7f8e5SKonstantin Belousov { 2566db7f8e5SKonstantin Belousov struct vm_page maa[bp->bio_ma_n]; 2576db7f8e5SKonstantin Belousov vm_page_t ma[bp->bio_ma_n]; 2586db7f8e5SKonstantin Belousov vm_memattr_t mattr; 2596db7f8e5SKonstantin Belousov int i; 2606db7f8e5SKonstantin Belousov 2616db7f8e5SKonstantin Belousov mattr = nvdimm_spa_memattr(spa); 2626db7f8e5SKonstantin Belousov for (i = 0; i < nitems(ma); i++) { 2636db7f8e5SKonstantin Belousov maa[i].flags = 0; 2646db7f8e5SKonstantin Belousov vm_page_initfake(&maa[i], spa->spa_phys_base + 2656db7f8e5SKonstantin Belousov trunc_page(bp->bio_offset) + PAGE_SIZE * i, mattr); 2666db7f8e5SKonstantin Belousov ma[i] = &maa[i]; 2676db7f8e5SKonstantin Belousov } 2686db7f8e5SKonstantin Belousov if (rw == BIO_READ) 2696db7f8e5SKonstantin Belousov pmap_copy_pages(ma, bp->bio_offset & PAGE_MASK, bp->bio_ma, 2706db7f8e5SKonstantin Belousov bp->bio_ma_offset, bp->bio_length); 2716db7f8e5SKonstantin Belousov else 2726db7f8e5SKonstantin Belousov pmap_copy_pages(bp->bio_ma, bp->bio_ma_offset, ma, 2736db7f8e5SKonstantin Belousov bp->bio_offset & PAGE_MASK, bp->bio_length); 2746db7f8e5SKonstantin Belousov } 2756db7f8e5SKonstantin Belousov 2766db7f8e5SKonstantin Belousov static void 2776db7f8e5SKonstantin Belousov nvdimm_spa_g_thread(void *arg) 2786db7f8e5SKonstantin Belousov { 2796db7f8e5SKonstantin Belousov struct SPA_mapping *spa; 2806db7f8e5SKonstantin Belousov struct bio *bp; 2816db7f8e5SKonstantin Belousov struct uio auio; 2826db7f8e5SKonstantin Belousov struct iovec aiovec; 2836db7f8e5SKonstantin Belousov int error; 2846db7f8e5SKonstantin Belousov 2856db7f8e5SKonstantin Belousov spa = arg; 2866db7f8e5SKonstantin Belousov for (;;) { 2876db7f8e5SKonstantin Belousov mtx_lock(&spa->spa_g_mtx); 2886db7f8e5SKonstantin Belousov for (;;) { 2896db7f8e5SKonstantin Belousov bp = bioq_takefirst(&spa->spa_g_queue); 2906db7f8e5SKonstantin Belousov if (bp != NULL) 2916db7f8e5SKonstantin Belousov break; 2926db7f8e5SKonstantin Belousov msleep(&spa->spa_g_queue, &spa->spa_g_mtx, PRIBIO, 2936db7f8e5SKonstantin Belousov "spa_g", 0); 2946db7f8e5SKonstantin Belousov if (!spa->spa_g_proc_run) { 2956db7f8e5SKonstantin Belousov spa->spa_g_proc_exiting = true; 2966db7f8e5SKonstantin Belousov wakeup(&spa->spa_g_queue); 2976db7f8e5SKonstantin Belousov mtx_unlock(&spa->spa_g_mtx); 2986db7f8e5SKonstantin Belousov kproc_exit(0); 2996db7f8e5SKonstantin Belousov } 3006db7f8e5SKonstantin Belousov continue; 3016db7f8e5SKonstantin Belousov } 3026db7f8e5SKonstantin Belousov mtx_unlock(&spa->spa_g_mtx); 3036db7f8e5SKonstantin Belousov if (bp->bio_cmd != BIO_READ && bp->bio_cmd != BIO_WRITE && 3046db7f8e5SKonstantin Belousov bp->bio_cmd != BIO_FLUSH) { 3056db7f8e5SKonstantin Belousov error = EOPNOTSUPP; 3066db7f8e5SKonstantin Belousov goto completed; 3076db7f8e5SKonstantin Belousov } 3086db7f8e5SKonstantin Belousov 3096db7f8e5SKonstantin Belousov error = 0; 3106db7f8e5SKonstantin Belousov if (bp->bio_cmd == BIO_FLUSH) { 3116db7f8e5SKonstantin Belousov if (spa->spa_kva != NULL) { 3126db7f8e5SKonstantin Belousov pmap_large_map_wb(spa->spa_kva, spa->spa_len); 3136db7f8e5SKonstantin Belousov } else { 3146db7f8e5SKonstantin Belousov pmap_flush_cache_phys_range( 3156db7f8e5SKonstantin Belousov (vm_paddr_t)spa->spa_phys_base, 3166db7f8e5SKonstantin Belousov (vm_paddr_t)spa->spa_phys_base + 3176db7f8e5SKonstantin Belousov spa->spa_len, nvdimm_spa_memattr(spa)); 3186db7f8e5SKonstantin Belousov } 3196db7f8e5SKonstantin Belousov /* 3206db7f8e5SKonstantin Belousov * XXX flush IMC 3216db7f8e5SKonstantin Belousov */ 3226db7f8e5SKonstantin Belousov goto completed; 3236db7f8e5SKonstantin Belousov } 3246db7f8e5SKonstantin Belousov 3256db7f8e5SKonstantin Belousov if ((bp->bio_flags & BIO_UNMAPPED) != 0) { 3266db7f8e5SKonstantin Belousov if (spa->spa_kva != NULL) { 3276db7f8e5SKonstantin Belousov aiovec.iov_base = (char *)spa->spa_kva + 3286db7f8e5SKonstantin Belousov bp->bio_offset; 3296db7f8e5SKonstantin Belousov aiovec.iov_len = bp->bio_length; 3306db7f8e5SKonstantin Belousov auio.uio_iov = &aiovec; 3316db7f8e5SKonstantin Belousov auio.uio_iovcnt = 1; 3326db7f8e5SKonstantin Belousov auio.uio_resid = bp->bio_length; 3336db7f8e5SKonstantin Belousov auio.uio_offset = bp->bio_offset; 3346db7f8e5SKonstantin Belousov auio.uio_segflg = UIO_SYSSPACE; 3356db7f8e5SKonstantin Belousov auio.uio_rw = bp->bio_cmd == BIO_READ ? 3366db7f8e5SKonstantin Belousov UIO_WRITE : UIO_READ; 3376db7f8e5SKonstantin Belousov auio.uio_td = curthread; 3386db7f8e5SKonstantin Belousov error = uiomove_fromphys(bp->bio_ma, 3396db7f8e5SKonstantin Belousov bp->bio_ma_offset, bp->bio_length, &auio); 340cbbdd283SKonstantin Belousov bp->bio_resid = auio.uio_resid; 3416db7f8e5SKonstantin Belousov } else { 3426db7f8e5SKonstantin Belousov nvdimm_spa_g_all_unmapped(spa, bp, bp->bio_cmd); 343cbbdd283SKonstantin Belousov bp->bio_resid = bp->bio_length; 3446db7f8e5SKonstantin Belousov error = 0; 3456db7f8e5SKonstantin Belousov } 3466db7f8e5SKonstantin Belousov } else { 3476db7f8e5SKonstantin Belousov aiovec.iov_base = bp->bio_data; 3486db7f8e5SKonstantin Belousov aiovec.iov_len = bp->bio_length; 3496db7f8e5SKonstantin Belousov auio.uio_iov = &aiovec; 3506db7f8e5SKonstantin Belousov auio.uio_iovcnt = 1; 3516db7f8e5SKonstantin Belousov auio.uio_resid = bp->bio_length; 3526db7f8e5SKonstantin Belousov auio.uio_offset = bp->bio_offset; 3536db7f8e5SKonstantin Belousov auio.uio_segflg = UIO_SYSSPACE; 3546db7f8e5SKonstantin Belousov auio.uio_rw = bp->bio_cmd == BIO_READ ? UIO_READ : 3556db7f8e5SKonstantin Belousov UIO_WRITE; 3566db7f8e5SKonstantin Belousov auio.uio_td = curthread; 3576db7f8e5SKonstantin Belousov error = nvdimm_spa_uio(spa, &auio); 358cbbdd283SKonstantin Belousov bp->bio_resid = auio.uio_resid; 3596db7f8e5SKonstantin Belousov } 360cbbdd283SKonstantin Belousov bp->bio_bcount = bp->bio_length; 3616db7f8e5SKonstantin Belousov devstat_end_transaction_bio(spa->spa_g_devstat, bp); 3626db7f8e5SKonstantin Belousov completed: 3636db7f8e5SKonstantin Belousov bp->bio_completed = bp->bio_length; 3646db7f8e5SKonstantin Belousov g_io_deliver(bp, error); 3656db7f8e5SKonstantin Belousov } 3666db7f8e5SKonstantin Belousov } 3676db7f8e5SKonstantin Belousov 3686db7f8e5SKonstantin Belousov static void 3696db7f8e5SKonstantin Belousov nvdimm_spa_g_start(struct bio *bp) 3706db7f8e5SKonstantin Belousov { 3716db7f8e5SKonstantin Belousov struct SPA_mapping *spa; 3726db7f8e5SKonstantin Belousov 3736db7f8e5SKonstantin Belousov spa = bp->bio_to->geom->softc; 3746db7f8e5SKonstantin Belousov if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) { 3756db7f8e5SKonstantin Belousov mtx_lock(&spa->spa_g_stat_mtx); 3766db7f8e5SKonstantin Belousov devstat_start_transaction_bio(spa->spa_g_devstat, bp); 3776db7f8e5SKonstantin Belousov mtx_unlock(&spa->spa_g_stat_mtx); 3786db7f8e5SKonstantin Belousov } 3796db7f8e5SKonstantin Belousov mtx_lock(&spa->spa_g_mtx); 3806db7f8e5SKonstantin Belousov bioq_disksort(&spa->spa_g_queue, bp); 3816db7f8e5SKonstantin Belousov wakeup(&spa->spa_g_queue); 3826db7f8e5SKonstantin Belousov mtx_unlock(&spa->spa_g_mtx); 3836db7f8e5SKonstantin Belousov } 3846db7f8e5SKonstantin Belousov 3856db7f8e5SKonstantin Belousov static int 3866db7f8e5SKonstantin Belousov nvdimm_spa_g_access(struct g_provider *pp, int r, int w, int e) 3876db7f8e5SKonstantin Belousov { 3886db7f8e5SKonstantin Belousov 3896db7f8e5SKonstantin Belousov return (0); 3906db7f8e5SKonstantin Belousov } 3916db7f8e5SKonstantin Belousov 3926db7f8e5SKonstantin Belousov static g_init_t nvdimm_spa_g_init; 3936db7f8e5SKonstantin Belousov static g_fini_t nvdimm_spa_g_fini; 3946db7f8e5SKonstantin Belousov 3956db7f8e5SKonstantin Belousov struct g_class nvdimm_spa_g_class = { 3966db7f8e5SKonstantin Belousov .name = "SPA", 3976db7f8e5SKonstantin Belousov .version = G_VERSION, 3986db7f8e5SKonstantin Belousov .start = nvdimm_spa_g_start, 3996db7f8e5SKonstantin Belousov .access = nvdimm_spa_g_access, 4006db7f8e5SKonstantin Belousov .init = nvdimm_spa_g_init, 4016db7f8e5SKonstantin Belousov .fini = nvdimm_spa_g_fini, 4026db7f8e5SKonstantin Belousov }; 4036db7f8e5SKonstantin Belousov DECLARE_GEOM_CLASS(nvdimm_spa_g_class, g_spa); 4046db7f8e5SKonstantin Belousov 4056db7f8e5SKonstantin Belousov static int 4066db7f8e5SKonstantin Belousov nvdimm_spa_init_one(struct SPA_mapping *spa, ACPI_NFIT_SYSTEM_ADDRESS *nfitaddr, 4076db7f8e5SKonstantin Belousov int spa_type) 4086db7f8e5SKonstantin Belousov { 4096db7f8e5SKonstantin Belousov struct make_dev_args mda; 4106db7f8e5SKonstantin Belousov struct sglist *spa_sg; 4116db7f8e5SKonstantin Belousov int error, error1; 4126db7f8e5SKonstantin Belousov 4136db7f8e5SKonstantin Belousov spa->spa_type = spa_type; 4146db7f8e5SKonstantin Belousov spa->spa_domain = ((nfitaddr->Flags & ACPI_NFIT_PROXIMITY_VALID) != 0) ? 4156db7f8e5SKonstantin Belousov nfitaddr->ProximityDomain : -1; 4166db7f8e5SKonstantin Belousov spa->spa_nfit_idx = nfitaddr->RangeIndex; 4176db7f8e5SKonstantin Belousov spa->spa_phys_base = nfitaddr->Address; 4186db7f8e5SKonstantin Belousov spa->spa_len = nfitaddr->Length; 4196db7f8e5SKonstantin Belousov spa->spa_efi_mem_flags = nfitaddr->MemoryMapping; 4206db7f8e5SKonstantin Belousov if (bootverbose) { 4216db7f8e5SKonstantin Belousov printf("NVDIMM SPA%d base %#016jx len %#016jx %s fl %#jx\n", 4226db7f8e5SKonstantin Belousov spa->spa_nfit_idx, 4236db7f8e5SKonstantin Belousov (uintmax_t)spa->spa_phys_base, (uintmax_t)spa->spa_len, 4246db7f8e5SKonstantin Belousov nvdimm_SPA_uuid_list[spa_type].u_name, 4256db7f8e5SKonstantin Belousov spa->spa_efi_mem_flags); 4266db7f8e5SKonstantin Belousov } 4276db7f8e5SKonstantin Belousov if (!nvdimm_SPA_uuid_list[spa_type].u_usr_acc) 4286db7f8e5SKonstantin Belousov return (0); 4296db7f8e5SKonstantin Belousov 4306db7f8e5SKonstantin Belousov error1 = pmap_large_map(spa->spa_phys_base, spa->spa_len, 4316db7f8e5SKonstantin Belousov &spa->spa_kva, nvdimm_spa_memattr(spa)); 4326db7f8e5SKonstantin Belousov if (error1 != 0) { 4336db7f8e5SKonstantin Belousov printf("NVDIMM SPA%d cannot map into KVA, error %d\n", 4346db7f8e5SKonstantin Belousov spa->spa_nfit_idx, error1); 4356db7f8e5SKonstantin Belousov spa->spa_kva = NULL; 4366db7f8e5SKonstantin Belousov } 4376db7f8e5SKonstantin Belousov 4386db7f8e5SKonstantin Belousov spa_sg = sglist_alloc(1, M_WAITOK); 4396db7f8e5SKonstantin Belousov error = sglist_append_phys(spa_sg, spa->spa_phys_base, 4406db7f8e5SKonstantin Belousov spa->spa_len); 4416db7f8e5SKonstantin Belousov if (error == 0) { 4426db7f8e5SKonstantin Belousov spa->spa_obj = vm_pager_allocate(OBJT_SG, spa_sg, spa->spa_len, 4436db7f8e5SKonstantin Belousov VM_PROT_ALL, 0, NULL); 4446db7f8e5SKonstantin Belousov if (spa->spa_obj == NULL) { 4456db7f8e5SKonstantin Belousov printf("NVDIMM SPA%d failed to alloc vm object", 4466db7f8e5SKonstantin Belousov spa->spa_nfit_idx); 4476db7f8e5SKonstantin Belousov sglist_free(spa_sg); 4486db7f8e5SKonstantin Belousov } 4496db7f8e5SKonstantin Belousov } else { 4506db7f8e5SKonstantin Belousov printf("NVDIMM SPA%d failed to init sglist, error %d", 4516db7f8e5SKonstantin Belousov spa->spa_nfit_idx, error); 4526db7f8e5SKonstantin Belousov sglist_free(spa_sg); 4536db7f8e5SKonstantin Belousov } 4546db7f8e5SKonstantin Belousov 4556db7f8e5SKonstantin Belousov make_dev_args_init(&mda); 4566db7f8e5SKonstantin Belousov mda.mda_flags = MAKEDEV_WAITOK | MAKEDEV_CHECKNAME; 4576db7f8e5SKonstantin Belousov mda.mda_devsw = &spa_cdevsw; 4586db7f8e5SKonstantin Belousov mda.mda_cr = NULL; 4596db7f8e5SKonstantin Belousov mda.mda_uid = UID_ROOT; 4606db7f8e5SKonstantin Belousov mda.mda_gid = GID_OPERATOR; 4616db7f8e5SKonstantin Belousov mda.mda_mode = 0660; 4626db7f8e5SKonstantin Belousov mda.mda_si_drv1 = spa; 4636db7f8e5SKonstantin Belousov error = make_dev_s(&mda, &spa->spa_dev, "nvdimm_spa%d", 4646db7f8e5SKonstantin Belousov spa->spa_nfit_idx); 4656db7f8e5SKonstantin Belousov if (error != 0) { 4666db7f8e5SKonstantin Belousov printf("NVDIMM SPA%d cannot create devfs node, error %d\n", 4676db7f8e5SKonstantin Belousov spa->spa_nfit_idx, error); 4686db7f8e5SKonstantin Belousov if (error1 == 0) 4696db7f8e5SKonstantin Belousov error1 = error; 4706db7f8e5SKonstantin Belousov } 4716db7f8e5SKonstantin Belousov 4726db7f8e5SKonstantin Belousov bioq_init(&spa->spa_g_queue); 4736db7f8e5SKonstantin Belousov mtx_init(&spa->spa_g_mtx, "spag", NULL, MTX_DEF); 4746db7f8e5SKonstantin Belousov mtx_init(&spa->spa_g_stat_mtx, "spagst", NULL, MTX_DEF); 4756db7f8e5SKonstantin Belousov spa->spa_g_proc_run = true; 4766db7f8e5SKonstantin Belousov spa->spa_g_proc_exiting = false; 4776db7f8e5SKonstantin Belousov error = kproc_create(nvdimm_spa_g_thread, spa, &spa->spa_g_proc, 0, 0, 4786db7f8e5SKonstantin Belousov "g_spa%d", spa->spa_nfit_idx); 4796db7f8e5SKonstantin Belousov if (error != 0) { 4806db7f8e5SKonstantin Belousov printf("NVDIMM SPA%d cannot create geom worker, error %d\n", 4816db7f8e5SKonstantin Belousov spa->spa_nfit_idx, error); 4826db7f8e5SKonstantin Belousov if (error1 == 0) 4836db7f8e5SKonstantin Belousov error1 = error; 4846db7f8e5SKonstantin Belousov } else { 4856db7f8e5SKonstantin Belousov g_topology_assert(); 4866db7f8e5SKonstantin Belousov spa->spa_g = g_new_geomf(&nvdimm_spa_g_class, "spa%d", 4876db7f8e5SKonstantin Belousov spa->spa_nfit_idx); 4886db7f8e5SKonstantin Belousov spa->spa_g->softc = spa; 4896db7f8e5SKonstantin Belousov spa->spa_p = g_new_providerf(spa->spa_g, "spa%d", 4906db7f8e5SKonstantin Belousov spa->spa_nfit_idx); 4916db7f8e5SKonstantin Belousov spa->spa_p->mediasize = spa->spa_len; 4926db7f8e5SKonstantin Belousov spa->spa_p->sectorsize = DEV_BSIZE; 4936db7f8e5SKonstantin Belousov spa->spa_p->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE | 4946db7f8e5SKonstantin Belousov G_PF_ACCEPT_UNMAPPED; 4956db7f8e5SKonstantin Belousov g_error_provider(spa->spa_p, 0); 4966db7f8e5SKonstantin Belousov spa->spa_g_devstat = devstat_new_entry("spa", spa->spa_nfit_idx, 4976db7f8e5SKonstantin Belousov DEV_BSIZE, DEVSTAT_ALL_SUPPORTED, DEVSTAT_TYPE_DIRECT, 4986db7f8e5SKonstantin Belousov DEVSTAT_PRIORITY_MAX); 4996db7f8e5SKonstantin Belousov } 5006db7f8e5SKonstantin Belousov return (error1); 5016db7f8e5SKonstantin Belousov } 5026db7f8e5SKonstantin Belousov 5036db7f8e5SKonstantin Belousov static void 5046db7f8e5SKonstantin Belousov nvdimm_spa_fini_one(struct SPA_mapping *spa) 5056db7f8e5SKonstantin Belousov { 5066db7f8e5SKonstantin Belousov 5076db7f8e5SKonstantin Belousov mtx_lock(&spa->spa_g_mtx); 5086db7f8e5SKonstantin Belousov spa->spa_g_proc_run = false; 5096db7f8e5SKonstantin Belousov wakeup(&spa->spa_g_queue); 5106db7f8e5SKonstantin Belousov while (!spa->spa_g_proc_exiting) 5116db7f8e5SKonstantin Belousov msleep(&spa->spa_g_queue, &spa->spa_g_mtx, PRIBIO, "spa_e", 0); 5126db7f8e5SKonstantin Belousov mtx_unlock(&spa->spa_g_mtx); 5136db7f8e5SKonstantin Belousov if (spa->spa_g != NULL) { 5146db7f8e5SKonstantin Belousov g_topology_lock(); 5156db7f8e5SKonstantin Belousov g_wither_geom(spa->spa_g, ENXIO); 5166db7f8e5SKonstantin Belousov g_topology_unlock(); 5176db7f8e5SKonstantin Belousov spa->spa_g = NULL; 5186db7f8e5SKonstantin Belousov spa->spa_p = NULL; 5196db7f8e5SKonstantin Belousov } 5206db7f8e5SKonstantin Belousov if (spa->spa_g_devstat != NULL) { 5216db7f8e5SKonstantin Belousov devstat_remove_entry(spa->spa_g_devstat); 5226db7f8e5SKonstantin Belousov spa->spa_g_devstat = NULL; 5236db7f8e5SKonstantin Belousov } 5246db7f8e5SKonstantin Belousov if (spa->spa_dev != NULL) { 5256db7f8e5SKonstantin Belousov destroy_dev(spa->spa_dev); 5266db7f8e5SKonstantin Belousov spa->spa_dev = NULL; 5276db7f8e5SKonstantin Belousov } 5286db7f8e5SKonstantin Belousov vm_object_deallocate(spa->spa_obj); 5296db7f8e5SKonstantin Belousov if (spa->spa_kva != NULL) { 5306db7f8e5SKonstantin Belousov pmap_large_unmap(spa->spa_kva, spa->spa_len); 5316db7f8e5SKonstantin Belousov spa->spa_kva = NULL; 5326db7f8e5SKonstantin Belousov } 5336db7f8e5SKonstantin Belousov mtx_destroy(&spa->spa_g_mtx); 5346db7f8e5SKonstantin Belousov mtx_destroy(&spa->spa_g_stat_mtx); 5356db7f8e5SKonstantin Belousov } 5366db7f8e5SKonstantin Belousov 5376db7f8e5SKonstantin Belousov static int 5386db7f8e5SKonstantin Belousov nvdimm_spa_parse(void *nfitsubtbl, void *arg) 5396db7f8e5SKonstantin Belousov { 5406db7f8e5SKonstantin Belousov ACPI_NFIT_SYSTEM_ADDRESS *nfitaddr; 5416db7f8e5SKonstantin Belousov struct SPA_mapping *spa; 5426db7f8e5SKonstantin Belousov int error, *i, j; 5436db7f8e5SKonstantin Belousov 5446db7f8e5SKonstantin Belousov i = arg; 5456db7f8e5SKonstantin Belousov spa = &spa_mappings[*i]; 5466db7f8e5SKonstantin Belousov nfitaddr = nfitsubtbl; 5476db7f8e5SKonstantin Belousov 5486db7f8e5SKonstantin Belousov for (j = 0; j < nitems(nvdimm_SPA_uuid_list); j++) { 5496db7f8e5SKonstantin Belousov /* XXXKIB: is ACPI UUID representation compatible ? */ 5506db7f8e5SKonstantin Belousov if (uuidcmp((struct uuid *)&nfitaddr->RangeGuid, 5516db7f8e5SKonstantin Belousov &nvdimm_SPA_uuid_list[j].u_id) != 0) 5526db7f8e5SKonstantin Belousov continue; 5536db7f8e5SKonstantin Belousov error = nvdimm_spa_init_one(spa, nfitaddr, j); 5546db7f8e5SKonstantin Belousov if (error != 0) 5556db7f8e5SKonstantin Belousov nvdimm_spa_fini_one(spa); 5566db7f8e5SKonstantin Belousov break; 5576db7f8e5SKonstantin Belousov } 5586db7f8e5SKonstantin Belousov if (j == nitems(nvdimm_SPA_uuid_list) && bootverbose) { 5596db7f8e5SKonstantin Belousov printf("Unknown SPA UUID %d ", nfitaddr->RangeIndex); 5606db7f8e5SKonstantin Belousov printf_uuid((struct uuid *)&nfitaddr->RangeGuid); 5616db7f8e5SKonstantin Belousov printf("\n"); 5626db7f8e5SKonstantin Belousov } 5636db7f8e5SKonstantin Belousov (*i)++; 5646db7f8e5SKonstantin Belousov return (0); 5656db7f8e5SKonstantin Belousov } 5666db7f8e5SKonstantin Belousov 5676db7f8e5SKonstantin Belousov static int 5686db7f8e5SKonstantin Belousov nvdimm_spa_init1(ACPI_TABLE_NFIT *nfitbl) 5696db7f8e5SKonstantin Belousov { 5706db7f8e5SKonstantin Belousov struct nvdimm_SPA_uuid_list_elm *sle; 5716db7f8e5SKonstantin Belousov int error, i; 5726db7f8e5SKonstantin Belousov 5736db7f8e5SKonstantin Belousov for (i = 0; i < nitems(nvdimm_SPA_uuid_list); i++) { 5746db7f8e5SKonstantin Belousov sle = &nvdimm_SPA_uuid_list[i]; 5756db7f8e5SKonstantin Belousov error = parse_uuid(sle->u_id_str, &sle->u_id); 5766db7f8e5SKonstantin Belousov if (error != 0) { 5776db7f8e5SKonstantin Belousov if (bootverbose) 5786db7f8e5SKonstantin Belousov printf("nvdimm_identify: error %d parsing " 5796db7f8e5SKonstantin Belousov "known SPA UUID %d %s\n", error, i, 5806db7f8e5SKonstantin Belousov sle->u_id_str); 5816db7f8e5SKonstantin Belousov return (error); 5826db7f8e5SKonstantin Belousov } 5836db7f8e5SKonstantin Belousov } 5846db7f8e5SKonstantin Belousov 5856db7f8e5SKonstantin Belousov error = nvdimm_iterate_nfit(nfitbl, ACPI_NFIT_TYPE_SYSTEM_ADDRESS, 5866db7f8e5SKonstantin Belousov nvdimm_spa_count, &spa_mappings_cnt); 5876db7f8e5SKonstantin Belousov if (error != 0) 5886db7f8e5SKonstantin Belousov return (error); 5896db7f8e5SKonstantin Belousov spa_mappings = malloc(sizeof(struct SPA_mapping) * spa_mappings_cnt, 5906db7f8e5SKonstantin Belousov M_NVDIMM, M_WAITOK | M_ZERO); 5916db7f8e5SKonstantin Belousov i = 0; 5926db7f8e5SKonstantin Belousov error = nvdimm_iterate_nfit(nfitbl, ACPI_NFIT_TYPE_SYSTEM_ADDRESS, 5936db7f8e5SKonstantin Belousov nvdimm_spa_parse, &i); 5946db7f8e5SKonstantin Belousov if (error != 0) { 5956db7f8e5SKonstantin Belousov free(spa_mappings, M_NVDIMM); 5966db7f8e5SKonstantin Belousov spa_mappings = NULL; 5976db7f8e5SKonstantin Belousov return (error); 5986db7f8e5SKonstantin Belousov } 5996db7f8e5SKonstantin Belousov return (0); 6006db7f8e5SKonstantin Belousov } 6016db7f8e5SKonstantin Belousov 6026db7f8e5SKonstantin Belousov static void 6036db7f8e5SKonstantin Belousov nvdimm_spa_g_init(struct g_class *mp __unused) 6046db7f8e5SKonstantin Belousov { 6056db7f8e5SKonstantin Belousov ACPI_TABLE_NFIT *nfitbl; 6066db7f8e5SKonstantin Belousov ACPI_STATUS status; 6076db7f8e5SKonstantin Belousov int error; 6086db7f8e5SKonstantin Belousov 6096db7f8e5SKonstantin Belousov spa_mappings_cnt = 0; 6106db7f8e5SKonstantin Belousov spa_mappings = NULL; 6116db7f8e5SKonstantin Belousov if (acpi_disabled("nvdimm")) 6126db7f8e5SKonstantin Belousov return; 6136db7f8e5SKonstantin Belousov status = AcpiGetTable(ACPI_SIG_NFIT, 1, (ACPI_TABLE_HEADER **)&nfitbl); 6146db7f8e5SKonstantin Belousov if (ACPI_FAILURE(status)) { 6156db7f8e5SKonstantin Belousov if (bootverbose) 6166db7f8e5SKonstantin Belousov printf("nvdimm_spa_g_init: cannot find NFIT\n"); 6176db7f8e5SKonstantin Belousov return; 6186db7f8e5SKonstantin Belousov } 6196db7f8e5SKonstantin Belousov error = nvdimm_spa_init1(nfitbl); 6206db7f8e5SKonstantin Belousov if (error != 0) 6216db7f8e5SKonstantin Belousov printf("nvdimm_spa_g_init: error %d\n", error); 6226db7f8e5SKonstantin Belousov AcpiPutTable(&nfitbl->Header); 6236db7f8e5SKonstantin Belousov } 6246db7f8e5SKonstantin Belousov 6256db7f8e5SKonstantin Belousov static void 6266db7f8e5SKonstantin Belousov nvdimm_spa_g_fini(struct g_class *mp __unused) 6276db7f8e5SKonstantin Belousov { 6286db7f8e5SKonstantin Belousov int i; 6296db7f8e5SKonstantin Belousov 6306db7f8e5SKonstantin Belousov if (spa_mappings == NULL) 6316db7f8e5SKonstantin Belousov return; 6326db7f8e5SKonstantin Belousov for (i = 0; i < spa_mappings_cnt; i++) 6336db7f8e5SKonstantin Belousov nvdimm_spa_fini_one(&spa_mappings[i]); 6346db7f8e5SKonstantin Belousov free(spa_mappings, M_NVDIMM); 6356db7f8e5SKonstantin Belousov spa_mappings = NULL; 6366db7f8e5SKonstantin Belousov spa_mappings_cnt = 0; 6376db7f8e5SKonstantin Belousov } 638