1 /* 2 * Copyright (c) 2021 Maxime Villard, m00nbsd.net 3 * All rights reserved. 4 * 5 * This code is part of the NVMM hypervisor. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/systm.h> 31 #include <sys/kernel.h> 32 #include <sys/mman.h> 33 34 #include "nvmm.h" 35 #include "nvmm_os.h" 36 #include "nvmm_internal.h" 37 38 os_vmspace_t * 39 os_vmspace_create(vaddr_t vmin, vaddr_t vmax) 40 { 41 return uvmspace_alloc(vmin, vmax, false); 42 } 43 44 void 45 os_vmspace_destroy(os_vmspace_t *vm) 46 { 47 uvmspace_free(vm); 48 } 49 50 int 51 os_vmspace_fault(os_vmspace_t *vm, vaddr_t va, vm_prot_t prot) 52 { 53 return uvm_fault(&vm->vm_map, va, prot); 54 } 55 56 os_vmobj_t * 57 os_vmobj_create(voff_t size) 58 { 59 return uao_create(size, 0); 60 } 61 62 void 63 os_vmobj_ref(os_vmobj_t *vmobj) 64 { 65 uao_reference(vmobj); 66 } 67 68 void 69 os_vmobj_rel(os_vmobj_t *vmobj) 70 { 71 uao_detach(vmobj); 72 } 73 74 int 75 os_vmobj_map(struct vm_map *map, vaddr_t *addr, vsize_t size, os_vmobj_t *vmobj, 76 voff_t offset, bool wired, bool fixed, bool shared, int prot, int maxprot) 77 { 78 uvm_flag_t uflags, uprot, umaxprot; 79 int error; 80 81 /* Convert prot. */ 82 uprot = 0; 83 if (prot & PROT_READ) 84 uprot |= UVM_PROT_R; 85 if (prot & PROT_WRITE) 86 uprot |= UVM_PROT_W; 87 if (prot & PROT_EXEC) 88 uprot |= UVM_PROT_X; 89 90 /* Convert maxprot. */ 91 umaxprot = 0; 92 if (maxprot & PROT_READ) 93 umaxprot |= UVM_PROT_R; 94 if (maxprot & PROT_WRITE) 95 umaxprot |= UVM_PROT_W; 96 if (maxprot & PROT_EXEC) 97 umaxprot |= UVM_PROT_X; 98 99 uflags = UVM_MAPFLAG(uprot, umaxprot, 100 shared ? UVM_INH_SHARE : UVM_INH_NONE, UVM_ADV_RANDOM, 101 fixed ? (UVM_FLAG_FIXED | UVM_FLAG_UNMAP) : 0); 102 103 if (!fixed) { 104 /* Need to provide a hint. */ 105 if (map == os_curproc_map) { 106 *addr = curproc->p_emul->e_vm_default_addr(curproc, 107 (vaddr_t)curproc->p_vmspace->vm_daddr, size, 108 curproc->p_vmspace->vm_map.flags & VM_MAP_TOPDOWN); 109 } else { 110 *addr = 0; 111 } 112 } 113 114 /* Get a reference to the object. */ 115 os_vmobj_ref(vmobj); 116 117 /* 118 * Map the object. This consumes the reference on success only. On 119 * failure we must drop the reference manually. 120 */ 121 error = uvm_map(map, addr, size, vmobj, offset, 0, uflags); 122 if (error) { 123 /* Drop the ref. */ 124 os_vmobj_rel(vmobj); 125 return error; 126 } 127 128 if (wired) { 129 error = uvm_map_pageable(map, *addr, *addr + size, false, 0); 130 if (error) { 131 os_vmobj_unmap(map, *addr, *addr + size, false); 132 return error; 133 } 134 } 135 136 return 0; 137 } 138 139 void 140 os_vmobj_unmap(struct vm_map *map, vaddr_t start, vaddr_t end, 141 bool wired __unused) 142 { 143 uvm_unmap(map, start, end); 144 } 145 146 void * 147 os_pagemem_zalloc(size_t size) 148 { 149 void *ret; 150 151 ret = (void *)uvm_km_alloc(kernel_map, roundup(size, PAGE_SIZE), 0, 152 UVM_KMF_WIRED | UVM_KMF_ZERO); 153 154 OS_ASSERT((uintptr_t)ret % PAGE_SIZE == 0); 155 156 return ret; 157 } 158 159 void 160 os_pagemem_free(void *ptr, size_t size) 161 { 162 uvm_km_free(kernel_map, (vaddr_t)ptr, roundup(size, PAGE_SIZE), 163 UVM_KMF_WIRED); 164 } 165 166 paddr_t 167 os_pa_zalloc(void) 168 { 169 struct vm_page *pg; 170 171 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 172 173 return VM_PAGE_TO_PHYS(pg); 174 } 175 176 void 177 os_pa_free(paddr_t pa) 178 { 179 uvm_pagefree(PHYS_TO_VM_PAGE(pa)); 180 } 181 182 int 183 os_contigpa_zalloc(paddr_t *pa, vaddr_t *va, size_t npages) 184 { 185 struct pglist pglist; 186 paddr_t _pa; 187 vaddr_t _va; 188 size_t i; 189 int ret; 190 191 ret = uvm_pglistalloc(npages * PAGE_SIZE, 0, ~0UL, PAGE_SIZE, 0, 192 &pglist, 1, 0); 193 if (ret != 0) 194 return ENOMEM; 195 _pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 196 _va = uvm_km_alloc(kernel_map, npages * PAGE_SIZE, 0, 197 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 198 if (_va == 0) 199 goto error; 200 201 for (i = 0; i < npages; i++) { 202 pmap_kenter_pa(_va + i * PAGE_SIZE, _pa + i * PAGE_SIZE, 203 VM_PROT_READ | VM_PROT_WRITE, PMAP_WRITE_BACK); 204 } 205 pmap_update(pmap_kernel()); 206 207 memset((void *)_va, 0, npages * PAGE_SIZE); 208 209 *pa = _pa; 210 *va = _va; 211 return 0; 212 213 error: 214 for (i = 0; i < npages; i++) { 215 uvm_pagefree(PHYS_TO_VM_PAGE(_pa + i * PAGE_SIZE)); 216 } 217 return ENOMEM; 218 } 219 220 void 221 os_contigpa_free(paddr_t pa, vaddr_t va, size_t npages) 222 { 223 size_t i; 224 225 pmap_kremove(va, npages * PAGE_SIZE); 226 pmap_update(pmap_kernel()); 227 uvm_km_free(kernel_map, va, npages * PAGE_SIZE, UVM_KMF_VAONLY); 228 for (i = 0; i < npages; i++) { 229 uvm_pagefree(PHYS_TO_VM_PAGE(pa + i * PAGE_SIZE)); 230 } 231 } 232 233 /* -------------------------------------------------------------------------- */ 234 235 #include <sys/conf.h> 236 #include <sys/device.h> 237 #include <sys/file.h> 238 #include <sys/filedesc.h> 239 #include <sys/module.h> 240 241 #include "ioconf.h" 242 243 static dev_type_open(nbsd_nvmm_open); 244 static int nbsd_nvmm_ioctl(file_t *, u_long, void *); 245 static int nbsd_nvmm_close(file_t *); 246 247 const struct cdevsw nvmm_cdevsw = { 248 .d_open = nbsd_nvmm_open, 249 .d_close = noclose, 250 .d_read = noread, 251 .d_write = nowrite, 252 .d_ioctl = noioctl, 253 .d_stop = nostop, 254 .d_tty = notty, 255 .d_poll = nopoll, 256 .d_mmap = nommap, 257 .d_kqfilter = nokqfilter, 258 .d_discard = nodiscard, 259 .d_flag = D_OTHER | D_MPSAFE 260 }; 261 262 static const struct fileops nvmm_fileops = { 263 .fo_read = fbadop_read, 264 .fo_write = fbadop_write, 265 .fo_ioctl = nbsd_nvmm_ioctl, 266 .fo_fcntl = fnullop_fcntl, 267 .fo_poll = fnullop_poll, 268 .fo_stat = fbadop_stat, 269 .fo_close = nbsd_nvmm_close, 270 .fo_kqfilter = fnullop_kqfilter, 271 .fo_restart = fnullop_restart, 272 .fo_mmap = NULL, 273 }; 274 275 static int 276 nbsd_nvmm_open(dev_t dev, int flags, int type, struct lwp *l) 277 { 278 struct nvmm_owner *owner; 279 struct file *fp; 280 int error, fd; 281 282 if (__predict_false(nvmm_impl == NULL)) 283 return ENXIO; 284 if (minor(dev) != 0) 285 return EXDEV; 286 if (!(flags & O_CLOEXEC)) 287 return EINVAL; 288 error = fd_allocfile(&fp, &fd); 289 if (error) 290 return error; 291 292 if (OFLAGS(flags) & O_WRONLY) { 293 owner = &nvmm_root_owner; 294 } else { 295 owner = os_mem_alloc(sizeof(*owner)); 296 owner->pid = l->l_proc->p_pid; 297 } 298 299 return fd_clone(fp, fd, flags, &nvmm_fileops, owner); 300 } 301 302 static int 303 nbsd_nvmm_ioctl(file_t *fp, u_long cmd, void *data) 304 { 305 struct nvmm_owner *owner = fp->f_data; 306 307 OS_ASSERT(owner != NULL); 308 309 return nvmm_ioctl(owner, cmd, data); 310 } 311 312 static int 313 nbsd_nvmm_close(file_t *fp) 314 { 315 struct nvmm_owner *owner = fp->f_data; 316 317 OS_ASSERT(owner != NULL); 318 nvmm_kill_machines(owner); 319 if (owner != &nvmm_root_owner) { 320 os_mem_free(owner, sizeof(*owner)); 321 } 322 fp->f_data = NULL; 323 324 return 0; 325 } 326 327 /* -------------------------------------------------------------------------- */ 328 329 static int nvmm_match(device_t, cfdata_t, void *); 330 static void nvmm_attach(device_t, device_t, void *); 331 static int nvmm_detach(device_t, int); 332 333 extern struct cfdriver nvmm_cd; 334 335 CFATTACH_DECL_NEW(nvmm, 0, nvmm_match, nvmm_attach, nvmm_detach, NULL); 336 337 static struct cfdata nvmm_cfdata[] = { 338 { 339 .cf_name = "nvmm", 340 .cf_atname = "nvmm", 341 .cf_unit = 0, 342 .cf_fstate = FSTATE_STAR, 343 .cf_loc = NULL, 344 .cf_flags = 0, 345 .cf_pspec = NULL, 346 }, 347 { NULL, NULL, 0, FSTATE_NOTFOUND, NULL, 0, NULL } 348 }; 349 350 static int 351 nvmm_match(device_t self, cfdata_t cfdata, void *arg) 352 { 353 return 1; 354 } 355 356 static void 357 nvmm_attach(device_t parent, device_t self, void *aux) 358 { 359 int error; 360 361 error = nvmm_init(); 362 if (error) 363 panic("%s: impossible", __func__); 364 aprint_normal_dev(self, "attached, using backend %s\n", 365 nvmm_impl->name); 366 } 367 368 static int 369 nvmm_detach(device_t self, int flags) 370 { 371 if (os_atomic_load_uint(&nmachines) > 0) 372 return EBUSY; 373 nvmm_fini(); 374 return 0; 375 } 376 377 void 378 nvmmattach(int nunits) 379 { 380 /* nothing */ 381 } 382 383 MODULE(MODULE_CLASS_DRIVER, nvmm, NULL); 384 385 #if defined(_MODULE) 386 CFDRIVER_DECL(nvmm, DV_VIRTUAL, NULL); 387 #endif 388 389 static int 390 nvmm_modcmd(modcmd_t cmd, void *arg) 391 { 392 #if defined(_MODULE) 393 devmajor_t bmajor = NODEVMAJOR; 394 devmajor_t cmajor = 345; 395 #endif 396 int error; 397 398 switch (cmd) { 399 case MODULE_CMD_INIT: 400 if (nvmm_ident() == NULL) { 401 aprint_error("%s: cpu not supported\n", 402 nvmm_cd.cd_name); 403 return ENOTSUP; 404 } 405 #if defined(_MODULE) 406 error = config_cfdriver_attach(&nvmm_cd); 407 if (error) 408 return error; 409 #endif 410 error = config_cfattach_attach(nvmm_cd.cd_name, &nvmm_ca); 411 if (error) { 412 config_cfdriver_detach(&nvmm_cd); 413 aprint_error("%s: config_cfattach_attach failed\n", 414 nvmm_cd.cd_name); 415 return error; 416 } 417 418 error = config_cfdata_attach(nvmm_cfdata, 1); 419 if (error) { 420 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 421 config_cfdriver_detach(&nvmm_cd); 422 aprint_error("%s: unable to register cfdata\n", 423 nvmm_cd.cd_name); 424 return error; 425 } 426 427 if (config_attach_pseudo(nvmm_cfdata) == NULL) { 428 aprint_error("%s: config_attach_pseudo failed\n", 429 nvmm_cd.cd_name); 430 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 431 config_cfdriver_detach(&nvmm_cd); 432 return ENXIO; 433 } 434 435 #if defined(_MODULE) 436 /* mknod /dev/nvmm c 345 0 */ 437 error = devsw_attach(nvmm_cd.cd_name, NULL, &bmajor, 438 &nvmm_cdevsw, &cmajor); 439 if (error) { 440 aprint_error("%s: unable to register devsw\n", 441 nvmm_cd.cd_name); 442 config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 443 config_cfdriver_detach(&nvmm_cd); 444 return error; 445 } 446 #endif 447 return 0; 448 case MODULE_CMD_FINI: 449 error = config_cfdata_detach(nvmm_cfdata); 450 if (error) 451 return error; 452 error = config_cfattach_detach(nvmm_cd.cd_name, &nvmm_ca); 453 if (error) 454 return error; 455 #if defined(_MODULE) 456 config_cfdriver_detach(&nvmm_cd); 457 devsw_detach(NULL, &nvmm_cdevsw); 458 #endif 459 return 0; 460 case MODULE_CMD_AUTOUNLOAD: 461 return EBUSY; 462 default: 463 return ENOTTY; 464 } 465 } 466