1.\" 2.\" Copyright (c) 2018-2021 Maxime Villard, m00nbsd.net 3.\" All rights reserved. 4.\" 5.\" This code is part of the NVMM hypervisor. 6.\" 7.\" Redistribution and use in source and binary forms, with or without 8.\" modification, are permitted provided that the following conditions 9.\" are met: 10.\" 1. Redistributions of source code must retain the above copyright 11.\" notice, this list of conditions and the following disclaimer. 12.\" 2. Redistributions in binary form must reproduce the above copyright 13.\" notice, this list of conditions and the following disclaimer in the 14.\" documentation and/or other materials provided with the distribution. 15.\" 16.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19.\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26.\" SUCH DAMAGE. 27.\" 28.Dd July 20, 2021 29.Dt LIBNVMM 3 30.Os 31.Sh NAME 32.Nm libnvmm 33.Nd NVMM Virtualization API 34.Sh LIBRARY 35.Lb libnvmm 36.Sh SYNOPSIS 37.In nvmm.h 38.Ft int 39.Fn nvmm_init "void" 40.Ft int 41.Fn nvmm_capability "struct nvmm_capability *cap" 42.Ft int 43.Fn nvmm_machine_create "struct nvmm_machine *mach" 44.Ft int 45.Fn nvmm_machine_destroy "struct nvmm_machine *mach" 46.Ft int 47.Fn nvmm_machine_configure "struct nvmm_machine *mach" "uint64_t op" \ 48 "void *conf" 49.Ft int 50.Fn nvmm_vcpu_create "struct nvmm_machine *mach" "nvmm_cpuid_t cpuid" \ 51 "struct nvmm_vcpu *vcpu" 52.Ft int 53.Fn nvmm_vcpu_destroy "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" 54.Ft int 55.Fn nvmm_vcpu_configure "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" \ 56 "uint64_t op" "void *conf" 57.Ft int 58.Fn nvmm_vcpu_getstate "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" \ 59 "uint64_t flags" 60.Ft int 61.Fn nvmm_vcpu_setstate "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" \ 62 "uint64_t flags" 63.Ft int 64.Fn nvmm_vcpu_inject "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" 65.Ft int 66.Fn nvmm_vcpu_run "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" 67.Ft int 68.Fn nvmm_hva_map "struct nvmm_machine *mach" "uintptr_t hva" "size_t size" 69.Ft int 70.Fn nvmm_hva_unmap "struct nvmm_machine *mach" "uintptr_t hva" "size_t size" 71.Ft int 72.Fn nvmm_gpa_map "struct nvmm_machine *mach" "uintptr_t hva" "gpaddr_t gpa" \ 73 "size_t size" "int prot" 74.Ft int 75.Fn nvmm_gpa_unmap "struct nvmm_machine *mach" "uintptr_t hva" "gpaddr_t gpa" \ 76 "size_t size" 77.Ft int 78.Fn nvmm_gva_to_gpa "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" \ 79 "gvaddr_t gva" "gpaddr_t *gpa" "nvmm_prot_t *prot" 80.Ft int 81.Fn nvmm_gpa_to_hva "struct nvmm_machine *mach" "gpaddr_t gpa" \ 82 "uintptr_t *hva" "nvmm_prot_t *prot" 83.Ft int 84.Fn nvmm_assist_io "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" 85.Ft int 86.Fn nvmm_assist_mem "struct nvmm_machine *mach" "struct nvmm_vcpu *vcpu" 87.Sh DESCRIPTION 88.Nm 89provides a library for emulator software to handle hardware-accelerated virtual 90machines in 91.Nx . 92A virtual machine is described by an opaque structure, 93.Cd nvmm_machine . 94Emulator software should not attempt to modify this structure directly, and 95should use the API provided by 96.Nm 97to manage virtual machines. 98A virtual CPU is described by a public structure, 99.Cd nvmm_vcpu . 100.Pp 101.Fn nvmm_init 102initializes NVMM. 103See 104.Sx NVMM Initialization 105below for details. 106.Pp 107.Fn nvmm_capability 108gets the capabilities of NVMM. 109See 110.Sx NVMM Capability 111below for details. 112.Pp 113.Fn nvmm_machine_create 114creates a virtual machine in the kernel. 115The 116.Fa mach 117structure is initialized, and describes the machine. 118.Pp 119.Fn nvmm_machine_destroy 120destroys the virtual machine described in 121.Fa mach . 122.Pp 123.Fn nvmm_machine_configure 124configures, on the machine 125.Fa mach , 126the parameter indicated in 127.Fa op . 128.Fa conf 129describes the value of the parameter. 130.Pp 131.Fn nvmm_vcpu_create 132creates a virtual CPU in the machine 133.Fa mach , 134giving it the CPU id 135.Fa cpuid , 136and initializes 137.Fa vcpu . 138.Pp 139.Fn nvmm_vcpu_destroy 140destroys the virtual CPU identified by 141.Fa vcpu 142in the machine 143.Fa mach . 144.Pp 145.Fn nvmm_vcpu_configure 146configures, on the VCPU 147.Fa vcpu 148of machine 149.Fa mach , 150the parameter indicated in 151.Fa op . 152.Fa conf 153describes the value of the parameter. 154.Pp 155.Fn nvmm_vcpu_getstate 156gets the state of the virtual CPU identified by 157.Fa vcpu 158in the machine 159.Fa mach . 160.Fa flags 161is the bitmap of the components that are to be retrieved. 162The components are located in 163.Fa vcpu->state . 164See 165.Sx VCPU State Area 166below for details. 167.Pp 168.Fn nvmm_vcpu_setstate 169sets the state of the virtual CPU identified by 170.Fa vcpu 171in the machine 172.Fa mach . 173.Fa flags 174is the bitmap of the components that are to be set. 175The components are located in 176.Fa vcpu->state . 177See 178.Sx VCPU State Area 179below for details. 180.Pp 181.Fn nvmm_vcpu_inject 182injects into the CPU identified by 183.Fa vcpu 184of the machine 185.Fa mach 186an event described by 187.Fa vcpu->event . 188See 189.Sx Event Injection 190below for details. 191.Pp 192.Fn nvmm_vcpu_run 193runs the CPU identified by 194.Fa vcpu 195in the machine 196.Fa mach , 197until a VM exit is triggered. 198The 199.Fa vcpu->exit 200structure is filled to indicate the exit reason, and the associated parameters 201if any. 202.Pp 203.Fn nvmm_hva_map 204maps at address 205.Fa hva 206a buffer of size 207.Fa size 208in the calling process' virtual address space. 209This buffer is allowed to be subsequently mapped in a virtual machine. 210.Pp 211.Fn nvmm_hva_unmap 212unmaps the buffer of size 213.Fa size 214at address 215.Fa hva 216from the calling process' virtual address space. 217.Pp 218.Fn nvmm_gpa_map 219maps into the guest physical memory beginning on address 220.Fa gpa 221the buffer of size 222.Fa size 223located at address 224.Fa hva 225of the calling process' virtual address space. 226The 227.Fa hva 228parameter must point to a buffer that was previously mapped with 229.Fn nvmm_hva_map . 230.Pp 231.Fn nvmm_gpa_unmap 232removes the guest physical memory area beginning on address 233.Fa gpa 234and of size 235.Fa size 236from the machine 237.Fa mach . 238.Pp 239.Fn nvmm_gva_to_gpa 240translates, on the CPU 241.Fa vcpu 242from the machine 243.Fa mach , 244the guest virtual address given in 245.Fa gva 246into a guest physical address returned in 247.Fa gpa . 248The associated page premissions are returned in 249.Fa prot . 250.Fa gva 251must be page-aligned. 252.Pp 253.Fn nvmm_gpa_to_hva 254translates, on the machine 255.Fa mach , 256the guest physical address indicated in 257.Fa gpa 258into a host virtual address returned in 259.Fa hva . 260The associated page premissions are returned in 261.Fa prot . 262.Fa gpa 263must be page-aligned. 264.Pp 265.Fn nvmm_assist_io 266emulates the I/O operation described in 267.Fa vcpu->exit 268on CPU 269.Fa vcpu 270from machine 271.Fa mach . 272See 273.Sx I/O Assist 274below for details. 275.Pp 276.Fn nvmm_assist_mem 277emulates the Mem operation described in 278.Fa vcpu->exit 279on CPU 280.Fa vcpu 281from machine 282.Fa mach . 283See 284.Sx Mem Assist 285below for details. 286.Ss NVMM Initialization 287NVMM initialization is performed by the 288.Fn nvmm_init 289function, which must be invoked by emulator software before any other NVMM 290function. 291.Pp 292.Fn nvmm_init 293opens the NVMM device, and expects to have the proper permissions to do so. 294In a default configuration, this implies being part of the "nvmm" group. 295If using a special configuration, emulator software should arrange to have the 296proper permissions before invoking 297.Fn nvmm_init , 298and can drop them after the call has completed. 299.Pp 300It is to be noted that 301.Fn nvmm_init 302may perform non-re-entrant operations, and should be called only once. 303.Ss NVMM Capability 304The 305.Cd nvmm_capability 306structure helps emulator software identify the capabilities offered by NVMM on 307the host: 308.Bd -literal 309struct nvmm_capability { 310 uint64_t version; 311 uint64_t state_size; 312 uint64_t comm_size; 313 uint64_t max_machines; 314 uint64_t max_vcpus; 315 uint64_t max_ram; 316 struct { 317 ... 318 } arch; 319}; 320.Ed 321.Pp 322For example, the 323.Cd max_machines 324field indicates the maximum number of virtual machines supported, while 325.Cd max_vcpus 326indicates the maximum number of VCPUs supported per virtual machine. 327.Ss Machine Ownership 328When a process creates a virtual machine via 329.Fn nvmm_machine_create , 330it is considered the owner of this machine. 331No other processes than the owner can operate a virtual machine. 332.Pp 333When an owner exits, all the virtual machines associated with it are destroyed, 334if they were not already destroyed by the owner itself via 335.Fn nvmm_machine_destroy . 336.Pp 337Virtual machines are not inherited across 338.Xr fork 2 339operations. 340.Ss Machine Configuration 341Emulator software can configure several parameters of a virtual machine by using 342.Fn nvmm_machine_configure . 343Currently, no parameters are implemented. 344.Ss VCPU Configuration 345Emulator software can configure several parameters of a VCPU by using 346.Fn nvmm_vcpu_configure , 347which can take the following operations: 348.Pp 349.Bl -bullet -offset indent -compact 350.It 351.Cd NVMM_VCPU_CONF_CALLBACKS : 352register assist callbacks. 353See 354.Sx Assist Callbacks 355below for details. 356.El 357.Pp 358The other fields depend on the architecture. 359.Pp 360On x86 there are two additional operations available: 361.Pp 362.Bl -bullet -offset indent -compact 363.It 364.Cd NVMM_VCPU_CONF_CPUID : 365configure the information returned to the guest by the CPUID instruction. 366.It 367.Cd NVMM_VCPU_CONF_TPR : 368configure whether to return to the emulator when the guest updates its TPR. 369.El 370.Ss Guest-Host Mappings 371Each virtual machine has an associated guest physical memory. 372Emulator software is allowed to modify this guest physical memory by mapping 373it into some parts of its virtual address space. 374.Pp 375Emulator software should follow the following steps to achieve that: 376.Pp 377.Bl -bullet -offset indent -compact 378.It 379Call 380.Fn nvmm_hva_map 381to create in the host's virtual address space an area of memory that can 382be shared with a guest. 383Typically, the 384.Fa hva 385parameter will be a pointer to an area that was previously mapped via 386.Fn mmap . 387.Fn nvmm_hva_map 388will replace the content of the area, and will make it read-write (but not 389executable). 390.It 391Make available in the guest an area of guest physical memory, by calling 392.Fn nvmm_gpa_map 393and passing in the 394.Fa hva 395parameter the value that was previously given to 396.Fn nvmm_hva_map . 397.Fn nvmm_gpa_map 398does not replace the content of any memory, it only creates a direct link 399from 400.Fa gpa 401into 402.Fa hva . 403.Fn nvmm_gpa_unmap 404removes this link without modifying 405.Fa hva . 406.El 407.Pp 408The guest will then be able to use the guest physical address passed in the 409.Fa gpa 410parameter of 411.Fn nvmm_gpa_map . 412Each change the guest makes in 413.Fa gpa 414will be reflected in the host's 415.Fa hva , 416and vice versa. 417.Pp 418It is illegal for emulator software to use 419.Fn munmap 420on an area that was mapped via 421.Fn nvmm_hva_map . 422.Ss VCPU State Area 423A VCPU state area is a structure that entirely defines the content of the 424registers of a VCPU. 425Only one such structure exists, for x86: 426.Bd -literal 427struct nvmm_x64_state { 428 struct nvmm_x64_state_seg segs[NVMM_X64_NSEG]; 429 uint64_t gprs[NVMM_X64_NGPR]; 430 uint64_t crs[NVMM_X64_NCR]; 431 uint64_t drs[NVMM_X64_NDR]; 432 uint64_t msrs[NVMM_X64_NMSR]; 433 struct nvmm_x64_state_intr intr; 434 struct nvmm_x64_state_fpu fpu; 435}; 436#define nvmm_vcpu_state nvmm_x64_state 437.Ed 438.Pp 439Refer to functional examples to see precisely how to use this structure. 440.Pp 441A VCPU state area is divided in sub-states. 442A 443.Fa flags 444parameter is used to get and set the VCPU state; it acts as a bitmap which 445indicates which sub-states to get or set. 446.Pp 447During VM exits, a partial VCPU state area is provided in 448.Va exitstate , 449see 450.Sx Exit Reasons 451below for details. 452.Ss VCPU Programming Model 453A VCPU is described by a public structure, 454.Cd nvmm_vcpu : 455.Bd -literal 456struct nvmm_vcpu { 457 nvmm_cpuid_t cpuid; 458 struct nvmm_vcpu_state *state; 459 struct nvmm_vcpu_event *event; 460 struct nvmm_vcpu_exit *exit; 461}; 462.Ed 463.Pp 464This structure is used both publicly by emulator software and internally by 465.Nm . 466Emulator software should not modify the pointers of this structure, because 467they are initialized to special values by 468.Nm . 469.Pp 470A call to 471.Fn nvmm_vcpu_getstate 472will fetch the desired parts of the VCPU state and put them in 473.Fa vcpu->state . 474A call to 475.Fn nvmm_vcpu_setstate 476will install in the VCPU the desired parts of 477.Fa vcpu->state . 478A call to 479.Fn nvmm_vcpu_inject 480will inject in the VCPU the event in 481.Fa vcpu->event . 482A call to 483.Fn nvmm_vcpu_run 484will fill 485.Fa vcpu->exit 486with the VCPU exit information. 487.Pp 488If emulator software uses several threads, a VCPU should be associated with 489only one thread, and only this thread should perform VCPU modifications. 490Emulator software should not modify the state of a VCPU with several 491different threads. 492.Ss Exit Reasons 493The 494.Cd nvmm_vcpu_exit 495structure is used to handle VM exits: 496.Bd -literal 497/* Generic. */ 498#define NVMM_VCPU_EXIT_NONE 0x0000000000000000ULL 499#define NVMM_VCPU_EXIT_INVALID 0xFFFFFFFFFFFFFFFFULL 500/* x86: operations. */ 501#define NVMM_VCPU_EXIT_MEMORY 0x0000000000000001ULL 502#define NVMM_VCPU_EXIT_IO 0x0000000000000002ULL 503/* x86: changes in VCPU state. */ 504#define NVMM_VCPU_EXIT_SHUTDOWN 0x0000000000001000ULL 505#define NVMM_VCPU_EXIT_INT_READY 0x0000000000001001ULL 506#define NVMM_VCPU_EXIT_NMI_READY 0x0000000000001002ULL 507#define NVMM_VCPU_EXIT_HALTED 0x0000000000001003ULL 508#define NVMM_VCPU_EXIT_TPR_CHANGED 0x0000000000001004ULL 509/* x86: instructions. */ 510#define NVMM_VCPU_EXIT_RDMSR 0x0000000000002000ULL 511#define NVMM_VCPU_EXIT_WRMSR 0x0000000000002001ULL 512#define NVMM_VCPU_EXIT_MONITOR 0x0000000000002002ULL 513#define NVMM_VCPU_EXIT_MWAIT 0x0000000000002003ULL 514#define NVMM_VCPU_EXIT_CPUID 0x0000000000002004ULL 515 516struct nvmm_vcpu_exit { 517 uint64_t reason; 518 union { 519 ... 520 } u; 521 struct { 522 ... 523 } exitstate; 524}; 525.Ed 526.Pp 527The 528.Va reason 529field indicates the reason of the VM exit. 530Additional parameters describing the exit can be present in 531.Va u . 532.Va exitstate 533contains a partial, implementation-specific VCPU state, usable as a fast-path 534to retrieve certain state values. 535.Pp 536It is possible that a VM exit was caused by a reason internal to the host 537kernel, and that emulator software should not be concerned with. 538In this case, the exit reason is set to 539.Cd NVMM_VCPU_EXIT_NONE . 540This gives a chance for emulator software to halt the VM in its tracks. 541.Pp 542Refer to functional examples to see precisely how to handle VM exits. 543.Ss Event Injection 544It is possible to inject an event into a VCPU. 545An event can be a hardware interrupt, a software interrupt, or a software 546exception, defined by: 547.Bd -literal 548#define NVMM_VCPU_EVENT_EXCP 0 549#define NVMM_VCPU_EVENT_INTR 1 550 551struct nvmm_vcpu_event { 552 u_int type; 553 uint8_t vector; 554 union { 555 struct { 556 uint64_t error; 557 } excp; 558 } u; 559}; 560.Ed 561.Pp 562This describes an event of type 563.Va type , 564to be sent to vector number 565.Va vector , 566with a possible additional 567.Va error 568code that is implementation-specific. 569.Pp 570It is possible that the VCPU is in a state where it cannot receive this 571event, if: 572.Pp 573.Bl -bullet -offset indent -compact 574.It 575the event is a hardware interrupt, and the VCPU runs with interrupts disabled, 576or 577.It 578the event is a non-maskable interrupt (NMI), and the VCPU is already in an 579in-NMI context. 580.El 581.Pp 582Emulator software can manage interrupt and NMI window-exiting via the 583.Va intr 584component of the VCPU state. 585When such window-exiting is enabled, NVMM will cause a VM exit with reason 586.Cd NVMM_VCPU_EXIT_INT_READY 587or 588.Cd NVMM_VCPU_EXIT_NMI_READY 589to indicate that the guest is now able to handle the corresponding class 590of interrupts. 591.Ss Assist Callbacks 592In order to assist emulation of certain operations, 593.Nm 594requires emulator software to register, via 595.Fn nvmm_vcpu_configure , 596a set of callbacks described in the following structure: 597.Bd -literal 598struct nvmm_assist_callbacks { 599 void (*io)(struct nvmm_io *); 600 void (*mem)(struct nvmm_mem *); 601}; 602.Ed 603.Pp 604These callbacks are used by 605.Nm 606each time 607.Fn nvmm_assist_io 608or 609.Fn nvmm_assist_mem 610are invoked. 611Emulator software that does not intend to use either of these assists can put 612.Dv NULL 613in the callbacks. 614.Ss I/O Assist 615When a VM exit occurs with reason 616.Cd NVMM_VCPU_EXIT_IO , 617it is necessary for emulator software to emulate the associated I/O operation. 618.Nm 619provides an easy way for emulator software to perform that. 620.Pp 621.Fn nvmm_assist_io 622will call the registered 623.Fa io 624callback function and give it a 625.Cd nvmm_io 626structure as argument. 627This structure describes an I/O transaction: 628.Bd -literal 629struct nvmm_io { 630 struct nvmm_machine *mach; 631 struct nvmm_vcpu *vcpu; 632 uint16_t port; 633 bool in; 634 size_t size; 635 uint8_t *data; 636}; 637.Ed 638.Pp 639The callback can emulate the operation using this descriptor, following two 640unique cases: 641.Pp 642.Bl -bullet -offset indent -compact 643.It 644The operation is an input. 645In this case, the callback should fill 646.Va data 647with the desired value. 648.It 649The operation is an output. 650In this case, the callback should read 651.Va data 652to retrieve the desired value. 653.El 654.Pp 655In either case, 656.Va port 657will indicate the I/O port, 658.Va in 659will indicate if the operation is an input, and 660.Va size 661will indicate the size of the access. 662.Ss Mem Assist 663When a VM exit occurs with reason 664.Cd NVMM_VCPU_EXIT_MEMORY , 665it is necessary for emulator software to emulate the associated memory 666operation. 667.Nm 668provides an easy way for emulator software to perform that, similar to the I/O 669Assist. 670.Pp 671.Fn nvmm_assist_mem 672will call the registered 673.Fa mem 674callback function and give it a 675.Cd nvmm_mem 676structure as argument. 677This structure describes a Mem transaction: 678.Bd -literal 679struct nvmm_mem { 680 struct nvmm_machine *mach; 681 struct nvmm_vcpu *vcpu; 682 gpaddr_t gpa; 683 bool write; 684 size_t size; 685 uint8_t *data; 686}; 687.Ed 688.Pp 689The callback can emulate the operation using this descriptor, following two 690unique cases: 691.Pp 692.Bl -bullet -offset indent -compact 693.It 694The operation is a read. 695In this case, the callback should fill 696.Va data 697with the desired value. 698.It 699The operation is a write. 700In this case, the callback should read 701.Va data 702to retrieve the desired value. 703.El 704.Pp 705In either case, 706.Va gpa 707will indicate the guest physical address, 708.Va write 709will indicate if the access is a write, and 710.Va size 711will indicate the size of the access. 712.Sh RETURN VALUES 713Upon successful completion, each of these functions returns zero. 714Otherwise, a value of \-1 is returned and the global 715variable 716.Va errno 717is set to indicate the error. 718.Sh FILES 719.Bl -tag -width indent 720.It Pa src/lib/libnvmm/ 721Source code of the 722.Nm 723library. 724.It Pa src/sys/dev/virtual/nvmm/ 725Source code of the kernel 726.Xr nvmm 4 727driver. 728.It Pa src/test/testcases/libnvmm 729Regression test cases for the 730.Nm 731library. 732.It Pa src/test/nvmm/calc-vm.c 733A minimal exmaple that uses the 734.Nm 735API to create a VM and perform a calculation within it. 736.It Pa src/test/nvmm/demo 737Functional demonstrator. 738Contains an emulator that uses the 739.Nm 740API, and a small kernel that exercises this emulator. 741.Pp 742Originally obtained from 743.Lk https://www.netbsd.org/~maxv/nvmm/nvmm-demo.zip 744but has been updated to match the current 745.Nm 746API, cleaned up, and ported to 747.Dx . 748.El 749.Sh ERRORS 750These functions will fail if: 751.Bl -tag -width Er 752.It Bq Er EEXIST 753An attempt was made to create a machine or a VCPU that already exists. 754.It Bq Er EFAULT 755An attempt was made to emulate a memory-based operation in a guest, and the 756guest page tables did not have the permissions necessary for the operation 757to complete successfully. 758.It Bq Er EINVAL 759An inappropriate parameter was used. 760.It Bq Er ENOBUFS 761The maximum number of machines or VCPUs was reached. 762.It Bq Er ENOENT 763A query was made on a machine or a VCPU that does not exist. 764.It Bq Er EPERM 765An attempt was made to access a machine that does not belong to the process. 766.El 767.Sh SEE ALSO 768.Xr nvmm 4 , 769.Xr nvmmctl 8 770.Sh AUTHORS 771NVMM was designed and implemented by 772.An Maxime Villard . 773