1 /* $NetBSD: libnvmm_x86.c,v 1.43 2020/12/27 20:56:14 reinoud Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the NVMM hypervisor. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 #include <sys/ioctl.h> 33 #include <sys/mman.h> 34 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <string.h> 38 #include <unistd.h> 39 #include <fcntl.h> 40 #include <errno.h> 41 42 #include <machine/psl.h> 43 #include <machine/specialreg.h> 44 45 /* -------------------------------------------------------------------------- */ 46 47 /* 48 * Undocumented debugging function. Helpful. 49 */ 50 int 51 nvmm_vcpu_dump(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 52 { 53 struct nvmm_x64_state *state = vcpu->state; 54 uint16_t *attr; 55 size_t i; 56 int ret; 57 58 const char *segnames[] = { 59 "ES", "CS", "SS", "DS", "FS", "GS", "GDT", "IDT", "LDT", "TR" 60 }; 61 62 ret = nvmm_vcpu_getstate(mach, vcpu, NVMM_X64_STATE_ALL); 63 if (ret == -1) 64 return -1; 65 66 printf("+ VCPU id=%d\n", (int)vcpu->cpuid); 67 printf("| -> RAX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RAX]); 68 printf("| -> RCX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RCX]); 69 printf("| -> RDX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RDX]); 70 printf("| -> RBX=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RBX]); 71 printf("| -> RSP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RSP]); 72 printf("| -> RBP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RBP]); 73 printf("| -> RSI=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RSI]); 74 printf("| -> RDI=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RDI]); 75 printf("| -> RIP=%"PRIx64"\n", state->gprs[NVMM_X64_GPR_RIP]); 76 printf("| -> RFLAGS=%p\n", (void *)state->gprs[NVMM_X64_GPR_RFLAGS]); 77 for (i = 0; i < NVMM_X64_NSEG; i++) { 78 attr = (uint16_t *)&state->segs[i].attrib; 79 printf("| -> %s: sel=0x%x base=%"PRIx64", limit=%x, " 80 "attrib=%x [type=%d,l=%d,def=%d]\n", 81 segnames[i], 82 state->segs[i].selector, 83 state->segs[i].base, 84 state->segs[i].limit, 85 *attr, 86 state->segs[i].attrib.type, 87 state->segs[i].attrib.l, 88 state->segs[i].attrib.def); 89 } 90 printf("| -> MSR_EFER=%"PRIx64"\n", state->msrs[NVMM_X64_MSR_EFER]); 91 printf("| -> CR0=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR0]); 92 printf("| -> CR3=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR3]); 93 printf("| -> CR4=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR4]); 94 printf("| -> CR8=%"PRIx64"\n", state->crs[NVMM_X64_CR_CR8]); 95 96 return 0; 97 } 98 99 /* -------------------------------------------------------------------------- */ 100 101 #define PTE32_L1_SHIFT 12 102 #define PTE32_L2_SHIFT 22 103 104 #define PTE32_L2_MASK 0xffc00000 105 #define PTE32_L1_MASK 0x003ff000 106 107 #define PTE32_L2_FRAME (PTE32_L2_MASK) 108 #define PTE32_L1_FRAME (PTE32_L2_FRAME|PTE32_L1_MASK) 109 110 #define pte32_l1idx(va) (((va) & PTE32_L1_MASK) >> PTE32_L1_SHIFT) 111 #define pte32_l2idx(va) (((va) & PTE32_L2_MASK) >> PTE32_L2_SHIFT) 112 113 #define CR3_FRAME_32BIT __BITS(31, 12) 114 115 typedef uint32_t pte_32bit_t; 116 117 static int 118 x86_gva_to_gpa_32bit(struct nvmm_machine *mach, uint64_t cr3, 119 gvaddr_t gva, gpaddr_t *gpa, bool has_pse, nvmm_prot_t *prot) 120 { 121 gpaddr_t L2gpa, L1gpa; 122 uintptr_t L2hva, L1hva; 123 pte_32bit_t *pdir, pte; 124 nvmm_prot_t pageprot; 125 126 /* We begin with an RWXU access. */ 127 *prot = NVMM_PROT_ALL; 128 129 /* Parse L2. */ 130 L2gpa = (cr3 & CR3_FRAME_32BIT); 131 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 132 return -1; 133 pdir = (pte_32bit_t *)L2hva; 134 pte = pdir[pte32_l2idx(gva)]; 135 if ((pte & PTE_P) == 0) 136 return -1; 137 if ((pte & PTE_U) == 0) 138 *prot &= ~NVMM_PROT_USER; 139 if ((pte & PTE_W) == 0) 140 *prot &= ~NVMM_PROT_WRITE; 141 if ((pte & PTE_PS) && !has_pse) 142 return -1; 143 if (pte & PTE_PS) { 144 *gpa = (pte & PTE32_L2_FRAME); 145 *gpa = *gpa + (gva & PTE32_L1_MASK); 146 return 0; 147 } 148 149 /* Parse L1. */ 150 L1gpa = (pte & PTE_FRAME); 151 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 152 return -1; 153 pdir = (pte_32bit_t *)L1hva; 154 pte = pdir[pte32_l1idx(gva)]; 155 if ((pte & PTE_P) == 0) 156 return -1; 157 if ((pte & PTE_U) == 0) 158 *prot &= ~NVMM_PROT_USER; 159 if ((pte & PTE_W) == 0) 160 *prot &= ~NVMM_PROT_WRITE; 161 if (pte & PTE_PS) 162 return -1; 163 164 *gpa = (pte & PTE_FRAME); 165 return 0; 166 } 167 168 /* -------------------------------------------------------------------------- */ 169 170 #define PTE32_PAE_L1_SHIFT 12 171 #define PTE32_PAE_L2_SHIFT 21 172 #define PTE32_PAE_L3_SHIFT 30 173 174 #define PTE32_PAE_L3_MASK 0xc0000000 175 #define PTE32_PAE_L2_MASK 0x3fe00000 176 #define PTE32_PAE_L1_MASK 0x001ff000 177 178 #define PTE32_PAE_L3_FRAME (PTE32_PAE_L3_MASK) 179 #define PTE32_PAE_L2_FRAME (PTE32_PAE_L3_FRAME|PTE32_PAE_L2_MASK) 180 #define PTE32_PAE_L1_FRAME (PTE32_PAE_L2_FRAME|PTE32_PAE_L1_MASK) 181 182 #define pte32_pae_l1idx(va) (((va) & PTE32_PAE_L1_MASK) >> PTE32_PAE_L1_SHIFT) 183 #define pte32_pae_l2idx(va) (((va) & PTE32_PAE_L2_MASK) >> PTE32_PAE_L2_SHIFT) 184 #define pte32_pae_l3idx(va) (((va) & PTE32_PAE_L3_MASK) >> PTE32_PAE_L3_SHIFT) 185 186 #define CR3_FRAME_32BIT_PAE __BITS(31, 5) 187 188 typedef uint64_t pte_32bit_pae_t; 189 190 static int 191 x86_gva_to_gpa_32bit_pae(struct nvmm_machine *mach, uint64_t cr3, 192 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 193 { 194 gpaddr_t L3gpa, L2gpa, L1gpa; 195 uintptr_t L3hva, L2hva, L1hva; 196 pte_32bit_pae_t *pdir, pte; 197 nvmm_prot_t pageprot; 198 199 /* We begin with an RWXU access. */ 200 *prot = NVMM_PROT_ALL; 201 202 /* Parse L3. */ 203 L3gpa = (cr3 & CR3_FRAME_32BIT_PAE); 204 if (nvmm_gpa_to_hva(mach, L3gpa, &L3hva, &pageprot) == -1) 205 return -1; 206 pdir = (pte_32bit_pae_t *)L3hva; 207 pte = pdir[pte32_pae_l3idx(gva)]; 208 if ((pte & PTE_P) == 0) 209 return -1; 210 if (pte & PTE_NX) 211 *prot &= ~NVMM_PROT_EXEC; 212 if (pte & PTE_PS) 213 return -1; 214 215 /* Parse L2. */ 216 L2gpa = (pte & PTE_FRAME); 217 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 218 return -1; 219 pdir = (pte_32bit_pae_t *)L2hva; 220 pte = pdir[pte32_pae_l2idx(gva)]; 221 if ((pte & PTE_P) == 0) 222 return -1; 223 if ((pte & PTE_U) == 0) 224 *prot &= ~NVMM_PROT_USER; 225 if ((pte & PTE_W) == 0) 226 *prot &= ~NVMM_PROT_WRITE; 227 if (pte & PTE_NX) 228 *prot &= ~NVMM_PROT_EXEC; 229 if (pte & PTE_PS) { 230 *gpa = (pte & PTE32_PAE_L2_FRAME); 231 *gpa = *gpa + (gva & PTE32_PAE_L1_MASK); 232 return 0; 233 } 234 235 /* Parse L1. */ 236 L1gpa = (pte & PTE_FRAME); 237 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 238 return -1; 239 pdir = (pte_32bit_pae_t *)L1hva; 240 pte = pdir[pte32_pae_l1idx(gva)]; 241 if ((pte & PTE_P) == 0) 242 return -1; 243 if ((pte & PTE_U) == 0) 244 *prot &= ~NVMM_PROT_USER; 245 if ((pte & PTE_W) == 0) 246 *prot &= ~NVMM_PROT_WRITE; 247 if (pte & PTE_NX) 248 *prot &= ~NVMM_PROT_EXEC; 249 if (pte & PTE_PS) 250 return -1; 251 252 *gpa = (pte & PTE_FRAME); 253 return 0; 254 } 255 256 /* -------------------------------------------------------------------------- */ 257 258 #define PTE64_L1_SHIFT 12 259 #define PTE64_L2_SHIFT 21 260 #define PTE64_L3_SHIFT 30 261 #define PTE64_L4_SHIFT 39 262 263 #define PTE64_L4_MASK 0x0000ff8000000000 264 #define PTE64_L3_MASK 0x0000007fc0000000 265 #define PTE64_L2_MASK 0x000000003fe00000 266 #define PTE64_L1_MASK 0x00000000001ff000 267 268 #define PTE64_L4_FRAME PTE64_L4_MASK 269 #define PTE64_L3_FRAME (PTE64_L4_FRAME|PTE64_L3_MASK) 270 #define PTE64_L2_FRAME (PTE64_L3_FRAME|PTE64_L2_MASK) 271 #define PTE64_L1_FRAME (PTE64_L2_FRAME|PTE64_L1_MASK) 272 273 #define pte64_l1idx(va) (((va) & PTE64_L1_MASK) >> PTE64_L1_SHIFT) 274 #define pte64_l2idx(va) (((va) & PTE64_L2_MASK) >> PTE64_L2_SHIFT) 275 #define pte64_l3idx(va) (((va) & PTE64_L3_MASK) >> PTE64_L3_SHIFT) 276 #define pte64_l4idx(va) (((va) & PTE64_L4_MASK) >> PTE64_L4_SHIFT) 277 278 #define CR3_FRAME_64BIT __BITS(51, 12) 279 280 typedef uint64_t pte_64bit_t; 281 282 static inline bool 283 x86_gva_64bit_canonical(gvaddr_t gva) 284 { 285 /* Bits 63:47 must have the same value. */ 286 #define SIGN_EXTEND 0xffff800000000000ULL 287 return (gva & SIGN_EXTEND) == 0 || (gva & SIGN_EXTEND) == SIGN_EXTEND; 288 } 289 290 static int 291 x86_gva_to_gpa_64bit(struct nvmm_machine *mach, uint64_t cr3, 292 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 293 { 294 gpaddr_t L4gpa, L3gpa, L2gpa, L1gpa; 295 uintptr_t L4hva, L3hva, L2hva, L1hva; 296 pte_64bit_t *pdir, pte; 297 nvmm_prot_t pageprot; 298 299 /* We begin with an RWXU access. */ 300 *prot = NVMM_PROT_ALL; 301 302 if (!x86_gva_64bit_canonical(gva)) 303 return -1; 304 305 /* Parse L4. */ 306 L4gpa = (cr3 & CR3_FRAME_64BIT); 307 if (nvmm_gpa_to_hva(mach, L4gpa, &L4hva, &pageprot) == -1) 308 return -1; 309 pdir = (pte_64bit_t *)L4hva; 310 pte = pdir[pte64_l4idx(gva)]; 311 if ((pte & PTE_P) == 0) 312 return -1; 313 if ((pte & PTE_U) == 0) 314 *prot &= ~NVMM_PROT_USER; 315 if ((pte & PTE_W) == 0) 316 *prot &= ~NVMM_PROT_WRITE; 317 if (pte & PTE_NX) 318 *prot &= ~NVMM_PROT_EXEC; 319 if (pte & PTE_PS) 320 return -1; 321 322 /* Parse L3. */ 323 L3gpa = (pte & PTE_FRAME); 324 if (nvmm_gpa_to_hva(mach, L3gpa, &L3hva, &pageprot) == -1) 325 return -1; 326 pdir = (pte_64bit_t *)L3hva; 327 pte = pdir[pte64_l3idx(gva)]; 328 if ((pte & PTE_P) == 0) 329 return -1; 330 if ((pte & PTE_U) == 0) 331 *prot &= ~NVMM_PROT_USER; 332 if ((pte & PTE_W) == 0) 333 *prot &= ~NVMM_PROT_WRITE; 334 if (pte & PTE_NX) 335 *prot &= ~NVMM_PROT_EXEC; 336 if (pte & PTE_PS) { 337 *gpa = (pte & PTE64_L3_FRAME); 338 *gpa = *gpa + (gva & (PTE64_L2_MASK|PTE64_L1_MASK)); 339 return 0; 340 } 341 342 /* Parse L2. */ 343 L2gpa = (pte & PTE_FRAME); 344 if (nvmm_gpa_to_hva(mach, L2gpa, &L2hva, &pageprot) == -1) 345 return -1; 346 pdir = (pte_64bit_t *)L2hva; 347 pte = pdir[pte64_l2idx(gva)]; 348 if ((pte & PTE_P) == 0) 349 return -1; 350 if ((pte & PTE_U) == 0) 351 *prot &= ~NVMM_PROT_USER; 352 if ((pte & PTE_W) == 0) 353 *prot &= ~NVMM_PROT_WRITE; 354 if (pte & PTE_NX) 355 *prot &= ~NVMM_PROT_EXEC; 356 if (pte & PTE_PS) { 357 *gpa = (pte & PTE64_L2_FRAME); 358 *gpa = *gpa + (gva & PTE64_L1_MASK); 359 return 0; 360 } 361 362 /* Parse L1. */ 363 L1gpa = (pte & PTE_FRAME); 364 if (nvmm_gpa_to_hva(mach, L1gpa, &L1hva, &pageprot) == -1) 365 return -1; 366 pdir = (pte_64bit_t *)L1hva; 367 pte = pdir[pte64_l1idx(gva)]; 368 if ((pte & PTE_P) == 0) 369 return -1; 370 if ((pte & PTE_U) == 0) 371 *prot &= ~NVMM_PROT_USER; 372 if ((pte & PTE_W) == 0) 373 *prot &= ~NVMM_PROT_WRITE; 374 if (pte & PTE_NX) 375 *prot &= ~NVMM_PROT_EXEC; 376 if (pte & PTE_PS) 377 return -1; 378 379 *gpa = (pte & PTE_FRAME); 380 return 0; 381 } 382 383 static inline int 384 x86_gva_to_gpa(struct nvmm_machine *mach, struct nvmm_x64_state *state, 385 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 386 { 387 bool is_pae, is_lng, has_pse; 388 uint64_t cr3; 389 size_t off; 390 int ret; 391 392 if ((state->crs[NVMM_X64_CR_CR0] & CR0_PG) == 0) { 393 /* No paging. */ 394 *prot = NVMM_PROT_ALL; 395 *gpa = gva; 396 return 0; 397 } 398 399 off = (gva & PAGE_MASK); 400 gva &= ~PAGE_MASK; 401 402 is_pae = (state->crs[NVMM_X64_CR_CR4] & CR4_PAE) != 0; 403 is_lng = (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) != 0; 404 has_pse = (state->crs[NVMM_X64_CR_CR4] & CR4_PSE) != 0; 405 cr3 = state->crs[NVMM_X64_CR_CR3]; 406 407 if (is_pae && is_lng) { 408 /* 64bit */ 409 ret = x86_gva_to_gpa_64bit(mach, cr3, gva, gpa, prot); 410 } else if (is_pae && !is_lng) { 411 /* 32bit PAE */ 412 ret = x86_gva_to_gpa_32bit_pae(mach, cr3, gva, gpa, prot); 413 } else if (!is_pae && !is_lng) { 414 /* 32bit */ 415 ret = x86_gva_to_gpa_32bit(mach, cr3, gva, gpa, has_pse, prot); 416 } else { 417 ret = -1; 418 } 419 420 if (ret == -1) { 421 errno = EFAULT; 422 } 423 424 *gpa = *gpa + off; 425 426 return ret; 427 } 428 429 int 430 nvmm_gva_to_gpa(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 431 gvaddr_t gva, gpaddr_t *gpa, nvmm_prot_t *prot) 432 { 433 struct nvmm_x64_state *state = vcpu->state; 434 int ret; 435 436 ret = nvmm_vcpu_getstate(mach, vcpu, 437 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 438 if (ret == -1) 439 return -1; 440 441 return x86_gva_to_gpa(mach, state, gva, gpa, prot); 442 } 443 444 /* -------------------------------------------------------------------------- */ 445 446 #define DISASSEMBLER_BUG() \ 447 do { \ 448 errno = EINVAL; \ 449 return -1; \ 450 } while (0); 451 452 static inline bool 453 is_long_mode(struct nvmm_x64_state *state) 454 { 455 return (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) != 0; 456 } 457 458 static inline bool 459 is_64bit(struct nvmm_x64_state *state) 460 { 461 return (state->segs[NVMM_X64_SEG_CS].attrib.l != 0); 462 } 463 464 static inline bool 465 is_32bit(struct nvmm_x64_state *state) 466 { 467 return (state->segs[NVMM_X64_SEG_CS].attrib.l == 0) && 468 (state->segs[NVMM_X64_SEG_CS].attrib.def == 1); 469 } 470 471 static inline bool 472 is_16bit(struct nvmm_x64_state *state) 473 { 474 return (state->segs[NVMM_X64_SEG_CS].attrib.l == 0) && 475 (state->segs[NVMM_X64_SEG_CS].attrib.def == 0); 476 } 477 478 static int 479 segment_check(struct nvmm_x64_state_seg *seg, gvaddr_t gva, size_t size) 480 { 481 uint64_t limit; 482 483 /* 484 * This is incomplete. We should check topdown, etc, really that's 485 * tiring. 486 */ 487 if (__predict_false(!seg->attrib.p)) { 488 goto error; 489 } 490 491 limit = (uint64_t)seg->limit + 1; 492 if (__predict_true(seg->attrib.g)) { 493 limit *= PAGE_SIZE; 494 } 495 496 if (__predict_false(gva + size > limit)) { 497 goto error; 498 } 499 500 return 0; 501 502 error: 503 errno = EFAULT; 504 return -1; 505 } 506 507 static inline void 508 segment_apply(struct nvmm_x64_state_seg *seg, gvaddr_t *gva) 509 { 510 *gva += seg->base; 511 } 512 513 static inline uint64_t 514 size_to_mask(size_t size) 515 { 516 switch (size) { 517 case 1: 518 return 0x00000000000000FF; 519 case 2: 520 return 0x000000000000FFFF; 521 case 4: 522 return 0x00000000FFFFFFFF; 523 case 8: 524 default: 525 return 0xFFFFFFFFFFFFFFFF; 526 } 527 } 528 529 static uint64_t 530 rep_get_cnt(struct nvmm_x64_state *state, size_t adsize) 531 { 532 uint64_t mask, cnt; 533 534 mask = size_to_mask(adsize); 535 cnt = state->gprs[NVMM_X64_GPR_RCX] & mask; 536 537 return cnt; 538 } 539 540 static void 541 rep_set_cnt(struct nvmm_x64_state *state, size_t adsize, uint64_t cnt) 542 { 543 uint64_t mask; 544 545 /* XXX: should we zero-extend? */ 546 mask = size_to_mask(adsize); 547 state->gprs[NVMM_X64_GPR_RCX] &= ~mask; 548 state->gprs[NVMM_X64_GPR_RCX] |= cnt; 549 } 550 551 static int 552 read_guest_memory(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 553 gvaddr_t gva, uint8_t *data, size_t size) 554 { 555 struct nvmm_x64_state *state = vcpu->state; 556 struct nvmm_mem mem; 557 nvmm_prot_t prot; 558 gpaddr_t gpa; 559 uintptr_t hva; 560 bool is_mmio; 561 int ret, remain; 562 563 ret = x86_gva_to_gpa(mach, state, gva, &gpa, &prot); 564 if (__predict_false(ret == -1)) { 565 return -1; 566 } 567 if (__predict_false(!(prot & NVMM_PROT_READ))) { 568 errno = EFAULT; 569 return -1; 570 } 571 572 if ((gva & PAGE_MASK) + size > PAGE_SIZE) { 573 remain = ((gva & PAGE_MASK) + size - PAGE_SIZE); 574 } else { 575 remain = 0; 576 } 577 size -= remain; 578 579 ret = nvmm_gpa_to_hva(mach, gpa, &hva, &prot); 580 is_mmio = (ret == -1); 581 582 if (is_mmio) { 583 mem.mach = mach; 584 mem.vcpu = vcpu; 585 mem.data = data; 586 mem.gpa = gpa; 587 mem.write = false; 588 mem.size = size; 589 (*vcpu->cbs.mem)(&mem); 590 } else { 591 if (__predict_false(!(prot & NVMM_PROT_READ))) { 592 errno = EFAULT; 593 return -1; 594 } 595 memcpy(data, (uint8_t *)hva, size); 596 } 597 598 if (remain > 0) { 599 ret = read_guest_memory(mach, vcpu, gva + size, 600 data + size, remain); 601 } else { 602 ret = 0; 603 } 604 605 return ret; 606 } 607 608 static int 609 write_guest_memory(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 610 gvaddr_t gva, uint8_t *data, size_t size) 611 { 612 struct nvmm_x64_state *state = vcpu->state; 613 struct nvmm_mem mem; 614 nvmm_prot_t prot; 615 gpaddr_t gpa; 616 uintptr_t hva; 617 bool is_mmio; 618 int ret, remain; 619 620 ret = x86_gva_to_gpa(mach, state, gva, &gpa, &prot); 621 if (__predict_false(ret == -1)) { 622 return -1; 623 } 624 if (__predict_false(!(prot & NVMM_PROT_WRITE))) { 625 errno = EFAULT; 626 return -1; 627 } 628 629 if ((gva & PAGE_MASK) + size > PAGE_SIZE) { 630 remain = ((gva & PAGE_MASK) + size - PAGE_SIZE); 631 } else { 632 remain = 0; 633 } 634 size -= remain; 635 636 ret = nvmm_gpa_to_hva(mach, gpa, &hva, &prot); 637 is_mmio = (ret == -1); 638 639 if (is_mmio) { 640 mem.mach = mach; 641 mem.vcpu = vcpu; 642 mem.data = data; 643 mem.gpa = gpa; 644 mem.write = true; 645 mem.size = size; 646 (*vcpu->cbs.mem)(&mem); 647 } else { 648 if (__predict_false(!(prot & NVMM_PROT_WRITE))) { 649 errno = EFAULT; 650 return -1; 651 } 652 memcpy((uint8_t *)hva, data, size); 653 } 654 655 if (remain > 0) { 656 ret = write_guest_memory(mach, vcpu, gva + size, 657 data + size, remain); 658 } else { 659 ret = 0; 660 } 661 662 return ret; 663 } 664 665 /* -------------------------------------------------------------------------- */ 666 667 static int fetch_segment(struct nvmm_machine *, struct nvmm_vcpu *); 668 669 #define NVMM_IO_BATCH_SIZE 32 670 671 static int 672 assist_io_batch(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 673 struct nvmm_io *io, gvaddr_t gva, uint64_t cnt) 674 { 675 uint8_t iobuf[NVMM_IO_BATCH_SIZE]; 676 size_t i, iosize, iocnt; 677 int ret; 678 679 cnt = MIN(cnt, NVMM_IO_BATCH_SIZE); 680 iosize = MIN(io->size * cnt, NVMM_IO_BATCH_SIZE); 681 iocnt = iosize / io->size; 682 683 io->data = iobuf; 684 685 if (!io->in) { 686 ret = read_guest_memory(mach, vcpu, gva, iobuf, iosize); 687 if (ret == -1) 688 return -1; 689 } 690 691 for (i = 0; i < iocnt; i++) { 692 (*vcpu->cbs.io)(io); 693 io->data += io->size; 694 } 695 696 if (io->in) { 697 ret = write_guest_memory(mach, vcpu, gva, iobuf, iosize); 698 if (ret == -1) 699 return -1; 700 } 701 702 return iocnt; 703 } 704 705 int 706 nvmm_assist_io(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 707 { 708 struct nvmm_x64_state *state = vcpu->state; 709 struct nvmm_vcpu_exit *exit = vcpu->exit; 710 struct nvmm_io io; 711 uint64_t cnt = 0; /* GCC */ 712 uint8_t iobuf[8]; 713 int iocnt = 1; 714 gvaddr_t gva = 0; /* GCC */ 715 int reg = 0; /* GCC */ 716 int ret, seg; 717 bool psld = false; 718 719 if (__predict_false(exit->reason != NVMM_VCPU_EXIT_IO)) { 720 errno = EINVAL; 721 return -1; 722 } 723 724 io.mach = mach; 725 io.vcpu = vcpu; 726 io.port = exit->u.io.port; 727 io.in = exit->u.io.in; 728 io.size = exit->u.io.operand_size; 729 io.data = iobuf; 730 731 ret = nvmm_vcpu_getstate(mach, vcpu, 732 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 733 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 734 if (ret == -1) 735 return -1; 736 737 if (exit->u.io.rep) { 738 cnt = rep_get_cnt(state, exit->u.io.address_size); 739 if (__predict_false(cnt == 0)) { 740 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 741 goto out; 742 } 743 } 744 745 if (__predict_false(state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_D)) { 746 psld = true; 747 } 748 749 /* 750 * Determine GVA. 751 */ 752 if (exit->u.io.str) { 753 if (io.in) { 754 reg = NVMM_X64_GPR_RDI; 755 } else { 756 reg = NVMM_X64_GPR_RSI; 757 } 758 759 gva = state->gprs[reg]; 760 gva &= size_to_mask(exit->u.io.address_size); 761 762 if (exit->u.io.seg != -1) { 763 seg = exit->u.io.seg; 764 } else { 765 if (io.in) { 766 seg = NVMM_X64_SEG_ES; 767 } else { 768 seg = fetch_segment(mach, vcpu); 769 if (seg == -1) 770 return -1; 771 } 772 } 773 774 if (__predict_true(is_long_mode(state))) { 775 if (seg == NVMM_X64_SEG_GS || seg == NVMM_X64_SEG_FS) { 776 segment_apply(&state->segs[seg], &gva); 777 } 778 } else { 779 ret = segment_check(&state->segs[seg], gva, io.size); 780 if (ret == -1) 781 return -1; 782 segment_apply(&state->segs[seg], &gva); 783 } 784 785 if (exit->u.io.rep && !psld) { 786 iocnt = assist_io_batch(mach, vcpu, &io, gva, cnt); 787 if (iocnt == -1) 788 return -1; 789 goto done; 790 } 791 } 792 793 if (!io.in) { 794 if (!exit->u.io.str) { 795 memcpy(io.data, &state->gprs[NVMM_X64_GPR_RAX], io.size); 796 } else { 797 ret = read_guest_memory(mach, vcpu, gva, io.data, 798 io.size); 799 if (ret == -1) 800 return -1; 801 } 802 } 803 804 (*vcpu->cbs.io)(&io); 805 806 if (io.in) { 807 if (!exit->u.io.str) { 808 memcpy(&state->gprs[NVMM_X64_GPR_RAX], io.data, io.size); 809 if (io.size == 4) { 810 /* Zero-extend to 64 bits. */ 811 state->gprs[NVMM_X64_GPR_RAX] &= size_to_mask(4); 812 } 813 } else { 814 ret = write_guest_memory(mach, vcpu, gva, io.data, 815 io.size); 816 if (ret == -1) 817 return -1; 818 } 819 } 820 821 done: 822 if (exit->u.io.str) { 823 if (__predict_false(psld)) { 824 state->gprs[reg] -= iocnt * io.size; 825 } else { 826 state->gprs[reg] += iocnt * io.size; 827 } 828 } 829 830 if (exit->u.io.rep) { 831 cnt -= iocnt; 832 rep_set_cnt(state, exit->u.io.address_size, cnt); 833 if (cnt == 0) { 834 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 835 } 836 } else { 837 state->gprs[NVMM_X64_GPR_RIP] = exit->u.io.npc; 838 } 839 840 out: 841 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 842 if (ret == -1) 843 return -1; 844 845 return 0; 846 } 847 848 /* -------------------------------------------------------------------------- */ 849 850 struct x86_emul { 851 bool readreg; 852 bool backprop; 853 bool notouch; 854 void (*func)(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 855 }; 856 857 static void x86_func_or(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 858 static void x86_func_and(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 859 static void x86_func_xchg(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 860 static void x86_func_sub(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 861 static void x86_func_xor(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 862 static void x86_func_cmp(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 863 static void x86_func_test(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 864 static void x86_func_mov(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 865 static void x86_func_stos(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 866 static void x86_func_lods(struct nvmm_vcpu *, struct nvmm_mem *, uint64_t *); 867 868 static const struct x86_emul x86_emul_or = { 869 .readreg = true, 870 .func = x86_func_or 871 }; 872 873 static const struct x86_emul x86_emul_and = { 874 .readreg = true, 875 .func = x86_func_and 876 }; 877 878 static const struct x86_emul x86_emul_xchg = { 879 .readreg = true, 880 .backprop = true, 881 .func = x86_func_xchg 882 }; 883 884 static const struct x86_emul x86_emul_sub = { 885 .readreg = true, 886 .func = x86_func_sub 887 }; 888 889 static const struct x86_emul x86_emul_xor = { 890 .readreg = true, 891 .func = x86_func_xor 892 }; 893 894 static const struct x86_emul x86_emul_cmp = { 895 .notouch = true, 896 .func = x86_func_cmp 897 }; 898 899 static const struct x86_emul x86_emul_test = { 900 .notouch = true, 901 .func = x86_func_test 902 }; 903 904 static const struct x86_emul x86_emul_mov = { 905 .func = x86_func_mov 906 }; 907 908 static const struct x86_emul x86_emul_stos = { 909 .func = x86_func_stos 910 }; 911 912 static const struct x86_emul x86_emul_lods = { 913 .func = x86_func_lods 914 }; 915 916 /* Legacy prefixes. */ 917 #define LEG_LOCK 0xF0 918 #define LEG_REPN 0xF2 919 #define LEG_REP 0xF3 920 #define LEG_OVR_CS 0x2E 921 #define LEG_OVR_SS 0x36 922 #define LEG_OVR_DS 0x3E 923 #define LEG_OVR_ES 0x26 924 #define LEG_OVR_FS 0x64 925 #define LEG_OVR_GS 0x65 926 #define LEG_OPR_OVR 0x66 927 #define LEG_ADR_OVR 0x67 928 929 struct x86_legpref { 930 bool opr_ovr:1; 931 bool adr_ovr:1; 932 bool rep:1; 933 bool repn:1; 934 int8_t seg; 935 }; 936 937 struct x86_rexpref { 938 bool b:1; 939 bool x:1; 940 bool r:1; 941 bool w:1; 942 bool present:1; 943 }; 944 945 struct x86_reg { 946 int num; /* NVMM GPR state index */ 947 uint64_t mask; 948 }; 949 950 struct x86_dualreg { 951 int reg1; 952 int reg2; 953 }; 954 955 enum x86_disp_type { 956 DISP_NONE, 957 DISP_0, 958 DISP_1, 959 DISP_2, 960 DISP_4 961 }; 962 963 struct x86_disp { 964 enum x86_disp_type type; 965 uint64_t data; /* 4 bytes, but can be sign-extended */ 966 }; 967 968 struct x86_regmodrm { 969 uint8_t mod:2; 970 uint8_t reg:3; 971 uint8_t rm:3; 972 }; 973 974 struct x86_immediate { 975 uint64_t data; 976 }; 977 978 struct x86_sib { 979 uint8_t scale; 980 const struct x86_reg *idx; 981 const struct x86_reg *bas; 982 }; 983 984 enum x86_store_type { 985 STORE_NONE, 986 STORE_REG, 987 STORE_DUALREG, 988 STORE_IMM, 989 STORE_SIB, 990 STORE_DMO 991 }; 992 993 struct x86_store { 994 enum x86_store_type type; 995 union { 996 const struct x86_reg *reg; 997 struct x86_dualreg dualreg; 998 struct x86_immediate imm; 999 struct x86_sib sib; 1000 uint64_t dmo; 1001 } u; 1002 struct x86_disp disp; 1003 int hardseg; 1004 }; 1005 1006 struct x86_instr { 1007 uint8_t len; 1008 struct x86_legpref legpref; 1009 struct x86_rexpref rexpref; 1010 struct x86_regmodrm regmodrm; 1011 uint8_t operand_size; 1012 uint8_t address_size; 1013 uint64_t zeroextend_mask; 1014 1015 const struct x86_opcode *opcode; 1016 const struct x86_emul *emul; 1017 1018 struct x86_store src; 1019 struct x86_store dst; 1020 struct x86_store *strm; 1021 }; 1022 1023 struct x86_decode_fsm { 1024 /* vcpu */ 1025 bool is64bit; 1026 bool is32bit; 1027 bool is16bit; 1028 1029 /* fsm */ 1030 int (*fn)(struct x86_decode_fsm *, struct x86_instr *); 1031 uint8_t *buf; 1032 uint8_t *end; 1033 }; 1034 1035 struct x86_opcode { 1036 bool valid:1; 1037 bool regmodrm:1; 1038 bool regtorm:1; 1039 bool dmo:1; 1040 bool todmo:1; 1041 bool movs:1; 1042 bool stos:1; 1043 bool lods:1; 1044 bool szoverride:1; 1045 bool group1:1; 1046 bool group3:1; 1047 bool group11:1; 1048 bool immediate:1; 1049 uint8_t defsize; 1050 uint8_t flags; 1051 const struct x86_emul *emul; 1052 }; 1053 1054 struct x86_group_entry { 1055 const struct x86_emul *emul; 1056 }; 1057 1058 #define OPSIZE_BYTE 0x01 1059 #define OPSIZE_WORD 0x02 /* 2 bytes */ 1060 #define OPSIZE_DOUB 0x04 /* 4 bytes */ 1061 #define OPSIZE_QUAD 0x08 /* 8 bytes */ 1062 1063 #define FLAG_imm8 0x01 1064 #define FLAG_immz 0x02 1065 #define FLAG_ze 0x04 1066 1067 static const struct x86_group_entry group1[8] __cacheline_aligned = { 1068 [1] = { .emul = &x86_emul_or }, 1069 [4] = { .emul = &x86_emul_and }, 1070 [6] = { .emul = &x86_emul_xor }, 1071 [7] = { .emul = &x86_emul_cmp } 1072 }; 1073 1074 static const struct x86_group_entry group3[8] __cacheline_aligned = { 1075 [0] = { .emul = &x86_emul_test }, 1076 [1] = { .emul = &x86_emul_test } 1077 }; 1078 1079 static const struct x86_group_entry group11[8] __cacheline_aligned = { 1080 [0] = { .emul = &x86_emul_mov } 1081 }; 1082 1083 static const struct x86_opcode primary_opcode_table[256] __cacheline_aligned = { 1084 /* 1085 * Group1 1086 */ 1087 [0x80] = { 1088 /* Eb, Ib */ 1089 .valid = true, 1090 .regmodrm = true, 1091 .regtorm = true, 1092 .szoverride = false, 1093 .defsize = OPSIZE_BYTE, 1094 .group1 = true, 1095 .immediate = true, 1096 .emul = NULL /* group1 */ 1097 }, 1098 [0x81] = { 1099 /* Ev, Iz */ 1100 .valid = true, 1101 .regmodrm = true, 1102 .regtorm = true, 1103 .szoverride = true, 1104 .defsize = -1, 1105 .group1 = true, 1106 .immediate = true, 1107 .flags = FLAG_immz, 1108 .emul = NULL /* group1 */ 1109 }, 1110 [0x83] = { 1111 /* Ev, Ib */ 1112 .valid = true, 1113 .regmodrm = true, 1114 .regtorm = true, 1115 .szoverride = true, 1116 .defsize = -1, 1117 .group1 = true, 1118 .immediate = true, 1119 .flags = FLAG_imm8, 1120 .emul = NULL /* group1 */ 1121 }, 1122 1123 /* 1124 * Group3 1125 */ 1126 [0xF6] = { 1127 /* Eb, Ib */ 1128 .valid = true, 1129 .regmodrm = true, 1130 .regtorm = true, 1131 .szoverride = false, 1132 .defsize = OPSIZE_BYTE, 1133 .group3 = true, 1134 .immediate = true, 1135 .emul = NULL /* group3 */ 1136 }, 1137 [0xF7] = { 1138 /* Ev, Iz */ 1139 .valid = true, 1140 .regmodrm = true, 1141 .regtorm = true, 1142 .szoverride = true, 1143 .defsize = -1, 1144 .group3 = true, 1145 .immediate = true, 1146 .flags = FLAG_immz, 1147 .emul = NULL /* group3 */ 1148 }, 1149 1150 /* 1151 * Group11 1152 */ 1153 [0xC6] = { 1154 /* Eb, Ib */ 1155 .valid = true, 1156 .regmodrm = true, 1157 .regtorm = true, 1158 .szoverride = false, 1159 .defsize = OPSIZE_BYTE, 1160 .group11 = true, 1161 .immediate = true, 1162 .emul = NULL /* group11 */ 1163 }, 1164 [0xC7] = { 1165 /* Ev, Iz */ 1166 .valid = true, 1167 .regmodrm = true, 1168 .regtorm = true, 1169 .szoverride = true, 1170 .defsize = -1, 1171 .group11 = true, 1172 .immediate = true, 1173 .flags = FLAG_immz, 1174 .emul = NULL /* group11 */ 1175 }, 1176 1177 /* 1178 * OR 1179 */ 1180 [0x08] = { 1181 /* Eb, Gb */ 1182 .valid = true, 1183 .regmodrm = true, 1184 .regtorm = true, 1185 .szoverride = false, 1186 .defsize = OPSIZE_BYTE, 1187 .emul = &x86_emul_or 1188 }, 1189 [0x09] = { 1190 /* Ev, Gv */ 1191 .valid = true, 1192 .regmodrm = true, 1193 .regtorm = true, 1194 .szoverride = true, 1195 .defsize = -1, 1196 .emul = &x86_emul_or 1197 }, 1198 [0x0A] = { 1199 /* Gb, Eb */ 1200 .valid = true, 1201 .regmodrm = true, 1202 .regtorm = false, 1203 .szoverride = false, 1204 .defsize = OPSIZE_BYTE, 1205 .emul = &x86_emul_or 1206 }, 1207 [0x0B] = { 1208 /* Gv, Ev */ 1209 .valid = true, 1210 .regmodrm = true, 1211 .regtorm = false, 1212 .szoverride = true, 1213 .defsize = -1, 1214 .emul = &x86_emul_or 1215 }, 1216 1217 /* 1218 * AND 1219 */ 1220 [0x20] = { 1221 /* Eb, Gb */ 1222 .valid = true, 1223 .regmodrm = true, 1224 .regtorm = true, 1225 .szoverride = false, 1226 .defsize = OPSIZE_BYTE, 1227 .emul = &x86_emul_and 1228 }, 1229 [0x21] = { 1230 /* Ev, Gv */ 1231 .valid = true, 1232 .regmodrm = true, 1233 .regtorm = true, 1234 .szoverride = true, 1235 .defsize = -1, 1236 .emul = &x86_emul_and 1237 }, 1238 [0x22] = { 1239 /* Gb, Eb */ 1240 .valid = true, 1241 .regmodrm = true, 1242 .regtorm = false, 1243 .szoverride = false, 1244 .defsize = OPSIZE_BYTE, 1245 .emul = &x86_emul_and 1246 }, 1247 [0x23] = { 1248 /* Gv, Ev */ 1249 .valid = true, 1250 .regmodrm = true, 1251 .regtorm = false, 1252 .szoverride = true, 1253 .defsize = -1, 1254 .emul = &x86_emul_and 1255 }, 1256 1257 /* 1258 * SUB 1259 */ 1260 [0x28] = { 1261 /* Eb, Gb */ 1262 .valid = true, 1263 .regmodrm = true, 1264 .regtorm = true, 1265 .szoverride = false, 1266 .defsize = OPSIZE_BYTE, 1267 .emul = &x86_emul_sub 1268 }, 1269 [0x29] = { 1270 /* Ev, Gv */ 1271 .valid = true, 1272 .regmodrm = true, 1273 .regtorm = true, 1274 .szoverride = true, 1275 .defsize = -1, 1276 .emul = &x86_emul_sub 1277 }, 1278 [0x2A] = { 1279 /* Gb, Eb */ 1280 .valid = true, 1281 .regmodrm = true, 1282 .regtorm = false, 1283 .szoverride = false, 1284 .defsize = OPSIZE_BYTE, 1285 .emul = &x86_emul_sub 1286 }, 1287 [0x2B] = { 1288 /* Gv, Ev */ 1289 .valid = true, 1290 .regmodrm = true, 1291 .regtorm = false, 1292 .szoverride = true, 1293 .defsize = -1, 1294 .emul = &x86_emul_sub 1295 }, 1296 1297 /* 1298 * XOR 1299 */ 1300 [0x30] = { 1301 /* Eb, Gb */ 1302 .valid = true, 1303 .regmodrm = true, 1304 .regtorm = true, 1305 .szoverride = false, 1306 .defsize = OPSIZE_BYTE, 1307 .emul = &x86_emul_xor 1308 }, 1309 [0x31] = { 1310 /* Ev, Gv */ 1311 .valid = true, 1312 .regmodrm = true, 1313 .regtorm = true, 1314 .szoverride = true, 1315 .defsize = -1, 1316 .emul = &x86_emul_xor 1317 }, 1318 [0x32] = { 1319 /* Gb, Eb */ 1320 .valid = true, 1321 .regmodrm = true, 1322 .regtorm = false, 1323 .szoverride = false, 1324 .defsize = OPSIZE_BYTE, 1325 .emul = &x86_emul_xor 1326 }, 1327 [0x33] = { 1328 /* Gv, Ev */ 1329 .valid = true, 1330 .regmodrm = true, 1331 .regtorm = false, 1332 .szoverride = true, 1333 .defsize = -1, 1334 .emul = &x86_emul_xor 1335 }, 1336 1337 /* 1338 * XCHG 1339 */ 1340 [0x86] = { 1341 /* Eb, Gb */ 1342 .valid = true, 1343 .regmodrm = true, 1344 .regtorm = true, 1345 .szoverride = false, 1346 .defsize = OPSIZE_BYTE, 1347 .emul = &x86_emul_xchg 1348 }, 1349 [0x87] = { 1350 /* Ev, Gv */ 1351 .valid = true, 1352 .regmodrm = true, 1353 .regtorm = true, 1354 .szoverride = true, 1355 .defsize = -1, 1356 .emul = &x86_emul_xchg 1357 }, 1358 1359 /* 1360 * MOV 1361 */ 1362 [0x88] = { 1363 /* Eb, Gb */ 1364 .valid = true, 1365 .regmodrm = true, 1366 .regtorm = true, 1367 .szoverride = false, 1368 .defsize = OPSIZE_BYTE, 1369 .emul = &x86_emul_mov 1370 }, 1371 [0x89] = { 1372 /* Ev, Gv */ 1373 .valid = true, 1374 .regmodrm = true, 1375 .regtorm = true, 1376 .szoverride = true, 1377 .defsize = -1, 1378 .emul = &x86_emul_mov 1379 }, 1380 [0x8A] = { 1381 /* Gb, Eb */ 1382 .valid = true, 1383 .regmodrm = true, 1384 .regtorm = false, 1385 .szoverride = false, 1386 .defsize = OPSIZE_BYTE, 1387 .emul = &x86_emul_mov 1388 }, 1389 [0x8B] = { 1390 /* Gv, Ev */ 1391 .valid = true, 1392 .regmodrm = true, 1393 .regtorm = false, 1394 .szoverride = true, 1395 .defsize = -1, 1396 .emul = &x86_emul_mov 1397 }, 1398 [0xA0] = { 1399 /* AL, Ob */ 1400 .valid = true, 1401 .dmo = true, 1402 .todmo = false, 1403 .szoverride = false, 1404 .defsize = OPSIZE_BYTE, 1405 .emul = &x86_emul_mov 1406 }, 1407 [0xA1] = { 1408 /* rAX, Ov */ 1409 .valid = true, 1410 .dmo = true, 1411 .todmo = false, 1412 .szoverride = true, 1413 .defsize = -1, 1414 .emul = &x86_emul_mov 1415 }, 1416 [0xA2] = { 1417 /* Ob, AL */ 1418 .valid = true, 1419 .dmo = true, 1420 .todmo = true, 1421 .szoverride = false, 1422 .defsize = OPSIZE_BYTE, 1423 .emul = &x86_emul_mov 1424 }, 1425 [0xA3] = { 1426 /* Ov, rAX */ 1427 .valid = true, 1428 .dmo = true, 1429 .todmo = true, 1430 .szoverride = true, 1431 .defsize = -1, 1432 .emul = &x86_emul_mov 1433 }, 1434 1435 /* 1436 * MOVS 1437 */ 1438 [0xA4] = { 1439 /* Yb, Xb */ 1440 .valid = true, 1441 .movs = true, 1442 .szoverride = false, 1443 .defsize = OPSIZE_BYTE, 1444 .emul = NULL /* assist_mem_double_movs */ 1445 }, 1446 [0xA5] = { 1447 /* Yv, Xv */ 1448 .valid = true, 1449 .movs = true, 1450 .szoverride = true, 1451 .defsize = -1, 1452 .emul = NULL /* assist_mem_double_movs */ 1453 }, 1454 1455 /* 1456 * STOS 1457 */ 1458 [0xAA] = { 1459 /* Yb, AL */ 1460 .valid = true, 1461 .stos = true, 1462 .szoverride = false, 1463 .defsize = OPSIZE_BYTE, 1464 .emul = &x86_emul_stos 1465 }, 1466 [0xAB] = { 1467 /* Yv, rAX */ 1468 .valid = true, 1469 .stos = true, 1470 .szoverride = true, 1471 .defsize = -1, 1472 .emul = &x86_emul_stos 1473 }, 1474 1475 /* 1476 * LODS 1477 */ 1478 [0xAC] = { 1479 /* AL, Xb */ 1480 .valid = true, 1481 .lods = true, 1482 .szoverride = false, 1483 .defsize = OPSIZE_BYTE, 1484 .emul = &x86_emul_lods 1485 }, 1486 [0xAD] = { 1487 /* rAX, Xv */ 1488 .valid = true, 1489 .lods = true, 1490 .szoverride = true, 1491 .defsize = -1, 1492 .emul = &x86_emul_lods 1493 }, 1494 }; 1495 1496 static const struct x86_opcode secondary_opcode_table[256] __cacheline_aligned = { 1497 /* 1498 * MOVZX 1499 */ 1500 [0xB6] = { 1501 /* Gv, Eb */ 1502 .valid = true, 1503 .regmodrm = true, 1504 .regtorm = false, 1505 .szoverride = true, 1506 .defsize = OPSIZE_BYTE, 1507 .flags = FLAG_ze, 1508 .emul = &x86_emul_mov 1509 }, 1510 [0xB7] = { 1511 /* Gv, Ew */ 1512 .valid = true, 1513 .regmodrm = true, 1514 .regtorm = false, 1515 .szoverride = true, 1516 .defsize = OPSIZE_WORD, 1517 .flags = FLAG_ze, 1518 .emul = &x86_emul_mov 1519 }, 1520 }; 1521 1522 static const struct x86_reg gpr_map__rip = { NVMM_X64_GPR_RIP, 0xFFFFFFFFFFFFFFFF }; 1523 1524 /* [REX-present][enc][opsize] */ 1525 static const struct x86_reg gpr_map__special[2][4][8] __cacheline_aligned = { 1526 [false] = { 1527 /* No REX prefix. */ 1528 [0b00] = { 1529 [0] = { NVMM_X64_GPR_RAX, 0x000000000000FF00 }, /* AH */ 1530 [1] = { NVMM_X64_GPR_RSP, 0x000000000000FFFF }, /* SP */ 1531 [2] = { -1, 0 }, 1532 [3] = { NVMM_X64_GPR_RSP, 0x00000000FFFFFFFF }, /* ESP */ 1533 [4] = { -1, 0 }, 1534 [5] = { -1, 0 }, 1535 [6] = { -1, 0 }, 1536 [7] = { -1, 0 }, 1537 }, 1538 [0b01] = { 1539 [0] = { NVMM_X64_GPR_RCX, 0x000000000000FF00 }, /* CH */ 1540 [1] = { NVMM_X64_GPR_RBP, 0x000000000000FFFF }, /* BP */ 1541 [2] = { -1, 0 }, 1542 [3] = { NVMM_X64_GPR_RBP, 0x00000000FFFFFFFF }, /* EBP */ 1543 [4] = { -1, 0 }, 1544 [5] = { -1, 0 }, 1545 [6] = { -1, 0 }, 1546 [7] = { -1, 0 }, 1547 }, 1548 [0b10] = { 1549 [0] = { NVMM_X64_GPR_RDX, 0x000000000000FF00 }, /* DH */ 1550 [1] = { NVMM_X64_GPR_RSI, 0x000000000000FFFF }, /* SI */ 1551 [2] = { -1, 0 }, 1552 [3] = { NVMM_X64_GPR_RSI, 0x00000000FFFFFFFF }, /* ESI */ 1553 [4] = { -1, 0 }, 1554 [5] = { -1, 0 }, 1555 [6] = { -1, 0 }, 1556 [7] = { -1, 0 }, 1557 }, 1558 [0b11] = { 1559 [0] = { NVMM_X64_GPR_RBX, 0x000000000000FF00 }, /* BH */ 1560 [1] = { NVMM_X64_GPR_RDI, 0x000000000000FFFF }, /* DI */ 1561 [2] = { -1, 0 }, 1562 [3] = { NVMM_X64_GPR_RDI, 0x00000000FFFFFFFF }, /* EDI */ 1563 [4] = { -1, 0 }, 1564 [5] = { -1, 0 }, 1565 [6] = { -1, 0 }, 1566 [7] = { -1, 0 }, 1567 } 1568 }, 1569 [true] = { 1570 /* Has REX prefix. */ 1571 [0b00] = { 1572 [0] = { NVMM_X64_GPR_RSP, 0x00000000000000FF }, /* SPL */ 1573 [1] = { NVMM_X64_GPR_RSP, 0x000000000000FFFF }, /* SP */ 1574 [2] = { -1, 0 }, 1575 [3] = { NVMM_X64_GPR_RSP, 0x00000000FFFFFFFF }, /* ESP */ 1576 [4] = { -1, 0 }, 1577 [5] = { -1, 0 }, 1578 [6] = { -1, 0 }, 1579 [7] = { NVMM_X64_GPR_RSP, 0xFFFFFFFFFFFFFFFF }, /* RSP */ 1580 }, 1581 [0b01] = { 1582 [0] = { NVMM_X64_GPR_RBP, 0x00000000000000FF }, /* BPL */ 1583 [1] = { NVMM_X64_GPR_RBP, 0x000000000000FFFF }, /* BP */ 1584 [2] = { -1, 0 }, 1585 [3] = { NVMM_X64_GPR_RBP, 0x00000000FFFFFFFF }, /* EBP */ 1586 [4] = { -1, 0 }, 1587 [5] = { -1, 0 }, 1588 [6] = { -1, 0 }, 1589 [7] = { NVMM_X64_GPR_RBP, 0xFFFFFFFFFFFFFFFF }, /* RBP */ 1590 }, 1591 [0b10] = { 1592 [0] = { NVMM_X64_GPR_RSI, 0x00000000000000FF }, /* SIL */ 1593 [1] = { NVMM_X64_GPR_RSI, 0x000000000000FFFF }, /* SI */ 1594 [2] = { -1, 0 }, 1595 [3] = { NVMM_X64_GPR_RSI, 0x00000000FFFFFFFF }, /* ESI */ 1596 [4] = { -1, 0 }, 1597 [5] = { -1, 0 }, 1598 [6] = { -1, 0 }, 1599 [7] = { NVMM_X64_GPR_RSI, 0xFFFFFFFFFFFFFFFF }, /* RSI */ 1600 }, 1601 [0b11] = { 1602 [0] = { NVMM_X64_GPR_RDI, 0x00000000000000FF }, /* DIL */ 1603 [1] = { NVMM_X64_GPR_RDI, 0x000000000000FFFF }, /* DI */ 1604 [2] = { -1, 0 }, 1605 [3] = { NVMM_X64_GPR_RDI, 0x00000000FFFFFFFF }, /* EDI */ 1606 [4] = { -1, 0 }, 1607 [5] = { -1, 0 }, 1608 [6] = { -1, 0 }, 1609 [7] = { NVMM_X64_GPR_RDI, 0xFFFFFFFFFFFFFFFF }, /* RDI */ 1610 } 1611 } 1612 }; 1613 1614 /* [depends][enc][size] */ 1615 static const struct x86_reg gpr_map[2][8][8] __cacheline_aligned = { 1616 [false] = { 1617 /* Not extended. */ 1618 [0b000] = { 1619 [0] = { NVMM_X64_GPR_RAX, 0x00000000000000FF }, /* AL */ 1620 [1] = { NVMM_X64_GPR_RAX, 0x000000000000FFFF }, /* AX */ 1621 [2] = { -1, 0 }, 1622 [3] = { NVMM_X64_GPR_RAX, 0x00000000FFFFFFFF }, /* EAX */ 1623 [4] = { -1, 0 }, 1624 [5] = { -1, 0 }, 1625 [6] = { -1, 0 }, 1626 [7] = { NVMM_X64_GPR_RAX, 0xFFFFFFFFFFFFFFFF }, /* RAX */ 1627 }, 1628 [0b001] = { 1629 [0] = { NVMM_X64_GPR_RCX, 0x00000000000000FF }, /* CL */ 1630 [1] = { NVMM_X64_GPR_RCX, 0x000000000000FFFF }, /* CX */ 1631 [2] = { -1, 0 }, 1632 [3] = { NVMM_X64_GPR_RCX, 0x00000000FFFFFFFF }, /* ECX */ 1633 [4] = { -1, 0 }, 1634 [5] = { -1, 0 }, 1635 [6] = { -1, 0 }, 1636 [7] = { NVMM_X64_GPR_RCX, 0xFFFFFFFFFFFFFFFF }, /* RCX */ 1637 }, 1638 [0b010] = { 1639 [0] = { NVMM_X64_GPR_RDX, 0x00000000000000FF }, /* DL */ 1640 [1] = { NVMM_X64_GPR_RDX, 0x000000000000FFFF }, /* DX */ 1641 [2] = { -1, 0 }, 1642 [3] = { NVMM_X64_GPR_RDX, 0x00000000FFFFFFFF }, /* EDX */ 1643 [4] = { -1, 0 }, 1644 [5] = { -1, 0 }, 1645 [6] = { -1, 0 }, 1646 [7] = { NVMM_X64_GPR_RDX, 0xFFFFFFFFFFFFFFFF }, /* RDX */ 1647 }, 1648 [0b011] = { 1649 [0] = { NVMM_X64_GPR_RBX, 0x00000000000000FF }, /* BL */ 1650 [1] = { NVMM_X64_GPR_RBX, 0x000000000000FFFF }, /* BX */ 1651 [2] = { -1, 0 }, 1652 [3] = { NVMM_X64_GPR_RBX, 0x00000000FFFFFFFF }, /* EBX */ 1653 [4] = { -1, 0 }, 1654 [5] = { -1, 0 }, 1655 [6] = { -1, 0 }, 1656 [7] = { NVMM_X64_GPR_RBX, 0xFFFFFFFFFFFFFFFF }, /* RBX */ 1657 }, 1658 [0b100] = { 1659 [0] = { -1, 0 }, /* SPECIAL */ 1660 [1] = { -1, 0 }, /* SPECIAL */ 1661 [2] = { -1, 0 }, 1662 [3] = { -1, 0 }, /* SPECIAL */ 1663 [4] = { -1, 0 }, 1664 [5] = { -1, 0 }, 1665 [6] = { -1, 0 }, 1666 [7] = { -1, 0 }, /* SPECIAL */ 1667 }, 1668 [0b101] = { 1669 [0] = { -1, 0 }, /* SPECIAL */ 1670 [1] = { -1, 0 }, /* SPECIAL */ 1671 [2] = { -1, 0 }, 1672 [3] = { -1, 0 }, /* SPECIAL */ 1673 [4] = { -1, 0 }, 1674 [5] = { -1, 0 }, 1675 [6] = { -1, 0 }, 1676 [7] = { -1, 0 }, /* SPECIAL */ 1677 }, 1678 [0b110] = { 1679 [0] = { -1, 0 }, /* SPECIAL */ 1680 [1] = { -1, 0 }, /* SPECIAL */ 1681 [2] = { -1, 0 }, 1682 [3] = { -1, 0 }, /* SPECIAL */ 1683 [4] = { -1, 0 }, 1684 [5] = { -1, 0 }, 1685 [6] = { -1, 0 }, 1686 [7] = { -1, 0 }, /* SPECIAL */ 1687 }, 1688 [0b111] = { 1689 [0] = { -1, 0 }, /* SPECIAL */ 1690 [1] = { -1, 0 }, /* SPECIAL */ 1691 [2] = { -1, 0 }, 1692 [3] = { -1, 0 }, /* SPECIAL */ 1693 [4] = { -1, 0 }, 1694 [5] = { -1, 0 }, 1695 [6] = { -1, 0 }, 1696 [7] = { -1, 0 }, /* SPECIAL */ 1697 }, 1698 }, 1699 [true] = { 1700 /* Extended. */ 1701 [0b000] = { 1702 [0] = { NVMM_X64_GPR_R8, 0x00000000000000FF }, /* R8B */ 1703 [1] = { NVMM_X64_GPR_R8, 0x000000000000FFFF }, /* R8W */ 1704 [2] = { -1, 0 }, 1705 [3] = { NVMM_X64_GPR_R8, 0x00000000FFFFFFFF }, /* R8D */ 1706 [4] = { -1, 0 }, 1707 [5] = { -1, 0 }, 1708 [6] = { -1, 0 }, 1709 [7] = { NVMM_X64_GPR_R8, 0xFFFFFFFFFFFFFFFF }, /* R8 */ 1710 }, 1711 [0b001] = { 1712 [0] = { NVMM_X64_GPR_R9, 0x00000000000000FF }, /* R9B */ 1713 [1] = { NVMM_X64_GPR_R9, 0x000000000000FFFF }, /* R9W */ 1714 [2] = { -1, 0 }, 1715 [3] = { NVMM_X64_GPR_R9, 0x00000000FFFFFFFF }, /* R9D */ 1716 [4] = { -1, 0 }, 1717 [5] = { -1, 0 }, 1718 [6] = { -1, 0 }, 1719 [7] = { NVMM_X64_GPR_R9, 0xFFFFFFFFFFFFFFFF }, /* R9 */ 1720 }, 1721 [0b010] = { 1722 [0] = { NVMM_X64_GPR_R10, 0x00000000000000FF }, /* R10B */ 1723 [1] = { NVMM_X64_GPR_R10, 0x000000000000FFFF }, /* R10W */ 1724 [2] = { -1, 0 }, 1725 [3] = { NVMM_X64_GPR_R10, 0x00000000FFFFFFFF }, /* R10D */ 1726 [4] = { -1, 0 }, 1727 [5] = { -1, 0 }, 1728 [6] = { -1, 0 }, 1729 [7] = { NVMM_X64_GPR_R10, 0xFFFFFFFFFFFFFFFF }, /* R10 */ 1730 }, 1731 [0b011] = { 1732 [0] = { NVMM_X64_GPR_R11, 0x00000000000000FF }, /* R11B */ 1733 [1] = { NVMM_X64_GPR_R11, 0x000000000000FFFF }, /* R11W */ 1734 [2] = { -1, 0 }, 1735 [3] = { NVMM_X64_GPR_R11, 0x00000000FFFFFFFF }, /* R11D */ 1736 [4] = { -1, 0 }, 1737 [5] = { -1, 0 }, 1738 [6] = { -1, 0 }, 1739 [7] = { NVMM_X64_GPR_R11, 0xFFFFFFFFFFFFFFFF }, /* R11 */ 1740 }, 1741 [0b100] = { 1742 [0] = { NVMM_X64_GPR_R12, 0x00000000000000FF }, /* R12B */ 1743 [1] = { NVMM_X64_GPR_R12, 0x000000000000FFFF }, /* R12W */ 1744 [2] = { -1, 0 }, 1745 [3] = { NVMM_X64_GPR_R12, 0x00000000FFFFFFFF }, /* R12D */ 1746 [4] = { -1, 0 }, 1747 [5] = { -1, 0 }, 1748 [6] = { -1, 0 }, 1749 [7] = { NVMM_X64_GPR_R12, 0xFFFFFFFFFFFFFFFF }, /* R12 */ 1750 }, 1751 [0b101] = { 1752 [0] = { NVMM_X64_GPR_R13, 0x00000000000000FF }, /* R13B */ 1753 [1] = { NVMM_X64_GPR_R13, 0x000000000000FFFF }, /* R13W */ 1754 [2] = { -1, 0 }, 1755 [3] = { NVMM_X64_GPR_R13, 0x00000000FFFFFFFF }, /* R13D */ 1756 [4] = { -1, 0 }, 1757 [5] = { -1, 0 }, 1758 [6] = { -1, 0 }, 1759 [7] = { NVMM_X64_GPR_R13, 0xFFFFFFFFFFFFFFFF }, /* R13 */ 1760 }, 1761 [0b110] = { 1762 [0] = { NVMM_X64_GPR_R14, 0x00000000000000FF }, /* R14B */ 1763 [1] = { NVMM_X64_GPR_R14, 0x000000000000FFFF }, /* R14W */ 1764 [2] = { -1, 0 }, 1765 [3] = { NVMM_X64_GPR_R14, 0x00000000FFFFFFFF }, /* R14D */ 1766 [4] = { -1, 0 }, 1767 [5] = { -1, 0 }, 1768 [6] = { -1, 0 }, 1769 [7] = { NVMM_X64_GPR_R14, 0xFFFFFFFFFFFFFFFF }, /* R14 */ 1770 }, 1771 [0b111] = { 1772 [0] = { NVMM_X64_GPR_R15, 0x00000000000000FF }, /* R15B */ 1773 [1] = { NVMM_X64_GPR_R15, 0x000000000000FFFF }, /* R15W */ 1774 [2] = { -1, 0 }, 1775 [3] = { NVMM_X64_GPR_R15, 0x00000000FFFFFFFF }, /* R15D */ 1776 [4] = { -1, 0 }, 1777 [5] = { -1, 0 }, 1778 [6] = { -1, 0 }, 1779 [7] = { NVMM_X64_GPR_R15, 0xFFFFFFFFFFFFFFFF }, /* R15 */ 1780 }, 1781 } 1782 }; 1783 1784 /* [enc] */ 1785 static const int gpr_dual_reg1_rm[8] __cacheline_aligned = { 1786 [0b000] = NVMM_X64_GPR_RBX, /* BX (+SI) */ 1787 [0b001] = NVMM_X64_GPR_RBX, /* BX (+DI) */ 1788 [0b010] = NVMM_X64_GPR_RBP, /* BP (+SI) */ 1789 [0b011] = NVMM_X64_GPR_RBP, /* BP (+DI) */ 1790 [0b100] = NVMM_X64_GPR_RSI, /* SI */ 1791 [0b101] = NVMM_X64_GPR_RDI, /* DI */ 1792 [0b110] = NVMM_X64_GPR_RBP, /* BP */ 1793 [0b111] = NVMM_X64_GPR_RBX, /* BX */ 1794 }; 1795 1796 static int 1797 node_overflow(struct x86_decode_fsm *fsm, struct x86_instr *instr __unused) 1798 { 1799 fsm->fn = NULL; 1800 return -1; 1801 } 1802 1803 static int 1804 fsm_read(struct x86_decode_fsm *fsm, uint8_t *bytes, size_t n) 1805 { 1806 if (fsm->buf + n > fsm->end) { 1807 return -1; 1808 } 1809 memcpy(bytes, fsm->buf, n); 1810 return 0; 1811 } 1812 1813 static inline void 1814 fsm_advance(struct x86_decode_fsm *fsm, size_t n, 1815 int (*fn)(struct x86_decode_fsm *, struct x86_instr *)) 1816 { 1817 fsm->buf += n; 1818 if (fsm->buf > fsm->end) { 1819 fsm->fn = node_overflow; 1820 } else { 1821 fsm->fn = fn; 1822 } 1823 } 1824 1825 static const struct x86_reg * 1826 resolve_special_register(struct x86_instr *instr, uint8_t enc, size_t regsize) 1827 { 1828 enc &= 0b11; 1829 if (regsize == 8) { 1830 /* May be 64bit without REX */ 1831 return &gpr_map__special[1][enc][regsize-1]; 1832 } 1833 return &gpr_map__special[instr->rexpref.present][enc][regsize-1]; 1834 } 1835 1836 /* 1837 * Special node, for MOVS. Fake two displacements of zero on the source and 1838 * destination registers. 1839 */ 1840 static int 1841 node_movs(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1842 { 1843 size_t adrsize; 1844 1845 adrsize = instr->address_size; 1846 1847 /* DS:RSI */ 1848 instr->src.type = STORE_REG; 1849 instr->src.u.reg = &gpr_map__special[1][2][adrsize-1]; 1850 instr->src.disp.type = DISP_0; 1851 1852 /* ES:RDI, force ES */ 1853 instr->dst.type = STORE_REG; 1854 instr->dst.u.reg = &gpr_map__special[1][3][adrsize-1]; 1855 instr->dst.disp.type = DISP_0; 1856 instr->dst.hardseg = NVMM_X64_SEG_ES; 1857 1858 fsm_advance(fsm, 0, NULL); 1859 1860 return 0; 1861 } 1862 1863 /* 1864 * Special node, for STOS and LODS. Fake a displacement of zero on the 1865 * destination register. 1866 */ 1867 static int 1868 node_stlo(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1869 { 1870 const struct x86_opcode *opcode = instr->opcode; 1871 struct x86_store *stlo, *streg; 1872 size_t adrsize, regsize; 1873 1874 adrsize = instr->address_size; 1875 regsize = instr->operand_size; 1876 1877 if (opcode->stos) { 1878 streg = &instr->src; 1879 stlo = &instr->dst; 1880 } else { 1881 streg = &instr->dst; 1882 stlo = &instr->src; 1883 } 1884 1885 streg->type = STORE_REG; 1886 streg->u.reg = &gpr_map[0][0][regsize-1]; /* ?AX */ 1887 1888 stlo->type = STORE_REG; 1889 if (opcode->stos) { 1890 /* ES:RDI, force ES */ 1891 stlo->u.reg = &gpr_map__special[1][3][adrsize-1]; 1892 stlo->hardseg = NVMM_X64_SEG_ES; 1893 } else { 1894 /* DS:RSI */ 1895 stlo->u.reg = &gpr_map__special[1][2][adrsize-1]; 1896 } 1897 stlo->disp.type = DISP_0; 1898 1899 fsm_advance(fsm, 0, NULL); 1900 1901 return 0; 1902 } 1903 1904 static int 1905 node_dmo(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1906 { 1907 const struct x86_opcode *opcode = instr->opcode; 1908 struct x86_store *stdmo, *streg; 1909 size_t adrsize, regsize; 1910 1911 adrsize = instr->address_size; 1912 regsize = instr->operand_size; 1913 1914 if (opcode->todmo) { 1915 streg = &instr->src; 1916 stdmo = &instr->dst; 1917 } else { 1918 streg = &instr->dst; 1919 stdmo = &instr->src; 1920 } 1921 1922 streg->type = STORE_REG; 1923 streg->u.reg = &gpr_map[0][0][regsize-1]; /* ?AX */ 1924 1925 stdmo->type = STORE_DMO; 1926 if (fsm_read(fsm, (uint8_t *)&stdmo->u.dmo, adrsize) == -1) { 1927 return -1; 1928 } 1929 fsm_advance(fsm, adrsize, NULL); 1930 1931 return 0; 1932 } 1933 1934 static inline uint64_t 1935 sign_extend(uint64_t val, int size) 1936 { 1937 if (size == 1) { 1938 if (val & __BIT(7)) 1939 val |= 0xFFFFFFFFFFFFFF00; 1940 } else if (size == 2) { 1941 if (val & __BIT(15)) 1942 val |= 0xFFFFFFFFFFFF0000; 1943 } else if (size == 4) { 1944 if (val & __BIT(31)) 1945 val |= 0xFFFFFFFF00000000; 1946 } 1947 return val; 1948 } 1949 1950 static int 1951 node_immediate(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1952 { 1953 const struct x86_opcode *opcode = instr->opcode; 1954 struct x86_store *store; 1955 uint8_t immsize; 1956 size_t sesize = 0; 1957 1958 /* The immediate is the source */ 1959 store = &instr->src; 1960 immsize = instr->operand_size; 1961 1962 if (opcode->flags & FLAG_imm8) { 1963 sesize = immsize; 1964 immsize = 1; 1965 } else if ((opcode->flags & FLAG_immz) && (immsize == 8)) { 1966 sesize = immsize; 1967 immsize = 4; 1968 } 1969 1970 store->type = STORE_IMM; 1971 if (fsm_read(fsm, (uint8_t *)&store->u.imm.data, immsize) == -1) { 1972 return -1; 1973 } 1974 fsm_advance(fsm, immsize, NULL); 1975 1976 if (sesize != 0) { 1977 store->u.imm.data = sign_extend(store->u.imm.data, sesize); 1978 } 1979 1980 return 0; 1981 } 1982 1983 static int 1984 node_disp(struct x86_decode_fsm *fsm, struct x86_instr *instr) 1985 { 1986 const struct x86_opcode *opcode = instr->opcode; 1987 uint64_t data = 0; 1988 size_t n; 1989 1990 if (instr->strm->disp.type == DISP_1) { 1991 n = 1; 1992 } else if (instr->strm->disp.type == DISP_2) { 1993 n = 2; 1994 } else if (instr->strm->disp.type == DISP_4) { 1995 n = 4; 1996 } else { 1997 DISASSEMBLER_BUG(); 1998 } 1999 2000 if (fsm_read(fsm, (uint8_t *)&data, n) == -1) { 2001 return -1; 2002 } 2003 2004 if (__predict_true(fsm->is64bit)) { 2005 data = sign_extend(data, n); 2006 } 2007 2008 instr->strm->disp.data = data; 2009 2010 if (opcode->immediate) { 2011 fsm_advance(fsm, n, node_immediate); 2012 } else { 2013 fsm_advance(fsm, n, NULL); 2014 } 2015 2016 return 0; 2017 } 2018 2019 /* 2020 * Special node to handle 16bit addressing encoding, which can reference two 2021 * registers at once. 2022 */ 2023 static int 2024 node_dual(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2025 { 2026 int reg1, reg2; 2027 2028 reg1 = gpr_dual_reg1_rm[instr->regmodrm.rm]; 2029 2030 if (instr->regmodrm.rm == 0b000 || 2031 instr->regmodrm.rm == 0b010) { 2032 reg2 = NVMM_X64_GPR_RSI; 2033 } else if (instr->regmodrm.rm == 0b001 || 2034 instr->regmodrm.rm == 0b011) { 2035 reg2 = NVMM_X64_GPR_RDI; 2036 } else { 2037 DISASSEMBLER_BUG(); 2038 } 2039 2040 instr->strm->type = STORE_DUALREG; 2041 instr->strm->u.dualreg.reg1 = reg1; 2042 instr->strm->u.dualreg.reg2 = reg2; 2043 2044 if (instr->strm->disp.type == DISP_NONE) { 2045 DISASSEMBLER_BUG(); 2046 } else if (instr->strm->disp.type == DISP_0) { 2047 /* Indirect register addressing mode */ 2048 if (instr->opcode->immediate) { 2049 fsm_advance(fsm, 1, node_immediate); 2050 } else { 2051 fsm_advance(fsm, 1, NULL); 2052 } 2053 } else { 2054 fsm_advance(fsm, 1, node_disp); 2055 } 2056 2057 return 0; 2058 } 2059 2060 static const struct x86_reg * 2061 get_register_idx(struct x86_instr *instr, uint8_t index) 2062 { 2063 uint8_t enc = index; 2064 const struct x86_reg *reg; 2065 size_t regsize; 2066 2067 regsize = instr->address_size; 2068 reg = &gpr_map[instr->rexpref.x][enc][regsize-1]; 2069 2070 if (reg->num == -1) { 2071 reg = resolve_special_register(instr, enc, regsize); 2072 } 2073 2074 return reg; 2075 } 2076 2077 static const struct x86_reg * 2078 get_register_bas(struct x86_instr *instr, uint8_t base) 2079 { 2080 uint8_t enc = base; 2081 const struct x86_reg *reg; 2082 size_t regsize; 2083 2084 regsize = instr->address_size; 2085 reg = &gpr_map[instr->rexpref.b][enc][regsize-1]; 2086 if (reg->num == -1) { 2087 reg = resolve_special_register(instr, enc, regsize); 2088 } 2089 2090 return reg; 2091 } 2092 2093 static int 2094 node_sib(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2095 { 2096 const struct x86_opcode *opcode; 2097 uint8_t scale, index, base; 2098 bool noindex, nobase; 2099 uint8_t byte; 2100 2101 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2102 return -1; 2103 } 2104 2105 scale = ((byte & 0b11000000) >> 6); 2106 index = ((byte & 0b00111000) >> 3); 2107 base = ((byte & 0b00000111) >> 0); 2108 2109 opcode = instr->opcode; 2110 2111 noindex = false; 2112 nobase = false; 2113 2114 if (index == 0b100 && !instr->rexpref.x) { 2115 /* Special case: the index is null */ 2116 noindex = true; 2117 } 2118 2119 if (instr->regmodrm.mod == 0b00 && base == 0b101) { 2120 /* Special case: the base is null + disp32 */ 2121 instr->strm->disp.type = DISP_4; 2122 nobase = true; 2123 } 2124 2125 instr->strm->type = STORE_SIB; 2126 instr->strm->u.sib.scale = (1 << scale); 2127 if (!noindex) 2128 instr->strm->u.sib.idx = get_register_idx(instr, index); 2129 if (!nobase) 2130 instr->strm->u.sib.bas = get_register_bas(instr, base); 2131 2132 /* May have a displacement, or an immediate */ 2133 if (instr->strm->disp.type == DISP_1 || 2134 instr->strm->disp.type == DISP_2 || 2135 instr->strm->disp.type == DISP_4) { 2136 fsm_advance(fsm, 1, node_disp); 2137 } else if (opcode->immediate) { 2138 fsm_advance(fsm, 1, node_immediate); 2139 } else { 2140 fsm_advance(fsm, 1, NULL); 2141 } 2142 2143 return 0; 2144 } 2145 2146 static const struct x86_reg * 2147 get_register_reg(struct x86_instr *instr, const struct x86_opcode *opcode __unused) 2148 { 2149 uint8_t enc = instr->regmodrm.reg; 2150 const struct x86_reg *reg; 2151 size_t regsize; 2152 2153 regsize = instr->operand_size; 2154 2155 reg = &gpr_map[instr->rexpref.r][enc][regsize-1]; 2156 if (reg->num == -1) { 2157 reg = resolve_special_register(instr, enc, regsize); 2158 } 2159 2160 return reg; 2161 } 2162 2163 static const struct x86_reg * 2164 get_register_rm(struct x86_instr *instr, const struct x86_opcode *opcode __unused) 2165 { 2166 uint8_t enc = instr->regmodrm.rm; 2167 const struct x86_reg *reg; 2168 size_t regsize; 2169 2170 if (instr->strm->disp.type == DISP_NONE) { 2171 regsize = instr->operand_size; 2172 } else { 2173 /* Indirect access, the size is that of the address. */ 2174 regsize = instr->address_size; 2175 } 2176 2177 reg = &gpr_map[instr->rexpref.b][enc][regsize-1]; 2178 if (reg->num == -1) { 2179 reg = resolve_special_register(instr, enc, regsize); 2180 } 2181 2182 return reg; 2183 } 2184 2185 static inline bool 2186 has_sib(struct x86_instr *instr) 2187 { 2188 return (instr->address_size != 2 && /* no SIB in 16bit addressing */ 2189 instr->regmodrm.mod != 0b11 && 2190 instr->regmodrm.rm == 0b100); 2191 } 2192 2193 static inline bool 2194 is_rip_relative(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2195 { 2196 return (fsm->is64bit && /* RIP-relative only in 64bit mode */ 2197 instr->regmodrm.mod == 0b00 && 2198 instr->regmodrm.rm == 0b101); 2199 } 2200 2201 static inline bool 2202 is_disp32_only(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2203 { 2204 return (!fsm->is64bit && /* no disp32-only in 64bit mode */ 2205 instr->address_size != 2 && /* no disp32-only in 16bit addressing */ 2206 instr->regmodrm.mod == 0b00 && 2207 instr->regmodrm.rm == 0b101); 2208 } 2209 2210 static inline bool 2211 is_disp16_only(struct x86_decode_fsm *fsm __unused, struct x86_instr *instr) 2212 { 2213 return (instr->address_size == 2 && /* disp16-only only in 16bit addr */ 2214 instr->regmodrm.mod == 0b00 && 2215 instr->regmodrm.rm == 0b110); 2216 } 2217 2218 static inline bool 2219 is_dual(struct x86_decode_fsm *fsm __unused, struct x86_instr *instr) 2220 { 2221 return (instr->address_size == 2 && 2222 instr->regmodrm.mod != 0b11 && 2223 instr->regmodrm.rm <= 0b011); 2224 } 2225 2226 static enum x86_disp_type 2227 get_disp_type(struct x86_instr *instr) 2228 { 2229 switch (instr->regmodrm.mod) { 2230 case 0b00: /* indirect */ 2231 return DISP_0; 2232 case 0b01: /* indirect+1 */ 2233 return DISP_1; 2234 case 0b10: /* indirect+{2,4} */ 2235 if (__predict_false(instr->address_size == 2)) { 2236 return DISP_2; 2237 } 2238 return DISP_4; 2239 case 0b11: /* direct */ 2240 default: /* llvm */ 2241 return DISP_NONE; 2242 } 2243 __unreachable(); 2244 } 2245 2246 static int 2247 node_regmodrm(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2248 { 2249 struct x86_store *strg, *strm; 2250 const struct x86_opcode *opcode; 2251 const struct x86_reg *reg; 2252 uint8_t byte; 2253 2254 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2255 return -1; 2256 } 2257 2258 opcode = instr->opcode; 2259 2260 instr->regmodrm.rm = ((byte & 0b00000111) >> 0); 2261 instr->regmodrm.reg = ((byte & 0b00111000) >> 3); 2262 instr->regmodrm.mod = ((byte & 0b11000000) >> 6); 2263 2264 if (opcode->regtorm) { 2265 strg = &instr->src; 2266 strm = &instr->dst; 2267 } else { /* RM to REG */ 2268 strm = &instr->src; 2269 strg = &instr->dst; 2270 } 2271 2272 /* Save for later use. */ 2273 instr->strm = strm; 2274 2275 /* 2276 * Special cases: Groups. The REG field of REGMODRM is the index in 2277 * the group. op1 gets overwritten in the Immediate node, if any. 2278 */ 2279 if (opcode->group1) { 2280 if (group1[instr->regmodrm.reg].emul == NULL) { 2281 return -1; 2282 } 2283 instr->emul = group1[instr->regmodrm.reg].emul; 2284 } else if (opcode->group3) { 2285 if (group3[instr->regmodrm.reg].emul == NULL) { 2286 return -1; 2287 } 2288 instr->emul = group3[instr->regmodrm.reg].emul; 2289 } else if (opcode->group11) { 2290 if (group11[instr->regmodrm.reg].emul == NULL) { 2291 return -1; 2292 } 2293 instr->emul = group11[instr->regmodrm.reg].emul; 2294 } 2295 2296 if (!opcode->immediate) { 2297 reg = get_register_reg(instr, opcode); 2298 if (reg == NULL) { 2299 return -1; 2300 } 2301 strg->type = STORE_REG; 2302 strg->u.reg = reg; 2303 } 2304 2305 /* The displacement applies to RM. */ 2306 strm->disp.type = get_disp_type(instr); 2307 2308 if (has_sib(instr)) { 2309 /* Overwrites RM */ 2310 fsm_advance(fsm, 1, node_sib); 2311 return 0; 2312 } 2313 2314 if (is_rip_relative(fsm, instr)) { 2315 /* Overwrites RM */ 2316 strm->type = STORE_REG; 2317 strm->u.reg = &gpr_map__rip; 2318 strm->disp.type = DISP_4; 2319 fsm_advance(fsm, 1, node_disp); 2320 return 0; 2321 } 2322 2323 if (is_disp32_only(fsm, instr)) { 2324 /* Overwrites RM */ 2325 strm->type = STORE_REG; 2326 strm->u.reg = NULL; 2327 strm->disp.type = DISP_4; 2328 fsm_advance(fsm, 1, node_disp); 2329 return 0; 2330 } 2331 2332 if (__predict_false(is_disp16_only(fsm, instr))) { 2333 /* Overwrites RM */ 2334 strm->type = STORE_REG; 2335 strm->u.reg = NULL; 2336 strm->disp.type = DISP_2; 2337 fsm_advance(fsm, 1, node_disp); 2338 return 0; 2339 } 2340 2341 if (__predict_false(is_dual(fsm, instr))) { 2342 /* Overwrites RM */ 2343 fsm_advance(fsm, 0, node_dual); 2344 return 0; 2345 } 2346 2347 reg = get_register_rm(instr, opcode); 2348 if (reg == NULL) { 2349 return -1; 2350 } 2351 strm->type = STORE_REG; 2352 strm->u.reg = reg; 2353 2354 if (strm->disp.type == DISP_NONE) { 2355 /* Direct register addressing mode */ 2356 if (opcode->immediate) { 2357 fsm_advance(fsm, 1, node_immediate); 2358 } else { 2359 fsm_advance(fsm, 1, NULL); 2360 } 2361 } else if (strm->disp.type == DISP_0) { 2362 /* Indirect register addressing mode */ 2363 if (opcode->immediate) { 2364 fsm_advance(fsm, 1, node_immediate); 2365 } else { 2366 fsm_advance(fsm, 1, NULL); 2367 } 2368 } else { 2369 fsm_advance(fsm, 1, node_disp); 2370 } 2371 2372 return 0; 2373 } 2374 2375 static size_t 2376 get_operand_size(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2377 { 2378 const struct x86_opcode *opcode = instr->opcode; 2379 int opsize; 2380 2381 /* Get the opsize */ 2382 if (!opcode->szoverride) { 2383 opsize = opcode->defsize; 2384 } else if (instr->rexpref.present && instr->rexpref.w) { 2385 opsize = 8; 2386 } else { 2387 if (!fsm->is16bit) { 2388 if (instr->legpref.opr_ovr) { 2389 opsize = 2; 2390 } else { 2391 opsize = 4; 2392 } 2393 } else { /* 16bit */ 2394 if (instr->legpref.opr_ovr) { 2395 opsize = 4; 2396 } else { 2397 opsize = 2; 2398 } 2399 } 2400 } 2401 2402 return opsize; 2403 } 2404 2405 static size_t 2406 get_address_size(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2407 { 2408 if (fsm->is64bit) { 2409 if (__predict_false(instr->legpref.adr_ovr)) { 2410 return 4; 2411 } 2412 return 8; 2413 } 2414 2415 if (fsm->is32bit) { 2416 if (__predict_false(instr->legpref.adr_ovr)) { 2417 return 2; 2418 } 2419 return 4; 2420 } 2421 2422 /* 16bit. */ 2423 if (__predict_false(instr->legpref.adr_ovr)) { 2424 return 4; 2425 } 2426 return 2; 2427 } 2428 2429 static int 2430 node_primary_opcode(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2431 { 2432 const struct x86_opcode *opcode; 2433 uint8_t byte; 2434 2435 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2436 return -1; 2437 } 2438 2439 opcode = &primary_opcode_table[byte]; 2440 if (__predict_false(!opcode->valid)) { 2441 return -1; 2442 } 2443 2444 instr->opcode = opcode; 2445 instr->emul = opcode->emul; 2446 instr->operand_size = get_operand_size(fsm, instr); 2447 instr->address_size = get_address_size(fsm, instr); 2448 2449 if (fsm->is64bit && (instr->operand_size == 4)) { 2450 /* Zero-extend to 64 bits. */ 2451 instr->zeroextend_mask = ~size_to_mask(4); 2452 } 2453 2454 if (opcode->regmodrm) { 2455 fsm_advance(fsm, 1, node_regmodrm); 2456 } else if (opcode->dmo) { 2457 /* Direct-Memory Offsets */ 2458 fsm_advance(fsm, 1, node_dmo); 2459 } else if (opcode->stos || opcode->lods) { 2460 fsm_advance(fsm, 1, node_stlo); 2461 } else if (opcode->movs) { 2462 fsm_advance(fsm, 1, node_movs); 2463 } else { 2464 return -1; 2465 } 2466 2467 return 0; 2468 } 2469 2470 static int 2471 node_secondary_opcode(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2472 { 2473 const struct x86_opcode *opcode; 2474 uint8_t byte; 2475 2476 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2477 return -1; 2478 } 2479 2480 opcode = &secondary_opcode_table[byte]; 2481 if (__predict_false(!opcode->valid)) { 2482 return -1; 2483 } 2484 2485 instr->opcode = opcode; 2486 instr->emul = opcode->emul; 2487 instr->operand_size = get_operand_size(fsm, instr); 2488 instr->address_size = get_address_size(fsm, instr); 2489 2490 if (fsm->is64bit && (instr->operand_size == 4)) { 2491 /* Zero-extend to 64 bits. */ 2492 instr->zeroextend_mask = ~size_to_mask(4); 2493 } 2494 2495 if (opcode->flags & FLAG_ze) { 2496 /* 2497 * Compute the mask for zero-extend. Update the operand size, 2498 * we move fewer bytes. 2499 */ 2500 instr->zeroextend_mask |= size_to_mask(instr->operand_size); 2501 instr->zeroextend_mask &= ~size_to_mask(opcode->defsize); 2502 instr->operand_size = opcode->defsize; 2503 } 2504 2505 if (opcode->regmodrm) { 2506 fsm_advance(fsm, 1, node_regmodrm); 2507 } else { 2508 return -1; 2509 } 2510 2511 return 0; 2512 } 2513 2514 static int 2515 node_main(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2516 { 2517 uint8_t byte; 2518 2519 #define ESCAPE 0x0F 2520 #define VEX_1 0xC5 2521 #define VEX_2 0xC4 2522 #define XOP 0x8F 2523 2524 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2525 return -1; 2526 } 2527 2528 /* 2529 * We don't take XOP. It is AMD-specific, and it was removed shortly 2530 * after being introduced. 2531 */ 2532 if (byte == ESCAPE) { 2533 fsm_advance(fsm, 1, node_secondary_opcode); 2534 } else if (!instr->rexpref.present) { 2535 if (byte == VEX_1) { 2536 return -1; 2537 } else if (byte == VEX_2) { 2538 return -1; 2539 } else { 2540 fsm->fn = node_primary_opcode; 2541 } 2542 } else { 2543 fsm->fn = node_primary_opcode; 2544 } 2545 2546 return 0; 2547 } 2548 2549 static int 2550 node_rex_prefix(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2551 { 2552 struct x86_rexpref *rexpref = &instr->rexpref; 2553 uint8_t byte; 2554 size_t n = 0; 2555 2556 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2557 return -1; 2558 } 2559 2560 if (byte >= 0x40 && byte <= 0x4F) { 2561 if (__predict_false(!fsm->is64bit)) { 2562 return -1; 2563 } 2564 rexpref->b = ((byte & 0x1) != 0); 2565 rexpref->x = ((byte & 0x2) != 0); 2566 rexpref->r = ((byte & 0x4) != 0); 2567 rexpref->w = ((byte & 0x8) != 0); 2568 rexpref->present = true; 2569 n = 1; 2570 } 2571 2572 fsm_advance(fsm, n, node_main); 2573 return 0; 2574 } 2575 2576 static int 2577 node_legacy_prefix(struct x86_decode_fsm *fsm, struct x86_instr *instr) 2578 { 2579 uint8_t byte; 2580 2581 if (fsm_read(fsm, &byte, sizeof(byte)) == -1) { 2582 return -1; 2583 } 2584 2585 if (byte == LEG_OPR_OVR) { 2586 instr->legpref.opr_ovr = 1; 2587 } else if (byte == LEG_OVR_DS) { 2588 instr->legpref.seg = NVMM_X64_SEG_DS; 2589 } else if (byte == LEG_OVR_ES) { 2590 instr->legpref.seg = NVMM_X64_SEG_ES; 2591 } else if (byte == LEG_REP) { 2592 instr->legpref.rep = 1; 2593 } else if (byte == LEG_OVR_GS) { 2594 instr->legpref.seg = NVMM_X64_SEG_GS; 2595 } else if (byte == LEG_OVR_FS) { 2596 instr->legpref.seg = NVMM_X64_SEG_FS; 2597 } else if (byte == LEG_ADR_OVR) { 2598 instr->legpref.adr_ovr = 1; 2599 } else if (byte == LEG_OVR_CS) { 2600 instr->legpref.seg = NVMM_X64_SEG_CS; 2601 } else if (byte == LEG_OVR_SS) { 2602 instr->legpref.seg = NVMM_X64_SEG_SS; 2603 } else if (byte == LEG_REPN) { 2604 instr->legpref.repn = 1; 2605 } else if (byte == LEG_LOCK) { 2606 /* ignore */ 2607 } else { 2608 /* not a legacy prefix */ 2609 fsm_advance(fsm, 0, node_rex_prefix); 2610 return 0; 2611 } 2612 2613 fsm_advance(fsm, 1, node_legacy_prefix); 2614 return 0; 2615 } 2616 2617 static int 2618 x86_decode(uint8_t *inst_bytes, size_t inst_len, struct x86_instr *instr, 2619 struct nvmm_x64_state *state) 2620 { 2621 struct x86_decode_fsm fsm; 2622 int ret; 2623 2624 memset(instr, 0, sizeof(*instr)); 2625 instr->legpref.seg = -1; 2626 instr->src.hardseg = -1; 2627 instr->dst.hardseg = -1; 2628 2629 fsm.is64bit = is_64bit(state); 2630 fsm.is32bit = is_32bit(state); 2631 fsm.is16bit = is_16bit(state); 2632 2633 fsm.fn = node_legacy_prefix; 2634 fsm.buf = inst_bytes; 2635 fsm.end = inst_bytes + inst_len; 2636 2637 while (fsm.fn != NULL) { 2638 ret = (*fsm.fn)(&fsm, instr); 2639 if (ret == -1) 2640 return -1; 2641 } 2642 2643 instr->len = fsm.buf - inst_bytes; 2644 2645 return 0; 2646 } 2647 2648 /* -------------------------------------------------------------------------- */ 2649 2650 #define EXEC_INSTR(sz, instr) \ 2651 static uint##sz##_t \ 2652 exec_##instr##sz(uint##sz##_t op1, uint##sz##_t op2, uint64_t *rflags) \ 2653 { \ 2654 uint##sz##_t res; \ 2655 __asm __volatile ( \ 2656 #instr" %2, %3;" \ 2657 "mov %3, %1;" \ 2658 "pushfq;" \ 2659 "popq %0" \ 2660 : "=r" (*rflags), "=r" (res) \ 2661 : "r" (op1), "r" (op2)); \ 2662 return res; \ 2663 } 2664 2665 #define EXEC_DISPATCHER(instr) \ 2666 static uint64_t \ 2667 exec_##instr(uint64_t op1, uint64_t op2, uint64_t *rflags, size_t opsize) \ 2668 { \ 2669 switch (opsize) { \ 2670 case 1: \ 2671 return exec_##instr##8(op1, op2, rflags); \ 2672 case 2: \ 2673 return exec_##instr##16(op1, op2, rflags); \ 2674 case 4: \ 2675 return exec_##instr##32(op1, op2, rflags); \ 2676 default: \ 2677 return exec_##instr##64(op1, op2, rflags); \ 2678 } \ 2679 } 2680 2681 /* SUB: ret = op1 - op2 */ 2682 #define PSL_SUB_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF|PSL_AF) 2683 EXEC_INSTR(8, sub) 2684 EXEC_INSTR(16, sub) 2685 EXEC_INSTR(32, sub) 2686 EXEC_INSTR(64, sub) 2687 EXEC_DISPATCHER(sub) 2688 2689 /* OR: ret = op1 | op2 */ 2690 #define PSL_OR_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2691 EXEC_INSTR(8, or) 2692 EXEC_INSTR(16, or) 2693 EXEC_INSTR(32, or) 2694 EXEC_INSTR(64, or) 2695 EXEC_DISPATCHER(or) 2696 2697 /* AND: ret = op1 & op2 */ 2698 #define PSL_AND_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2699 EXEC_INSTR(8, and) 2700 EXEC_INSTR(16, and) 2701 EXEC_INSTR(32, and) 2702 EXEC_INSTR(64, and) 2703 EXEC_DISPATCHER(and) 2704 2705 /* XOR: ret = op1 ^ op2 */ 2706 #define PSL_XOR_MASK (PSL_V|PSL_C|PSL_Z|PSL_N|PSL_PF) 2707 EXEC_INSTR(8, xor) 2708 EXEC_INSTR(16, xor) 2709 EXEC_INSTR(32, xor) 2710 EXEC_INSTR(64, xor) 2711 EXEC_DISPATCHER(xor) 2712 2713 /* -------------------------------------------------------------------------- */ 2714 2715 /* 2716 * Emulation functions. We don't care about the order of the operands, except 2717 * for SUB, CMP and TEST. For these ones we look at mem->write to determine who 2718 * is op1 and who is op2. 2719 */ 2720 2721 static void 2722 x86_func_or(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2723 { 2724 uint64_t *retval = (uint64_t *)mem->data; 2725 const bool write = mem->write; 2726 uint64_t *op1, op2, fl, ret; 2727 2728 op1 = (uint64_t *)mem->data; 2729 op2 = 0; 2730 2731 /* Fetch the value to be OR'ed (op2). */ 2732 mem->data = (uint8_t *)&op2; 2733 mem->write = false; 2734 (*vcpu->cbs.mem)(mem); 2735 2736 /* Perform the OR. */ 2737 ret = exec_or(*op1, op2, &fl, mem->size); 2738 2739 if (write) { 2740 /* Write back the result. */ 2741 mem->data = (uint8_t *)&ret; 2742 mem->write = true; 2743 (*vcpu->cbs.mem)(mem); 2744 } else { 2745 /* Return data to the caller. */ 2746 *retval = ret; 2747 } 2748 2749 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_OR_MASK; 2750 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_OR_MASK); 2751 } 2752 2753 static void 2754 x86_func_and(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2755 { 2756 uint64_t *retval = (uint64_t *)mem->data; 2757 const bool write = mem->write; 2758 uint64_t *op1, op2, fl, ret; 2759 2760 op1 = (uint64_t *)mem->data; 2761 op2 = 0; 2762 2763 /* Fetch the value to be AND'ed (op2). */ 2764 mem->data = (uint8_t *)&op2; 2765 mem->write = false; 2766 (*vcpu->cbs.mem)(mem); 2767 2768 /* Perform the AND. */ 2769 ret = exec_and(*op1, op2, &fl, mem->size); 2770 2771 if (write) { 2772 /* Write back the result. */ 2773 mem->data = (uint8_t *)&ret; 2774 mem->write = true; 2775 (*vcpu->cbs.mem)(mem); 2776 } else { 2777 /* Return data to the caller. */ 2778 *retval = ret; 2779 } 2780 2781 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_AND_MASK; 2782 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_AND_MASK); 2783 } 2784 2785 static void 2786 x86_func_xchg(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs __unused) 2787 { 2788 uint64_t *op1, op2; 2789 2790 op1 = (uint64_t *)mem->data; 2791 op2 = 0; 2792 2793 /* Fetch op2. */ 2794 mem->data = (uint8_t *)&op2; 2795 mem->write = false; 2796 (*vcpu->cbs.mem)(mem); 2797 2798 /* Write op1 in op2. */ 2799 mem->data = (uint8_t *)op1; 2800 mem->write = true; 2801 (*vcpu->cbs.mem)(mem); 2802 2803 /* Write op2 in op1. */ 2804 *op1 = op2; 2805 } 2806 2807 static void 2808 x86_func_sub(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2809 { 2810 uint64_t *retval = (uint64_t *)mem->data; 2811 const bool write = mem->write; 2812 uint64_t *op1, *op2, fl, ret; 2813 uint64_t tmp; 2814 bool memop1; 2815 2816 memop1 = !mem->write; 2817 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2818 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2819 2820 /* Fetch the value to be SUB'ed (op1 or op2). */ 2821 mem->data = (uint8_t *)&tmp; 2822 mem->write = false; 2823 (*vcpu->cbs.mem)(mem); 2824 2825 /* Perform the SUB. */ 2826 ret = exec_sub(*op1, *op2, &fl, mem->size); 2827 2828 if (write) { 2829 /* Write back the result. */ 2830 mem->data = (uint8_t *)&ret; 2831 mem->write = true; 2832 (*vcpu->cbs.mem)(mem); 2833 } else { 2834 /* Return data to the caller. */ 2835 *retval = ret; 2836 } 2837 2838 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_SUB_MASK; 2839 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_SUB_MASK); 2840 } 2841 2842 static void 2843 x86_func_xor(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2844 { 2845 uint64_t *retval = (uint64_t *)mem->data; 2846 const bool write = mem->write; 2847 uint64_t *op1, op2, fl, ret; 2848 2849 op1 = (uint64_t *)mem->data; 2850 op2 = 0; 2851 2852 /* Fetch the value to be XOR'ed (op2). */ 2853 mem->data = (uint8_t *)&op2; 2854 mem->write = false; 2855 (*vcpu->cbs.mem)(mem); 2856 2857 /* Perform the XOR. */ 2858 ret = exec_xor(*op1, op2, &fl, mem->size); 2859 2860 if (write) { 2861 /* Write back the result. */ 2862 mem->data = (uint8_t *)&ret; 2863 mem->write = true; 2864 (*vcpu->cbs.mem)(mem); 2865 } else { 2866 /* Return data to the caller. */ 2867 *retval = ret; 2868 } 2869 2870 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_XOR_MASK; 2871 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_XOR_MASK); 2872 } 2873 2874 static void 2875 x86_func_cmp(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2876 { 2877 uint64_t *op1, *op2, fl; 2878 uint64_t tmp; 2879 bool memop1; 2880 2881 memop1 = !mem->write; 2882 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2883 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2884 2885 /* Fetch the value to be CMP'ed (op1 or op2). */ 2886 mem->data = (uint8_t *)&tmp; 2887 mem->write = false; 2888 (*vcpu->cbs.mem)(mem); 2889 2890 /* Perform the CMP. */ 2891 exec_sub(*op1, *op2, &fl, mem->size); 2892 2893 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_SUB_MASK; 2894 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_SUB_MASK); 2895 } 2896 2897 static void 2898 x86_func_test(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2899 { 2900 uint64_t *op1, *op2, fl; 2901 uint64_t tmp; 2902 bool memop1; 2903 2904 memop1 = !mem->write; 2905 op1 = memop1 ? &tmp : (uint64_t *)mem->data; 2906 op2 = memop1 ? (uint64_t *)mem->data : &tmp; 2907 2908 /* Fetch the value to be TEST'ed (op1 or op2). */ 2909 mem->data = (uint8_t *)&tmp; 2910 mem->write = false; 2911 (*vcpu->cbs.mem)(mem); 2912 2913 /* Perform the TEST. */ 2914 exec_and(*op1, *op2, &fl, mem->size); 2915 2916 gprs[NVMM_X64_GPR_RFLAGS] &= ~PSL_AND_MASK; 2917 gprs[NVMM_X64_GPR_RFLAGS] |= (fl & PSL_AND_MASK); 2918 } 2919 2920 static void 2921 x86_func_mov(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs __unused) 2922 { 2923 /* 2924 * Nothing special, just move without emulation. 2925 */ 2926 (*vcpu->cbs.mem)(mem); 2927 } 2928 2929 static void 2930 x86_func_stos(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2931 { 2932 /* 2933 * Just move, and update RDI. 2934 */ 2935 (*vcpu->cbs.mem)(mem); 2936 2937 if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2938 gprs[NVMM_X64_GPR_RDI] -= mem->size; 2939 } else { 2940 gprs[NVMM_X64_GPR_RDI] += mem->size; 2941 } 2942 } 2943 2944 static void 2945 x86_func_lods(struct nvmm_vcpu *vcpu, struct nvmm_mem *mem, uint64_t *gprs) 2946 { 2947 /* 2948 * Just move, and update RSI. 2949 */ 2950 (*vcpu->cbs.mem)(mem); 2951 2952 if (gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 2953 gprs[NVMM_X64_GPR_RSI] -= mem->size; 2954 } else { 2955 gprs[NVMM_X64_GPR_RSI] += mem->size; 2956 } 2957 } 2958 2959 /* -------------------------------------------------------------------------- */ 2960 2961 static inline uint64_t 2962 gpr_read_address(struct x86_instr *instr, struct nvmm_x64_state *state, int gpr) 2963 { 2964 uint64_t val; 2965 2966 val = state->gprs[gpr]; 2967 val &= size_to_mask(instr->address_size); 2968 2969 return val; 2970 } 2971 2972 static int 2973 store_to_gva(struct nvmm_x64_state *state, struct x86_instr *instr, 2974 struct x86_store *store, gvaddr_t *gvap, size_t size) 2975 { 2976 struct x86_sib *sib; 2977 gvaddr_t gva = 0; 2978 uint64_t reg; 2979 int ret, seg; 2980 2981 if (store->type == STORE_SIB) { 2982 sib = &store->u.sib; 2983 if (sib->bas != NULL) 2984 gva += gpr_read_address(instr, state, sib->bas->num); 2985 if (sib->idx != NULL) { 2986 reg = gpr_read_address(instr, state, sib->idx->num); 2987 gva += sib->scale * reg; 2988 } 2989 } else if (store->type == STORE_REG) { 2990 if (store->u.reg == NULL) { 2991 /* The base is null. Happens with disp32-only and 2992 * disp16-only. */ 2993 } else { 2994 gva = gpr_read_address(instr, state, store->u.reg->num); 2995 } 2996 } else if (store->type == STORE_DUALREG) { 2997 gva = gpr_read_address(instr, state, store->u.dualreg.reg1) + 2998 gpr_read_address(instr, state, store->u.dualreg.reg2); 2999 } else { 3000 gva = store->u.dmo; 3001 } 3002 3003 if (store->disp.type != DISP_NONE) { 3004 gva += store->disp.data; 3005 } 3006 3007 if (store->hardseg != -1) { 3008 seg = store->hardseg; 3009 } else { 3010 if (__predict_false(instr->legpref.seg != -1)) { 3011 seg = instr->legpref.seg; 3012 } else { 3013 seg = NVMM_X64_SEG_DS; 3014 } 3015 } 3016 3017 if (__predict_true(is_long_mode(state))) { 3018 if (seg == NVMM_X64_SEG_GS || seg == NVMM_X64_SEG_FS) { 3019 segment_apply(&state->segs[seg], &gva); 3020 } 3021 } else { 3022 ret = segment_check(&state->segs[seg], gva, size); 3023 if (ret == -1) 3024 return -1; 3025 segment_apply(&state->segs[seg], &gva); 3026 } 3027 3028 *gvap = gva; 3029 return 0; 3030 } 3031 3032 static int 3033 fetch_segment(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 3034 { 3035 struct nvmm_x64_state *state = vcpu->state; 3036 uint8_t inst_bytes[5], byte; 3037 size_t i, fetchsize; 3038 gvaddr_t gva; 3039 int ret, seg; 3040 3041 fetchsize = sizeof(inst_bytes); 3042 3043 gva = state->gprs[NVMM_X64_GPR_RIP]; 3044 if (__predict_false(!is_long_mode(state))) { 3045 ret = segment_check(&state->segs[NVMM_X64_SEG_CS], gva, 3046 fetchsize); 3047 if (ret == -1) 3048 return -1; 3049 segment_apply(&state->segs[NVMM_X64_SEG_CS], &gva); 3050 } 3051 3052 ret = read_guest_memory(mach, vcpu, gva, inst_bytes, fetchsize); 3053 if (ret == -1) 3054 return -1; 3055 3056 seg = NVMM_X64_SEG_DS; 3057 for (i = 0; i < fetchsize; i++) { 3058 byte = inst_bytes[i]; 3059 3060 if (byte == LEG_OVR_DS) { 3061 seg = NVMM_X64_SEG_DS; 3062 } else if (byte == LEG_OVR_ES) { 3063 seg = NVMM_X64_SEG_ES; 3064 } else if (byte == LEG_OVR_GS) { 3065 seg = NVMM_X64_SEG_GS; 3066 } else if (byte == LEG_OVR_FS) { 3067 seg = NVMM_X64_SEG_FS; 3068 } else if (byte == LEG_OVR_CS) { 3069 seg = NVMM_X64_SEG_CS; 3070 } else if (byte == LEG_OVR_SS) { 3071 seg = NVMM_X64_SEG_SS; 3072 } else if (byte == LEG_OPR_OVR) { 3073 /* nothing */ 3074 } else if (byte == LEG_ADR_OVR) { 3075 /* nothing */ 3076 } else if (byte == LEG_REP) { 3077 /* nothing */ 3078 } else if (byte == LEG_REPN) { 3079 /* nothing */ 3080 } else if (byte == LEG_LOCK) { 3081 /* nothing */ 3082 } else { 3083 return seg; 3084 } 3085 } 3086 3087 return seg; 3088 } 3089 3090 static int 3091 fetch_instruction(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 3092 struct nvmm_vcpu_exit *exit) 3093 { 3094 struct nvmm_x64_state *state = vcpu->state; 3095 size_t fetchsize; 3096 gvaddr_t gva; 3097 int ret; 3098 3099 fetchsize = sizeof(exit->u.mem.inst_bytes); 3100 3101 gva = state->gprs[NVMM_X64_GPR_RIP]; 3102 if (__predict_false(!is_long_mode(state))) { 3103 ret = segment_check(&state->segs[NVMM_X64_SEG_CS], gva, 3104 fetchsize); 3105 if (ret == -1) 3106 return -1; 3107 segment_apply(&state->segs[NVMM_X64_SEG_CS], &gva); 3108 } 3109 3110 ret = read_guest_memory(mach, vcpu, gva, exit->u.mem.inst_bytes, 3111 fetchsize); 3112 if (ret == -1) 3113 return -1; 3114 3115 exit->u.mem.inst_len = fetchsize; 3116 3117 return 0; 3118 } 3119 3120 /* 3121 * Double memory operand, MOVS only. 3122 */ 3123 static int 3124 assist_mem_double_movs(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 3125 struct x86_instr *instr) 3126 { 3127 struct nvmm_x64_state *state = vcpu->state; 3128 uint8_t data[8]; 3129 gvaddr_t gva; 3130 size_t size; 3131 int ret; 3132 3133 size = instr->operand_size; 3134 3135 /* Source. */ 3136 ret = store_to_gva(state, instr, &instr->src, &gva, size); 3137 if (ret == -1) 3138 return -1; 3139 ret = read_guest_memory(mach, vcpu, gva, data, size); 3140 if (ret == -1) 3141 return -1; 3142 3143 /* Destination. */ 3144 ret = store_to_gva(state, instr, &instr->dst, &gva, size); 3145 if (ret == -1) 3146 return -1; 3147 ret = write_guest_memory(mach, vcpu, gva, data, size); 3148 if (ret == -1) 3149 return -1; 3150 3151 if (state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_D) { 3152 state->gprs[NVMM_X64_GPR_RSI] -= size; 3153 state->gprs[NVMM_X64_GPR_RDI] -= size; 3154 } else { 3155 state->gprs[NVMM_X64_GPR_RSI] += size; 3156 state->gprs[NVMM_X64_GPR_RDI] += size; 3157 } 3158 3159 return 0; 3160 } 3161 3162 /* 3163 * Single memory operand, covers most instructions. 3164 */ 3165 static int 3166 assist_mem_single(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu, 3167 struct x86_instr *instr) 3168 { 3169 struct nvmm_x64_state *state = vcpu->state; 3170 struct nvmm_vcpu_exit *exit = vcpu->exit; 3171 struct nvmm_mem mem; 3172 uint8_t membuf[8]; 3173 uint64_t val; 3174 3175 memset(membuf, 0, sizeof(membuf)); 3176 3177 mem.mach = mach; 3178 mem.vcpu = vcpu; 3179 mem.gpa = exit->u.mem.gpa; 3180 mem.size = instr->operand_size; 3181 mem.data = membuf; 3182 3183 /* Determine the direction. */ 3184 switch (instr->src.type) { 3185 case STORE_REG: 3186 if (instr->src.disp.type != DISP_NONE) { 3187 /* Indirect access. */ 3188 mem.write = false; 3189 } else { 3190 /* Direct access. */ 3191 mem.write = true; 3192 } 3193 break; 3194 case STORE_DUALREG: 3195 if (instr->src.disp.type == DISP_NONE) { 3196 DISASSEMBLER_BUG(); 3197 } 3198 mem.write = false; 3199 break; 3200 case STORE_IMM: 3201 mem.write = true; 3202 break; 3203 case STORE_SIB: 3204 mem.write = false; 3205 break; 3206 case STORE_DMO: 3207 mem.write = false; 3208 break; 3209 default: 3210 DISASSEMBLER_BUG(); 3211 } 3212 3213 if (mem.write) { 3214 switch (instr->src.type) { 3215 case STORE_REG: 3216 /* The instruction was "reg -> mem". Fetch the register 3217 * in membuf. */ 3218 if (__predict_false(instr->src.disp.type != DISP_NONE)) { 3219 DISASSEMBLER_BUG(); 3220 } 3221 val = state->gprs[instr->src.u.reg->num]; 3222 val = __SHIFTOUT(val, instr->src.u.reg->mask); 3223 memcpy(mem.data, &val, mem.size); 3224 break; 3225 case STORE_IMM: 3226 /* The instruction was "imm -> mem". Fetch the immediate 3227 * in membuf. */ 3228 memcpy(mem.data, &instr->src.u.imm.data, mem.size); 3229 break; 3230 default: 3231 DISASSEMBLER_BUG(); 3232 } 3233 } else if (instr->emul->readreg) { 3234 /* The instruction was "mem -> reg", but the value of the 3235 * register matters for the emul func. Fetch it in membuf. */ 3236 if (__predict_false(instr->dst.type != STORE_REG)) { 3237 DISASSEMBLER_BUG(); 3238 } 3239 if (__predict_false(instr->dst.disp.type != DISP_NONE)) { 3240 DISASSEMBLER_BUG(); 3241 } 3242 val = state->gprs[instr->dst.u.reg->num]; 3243 val = __SHIFTOUT(val, instr->dst.u.reg->mask); 3244 memcpy(mem.data, &val, mem.size); 3245 } 3246 3247 (*instr->emul->func)(vcpu, &mem, state->gprs); 3248 3249 if (instr->emul->notouch) { 3250 /* We're done. */ 3251 return 0; 3252 } 3253 3254 if (!mem.write) { 3255 /* The instruction was "mem -> reg". The emul func has filled 3256 * membuf with the memory content. Install membuf in the 3257 * register. */ 3258 if (__predict_false(instr->dst.type != STORE_REG)) { 3259 DISASSEMBLER_BUG(); 3260 } 3261 if (__predict_false(instr->dst.disp.type != DISP_NONE)) { 3262 DISASSEMBLER_BUG(); 3263 } 3264 memcpy(&val, membuf, sizeof(uint64_t)); 3265 val = __SHIFTIN(val, instr->dst.u.reg->mask); 3266 state->gprs[instr->dst.u.reg->num] &= ~instr->dst.u.reg->mask; 3267 state->gprs[instr->dst.u.reg->num] |= val; 3268 state->gprs[instr->dst.u.reg->num] &= ~instr->zeroextend_mask; 3269 } else if (instr->emul->backprop) { 3270 /* The instruction was "reg -> mem", but the memory must be 3271 * back-propagated to the register. Install membuf in the 3272 * register. */ 3273 if (__predict_false(instr->src.type != STORE_REG)) { 3274 DISASSEMBLER_BUG(); 3275 } 3276 if (__predict_false(instr->src.disp.type != DISP_NONE)) { 3277 DISASSEMBLER_BUG(); 3278 } 3279 memcpy(&val, membuf, sizeof(uint64_t)); 3280 val = __SHIFTIN(val, instr->src.u.reg->mask); 3281 state->gprs[instr->src.u.reg->num] &= ~instr->src.u.reg->mask; 3282 state->gprs[instr->src.u.reg->num] |= val; 3283 state->gprs[instr->src.u.reg->num] &= ~instr->zeroextend_mask; 3284 } 3285 3286 return 0; 3287 } 3288 3289 int 3290 nvmm_assist_mem(struct nvmm_machine *mach, struct nvmm_vcpu *vcpu) 3291 { 3292 struct nvmm_x64_state *state = vcpu->state; 3293 struct nvmm_vcpu_exit *exit = vcpu->exit; 3294 struct x86_instr instr; 3295 uint64_t cnt = 0; /* GCC */ 3296 int ret; 3297 3298 if (__predict_false(exit->reason != NVMM_VCPU_EXIT_MEMORY)) { 3299 errno = EINVAL; 3300 return -1; 3301 } 3302 3303 ret = nvmm_vcpu_getstate(mach, vcpu, 3304 NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS | 3305 NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS); 3306 if (ret == -1) 3307 return -1; 3308 3309 if (exit->u.mem.inst_len == 0) { 3310 /* 3311 * The instruction was not fetched from the kernel. Fetch 3312 * it ourselves. 3313 */ 3314 ret = fetch_instruction(mach, vcpu, exit); 3315 if (ret == -1) 3316 return -1; 3317 } 3318 3319 ret = x86_decode(exit->u.mem.inst_bytes, exit->u.mem.inst_len, 3320 &instr, state); 3321 if (ret == -1) { 3322 errno = ENODEV; 3323 return -1; 3324 } 3325 3326 if (instr.legpref.rep || instr.legpref.repn) { 3327 cnt = rep_get_cnt(state, instr.address_size); 3328 if (__predict_false(cnt == 0)) { 3329 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3330 goto out; 3331 } 3332 } 3333 3334 if (instr.opcode->movs) { 3335 ret = assist_mem_double_movs(mach, vcpu, &instr); 3336 } else { 3337 ret = assist_mem_single(mach, vcpu, &instr); 3338 } 3339 if (ret == -1) { 3340 errno = ENODEV; 3341 return -1; 3342 } 3343 3344 if (instr.legpref.rep || instr.legpref.repn) { 3345 cnt -= 1; 3346 rep_set_cnt(state, instr.address_size, cnt); 3347 if (cnt == 0) { 3348 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3349 } else if (__predict_false(instr.legpref.repn)) { 3350 if (state->gprs[NVMM_X64_GPR_RFLAGS] & PSL_Z) { 3351 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3352 } 3353 } 3354 } else { 3355 state->gprs[NVMM_X64_GPR_RIP] += instr.len; 3356 } 3357 3358 out: 3359 ret = nvmm_vcpu_setstate(mach, vcpu, NVMM_X64_STATE_GPRS); 3360 if (ret == -1) 3361 return -1; 3362 3363 return 0; 3364 } 3365