1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * derived from drivers/kvm/kvm_main.c
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 * Copyright (C) 2008 Qumranet, Inc.
9 * Copyright IBM Corporation, 2008
10 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
11 *
12 * Authors:
13 * Avi Kivity <avi@qumranet.com>
14 * Yaniv Kamay <yaniv@qumranet.com>
15 * Amit Shah <amit.shah@qumranet.com>
16 * Ben-Ami Yassour <benami@il.ibm.com>
17 */
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/kvm_host.h>
21 #include "irq.h"
22 #include "ioapic.h"
23 #include "mmu.h"
24 #include "i8254.h"
25 #include "tss.h"
26 #include "kvm_cache_regs.h"
27 #include "kvm_emulate.h"
28 #include "mmu/page_track.h"
29 #include "x86.h"
30 #include "cpuid.h"
31 #include "pmu.h"
32 #include "hyperv.h"
33 #include "lapic.h"
34 #include "xen.h"
35 #include "smm.h"
36
37 #include <linux/clocksource.h>
38 #include <linux/interrupt.h>
39 #include <linux/kvm.h>
40 #include <linux/fs.h>
41 #include <linux/vmalloc.h>
42 #include <linux/export.h>
43 #include <linux/moduleparam.h>
44 #include <linux/mman.h>
45 #include <linux/highmem.h>
46 #include <linux/iommu.h>
47 #include <linux/cpufreq.h>
48 #include <linux/user-return-notifier.h>
49 #include <linux/srcu.h>
50 #include <linux/slab.h>
51 #include <linux/perf_event.h>
52 #include <linux/uaccess.h>
53 #include <linux/hash.h>
54 #include <linux/pci.h>
55 #include <linux/timekeeper_internal.h>
56 #include <linux/pvclock_gtod.h>
57 #include <linux/kvm_irqfd.h>
58 #include <linux/irqbypass.h>
59 #include <linux/sched/stat.h>
60 #include <linux/sched/isolation.h>
61 #include <linux/mem_encrypt.h>
62 #include <linux/entry-kvm.h>
63 #include <linux/suspend.h>
64 #include <linux/smp.h>
65
66 #include <trace/events/ipi.h>
67 #include <trace/events/kvm.h>
68
69 #include <asm/debugreg.h>
70 #include <asm/msr.h>
71 #include <asm/desc.h>
72 #include <asm/mce.h>
73 #include <asm/pkru.h>
74 #include <linux/kernel_stat.h>
75 #include <asm/fpu/api.h>
76 #include <asm/fpu/xcr.h>
77 #include <asm/fpu/xstate.h>
78 #include <asm/pvclock.h>
79 #include <asm/div64.h>
80 #include <asm/irq_remapping.h>
81 #include <asm/mshyperv.h>
82 #include <asm/hypervisor.h>
83 #include <asm/tlbflush.h>
84 #include <asm/intel_pt.h>
85 #include <asm/emulate_prefix.h>
86 #include <asm/sgx.h>
87 #include <clocksource/hyperv_timer.h>
88
89 #define CREATE_TRACE_POINTS
90 #include "trace.h"
91
92 #define MAX_IO_MSRS 256
93 #define KVM_MAX_MCE_BANKS 32
94
95 /*
96 * Note, kvm_caps fields should *never* have default values, all fields must be
97 * recomputed from scratch during vendor module load, e.g. to account for a
98 * vendor module being reloaded with different module parameters.
99 */
100 struct kvm_caps kvm_caps __read_mostly;
101 EXPORT_SYMBOL_GPL(kvm_caps);
102
103 #define ERR_PTR_USR(e) ((void __user *)ERR_PTR(e))
104
105 #define emul_to_vcpu(ctxt) \
106 ((struct kvm_vcpu *)(ctxt)->vcpu)
107
108 /* EFER defaults:
109 * - enable syscall per default because its emulated by KVM
110 * - enable LME and LMA per default on 64 bit KVM
111 */
112 #ifdef CONFIG_X86_64
113 static
114 u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
115 #else
116 static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
117 #endif
118
119 static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS;
120
121 #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE)
122
123 #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE
124
125 #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \
126 KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
127
128 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
129 static void process_nmi(struct kvm_vcpu *vcpu);
130 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
131 static void store_regs(struct kvm_vcpu *vcpu);
132 static int sync_regs(struct kvm_vcpu *vcpu);
133 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu);
134
135 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
136 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2);
137
138 static DEFINE_MUTEX(vendor_module_lock);
139 struct kvm_x86_ops kvm_x86_ops __read_mostly;
140
141 #define KVM_X86_OP(func) \
142 DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \
143 *(((struct kvm_x86_ops *)0)->func));
144 #define KVM_X86_OP_OPTIONAL KVM_X86_OP
145 #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP
146 #include <asm/kvm-x86-ops.h>
147 EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits);
148 EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg);
149
150 static bool __read_mostly ignore_msrs = 0;
151 module_param(ignore_msrs, bool, 0644);
152
153 bool __read_mostly report_ignored_msrs = true;
154 module_param(report_ignored_msrs, bool, 0644);
155 EXPORT_SYMBOL_GPL(report_ignored_msrs);
156
157 unsigned int min_timer_period_us = 200;
158 module_param(min_timer_period_us, uint, 0644);
159
160 static bool __read_mostly kvmclock_periodic_sync = true;
161 module_param(kvmclock_periodic_sync, bool, 0444);
162
163 /* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
164 static u32 __read_mostly tsc_tolerance_ppm = 250;
165 module_param(tsc_tolerance_ppm, uint, 0644);
166
167 static bool __read_mostly vector_hashing = true;
168 module_param(vector_hashing, bool, 0444);
169
170 bool __read_mostly enable_vmware_backdoor = false;
171 module_param(enable_vmware_backdoor, bool, 0444);
172 EXPORT_SYMBOL_GPL(enable_vmware_backdoor);
173
174 /*
175 * Flags to manipulate forced emulation behavior (any non-zero value will
176 * enable forced emulation).
177 */
178 #define KVM_FEP_CLEAR_RFLAGS_RF BIT(1)
179 static int __read_mostly force_emulation_prefix;
180 module_param(force_emulation_prefix, int, 0644);
181
182 int __read_mostly pi_inject_timer = -1;
183 module_param(pi_inject_timer, bint, 0644);
184
185 /* Enable/disable PMU virtualization */
186 bool __read_mostly enable_pmu = true;
187 EXPORT_SYMBOL_GPL(enable_pmu);
188 module_param(enable_pmu, bool, 0444);
189
190 bool __read_mostly eager_page_split = true;
191 module_param(eager_page_split, bool, 0644);
192
193 /* Enable/disable SMT_RSB bug mitigation */
194 static bool __read_mostly mitigate_smt_rsb;
195 module_param(mitigate_smt_rsb, bool, 0444);
196
197 /*
198 * Restoring the host value for MSRs that are only consumed when running in
199 * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU
200 * returns to userspace, i.e. the kernel can run with the guest's value.
201 */
202 #define KVM_MAX_NR_USER_RETURN_MSRS 16
203
204 struct kvm_user_return_msrs {
205 struct user_return_notifier urn;
206 bool registered;
207 struct kvm_user_return_msr_values {
208 u64 host;
209 u64 curr;
210 } values[KVM_MAX_NR_USER_RETURN_MSRS];
211 };
212
213 u32 __read_mostly kvm_nr_uret_msrs;
214 EXPORT_SYMBOL_GPL(kvm_nr_uret_msrs);
215 static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS];
216 static struct kvm_user_return_msrs __percpu *user_return_msrs;
217
218 #define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
219 | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
220 | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
221 | XFEATURE_MASK_PKRU | XFEATURE_MASK_XTILE)
222
223 u64 __read_mostly host_efer;
224 EXPORT_SYMBOL_GPL(host_efer);
225
226 bool __read_mostly allow_smaller_maxphyaddr = 0;
227 EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr);
228
229 bool __read_mostly enable_apicv = true;
230 EXPORT_SYMBOL_GPL(enable_apicv);
231
232 u64 __read_mostly host_xss;
233 EXPORT_SYMBOL_GPL(host_xss);
234
235 u64 __read_mostly host_arch_capabilities;
236 EXPORT_SYMBOL_GPL(host_arch_capabilities);
237
238 const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
239 KVM_GENERIC_VM_STATS(),
240 STATS_DESC_COUNTER(VM, mmu_shadow_zapped),
241 STATS_DESC_COUNTER(VM, mmu_pte_write),
242 STATS_DESC_COUNTER(VM, mmu_pde_zapped),
243 STATS_DESC_COUNTER(VM, mmu_flooded),
244 STATS_DESC_COUNTER(VM, mmu_recycled),
245 STATS_DESC_COUNTER(VM, mmu_cache_miss),
246 STATS_DESC_ICOUNTER(VM, mmu_unsync),
247 STATS_DESC_ICOUNTER(VM, pages_4k),
248 STATS_DESC_ICOUNTER(VM, pages_2m),
249 STATS_DESC_ICOUNTER(VM, pages_1g),
250 STATS_DESC_ICOUNTER(VM, nx_lpage_splits),
251 STATS_DESC_PCOUNTER(VM, max_mmu_rmap_size),
252 STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions)
253 };
254
255 const struct kvm_stats_header kvm_vm_stats_header = {
256 .name_size = KVM_STATS_NAME_SIZE,
257 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
258 .id_offset = sizeof(struct kvm_stats_header),
259 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
260 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
261 sizeof(kvm_vm_stats_desc),
262 };
263
264 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
265 KVM_GENERIC_VCPU_STATS(),
266 STATS_DESC_COUNTER(VCPU, pf_taken),
267 STATS_DESC_COUNTER(VCPU, pf_fixed),
268 STATS_DESC_COUNTER(VCPU, pf_emulate),
269 STATS_DESC_COUNTER(VCPU, pf_spurious),
270 STATS_DESC_COUNTER(VCPU, pf_fast),
271 STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created),
272 STATS_DESC_COUNTER(VCPU, pf_guest),
273 STATS_DESC_COUNTER(VCPU, tlb_flush),
274 STATS_DESC_COUNTER(VCPU, invlpg),
275 STATS_DESC_COUNTER(VCPU, exits),
276 STATS_DESC_COUNTER(VCPU, io_exits),
277 STATS_DESC_COUNTER(VCPU, mmio_exits),
278 STATS_DESC_COUNTER(VCPU, signal_exits),
279 STATS_DESC_COUNTER(VCPU, irq_window_exits),
280 STATS_DESC_COUNTER(VCPU, nmi_window_exits),
281 STATS_DESC_COUNTER(VCPU, l1d_flush),
282 STATS_DESC_COUNTER(VCPU, halt_exits),
283 STATS_DESC_COUNTER(VCPU, request_irq_exits),
284 STATS_DESC_COUNTER(VCPU, irq_exits),
285 STATS_DESC_COUNTER(VCPU, host_state_reload),
286 STATS_DESC_COUNTER(VCPU, fpu_reload),
287 STATS_DESC_COUNTER(VCPU, insn_emulation),
288 STATS_DESC_COUNTER(VCPU, insn_emulation_fail),
289 STATS_DESC_COUNTER(VCPU, hypercalls),
290 STATS_DESC_COUNTER(VCPU, irq_injections),
291 STATS_DESC_COUNTER(VCPU, nmi_injections),
292 STATS_DESC_COUNTER(VCPU, req_event),
293 STATS_DESC_COUNTER(VCPU, nested_run),
294 STATS_DESC_COUNTER(VCPU, directed_yield_attempted),
295 STATS_DESC_COUNTER(VCPU, directed_yield_successful),
296 STATS_DESC_COUNTER(VCPU, preemption_reported),
297 STATS_DESC_COUNTER(VCPU, preemption_other),
298 STATS_DESC_IBOOLEAN(VCPU, guest_mode),
299 STATS_DESC_COUNTER(VCPU, notify_window_exits),
300 };
301
302 const struct kvm_stats_header kvm_vcpu_stats_header = {
303 .name_size = KVM_STATS_NAME_SIZE,
304 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
305 .id_offset = sizeof(struct kvm_stats_header),
306 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
307 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
308 sizeof(kvm_vcpu_stats_desc),
309 };
310
311 u64 __read_mostly host_xcr0;
312
313 static struct kmem_cache *x86_emulator_cache;
314
315 /*
316 * When called, it means the previous get/set msr reached an invalid msr.
317 * Return true if we want to ignore/silent this failed msr access.
318 */
kvm_msr_ignored_check(u32 msr,u64 data,bool write)319 static bool kvm_msr_ignored_check(u32 msr, u64 data, bool write)
320 {
321 const char *op = write ? "wrmsr" : "rdmsr";
322
323 if (ignore_msrs) {
324 if (report_ignored_msrs)
325 kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n",
326 op, msr, data);
327 /* Mask the error */
328 return true;
329 } else {
330 kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n",
331 op, msr, data);
332 return false;
333 }
334 }
335
kvm_alloc_emulator_cache(void)336 static struct kmem_cache *kvm_alloc_emulator_cache(void)
337 {
338 unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src);
339 unsigned int size = sizeof(struct x86_emulate_ctxt);
340
341 return kmem_cache_create_usercopy("x86_emulator", size,
342 __alignof__(struct x86_emulate_ctxt),
343 SLAB_ACCOUNT, useroffset,
344 size - useroffset, NULL);
345 }
346
347 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);
348
kvm_async_pf_hash_reset(struct kvm_vcpu * vcpu)349 static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
350 {
351 int i;
352 for (i = 0; i < ASYNC_PF_PER_VCPU; i++)
353 vcpu->arch.apf.gfns[i] = ~0;
354 }
355
kvm_on_user_return(struct user_return_notifier * urn)356 static void kvm_on_user_return(struct user_return_notifier *urn)
357 {
358 unsigned slot;
359 struct kvm_user_return_msrs *msrs
360 = container_of(urn, struct kvm_user_return_msrs, urn);
361 struct kvm_user_return_msr_values *values;
362 unsigned long flags;
363
364 /*
365 * Disabling irqs at this point since the following code could be
366 * interrupted and executed through kvm_arch_hardware_disable()
367 */
368 local_irq_save(flags);
369 if (msrs->registered) {
370 msrs->registered = false;
371 user_return_notifier_unregister(urn);
372 }
373 local_irq_restore(flags);
374 for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) {
375 values = &msrs->values[slot];
376 if (values->host != values->curr) {
377 wrmsrl(kvm_uret_msrs_list[slot], values->host);
378 values->curr = values->host;
379 }
380 }
381 }
382
kvm_probe_user_return_msr(u32 msr)383 static int kvm_probe_user_return_msr(u32 msr)
384 {
385 u64 val;
386 int ret;
387
388 preempt_disable();
389 ret = rdmsrl_safe(msr, &val);
390 if (ret)
391 goto out;
392 ret = wrmsrl_safe(msr, val);
393 out:
394 preempt_enable();
395 return ret;
396 }
397
kvm_add_user_return_msr(u32 msr)398 int kvm_add_user_return_msr(u32 msr)
399 {
400 BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS);
401
402 if (kvm_probe_user_return_msr(msr))
403 return -1;
404
405 kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr;
406 return kvm_nr_uret_msrs++;
407 }
408 EXPORT_SYMBOL_GPL(kvm_add_user_return_msr);
409
kvm_find_user_return_msr(u32 msr)410 int kvm_find_user_return_msr(u32 msr)
411 {
412 int i;
413
414 for (i = 0; i < kvm_nr_uret_msrs; ++i) {
415 if (kvm_uret_msrs_list[i] == msr)
416 return i;
417 }
418 return -1;
419 }
420 EXPORT_SYMBOL_GPL(kvm_find_user_return_msr);
421
kvm_user_return_msr_cpu_online(void)422 static void kvm_user_return_msr_cpu_online(void)
423 {
424 unsigned int cpu = smp_processor_id();
425 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
426 u64 value;
427 int i;
428
429 for (i = 0; i < kvm_nr_uret_msrs; ++i) {
430 rdmsrl_safe(kvm_uret_msrs_list[i], &value);
431 msrs->values[i].host = value;
432 msrs->values[i].curr = value;
433 }
434 }
435
kvm_set_user_return_msr(unsigned slot,u64 value,u64 mask)436 int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask)
437 {
438 unsigned int cpu = smp_processor_id();
439 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
440 int err;
441
442 value = (value & mask) | (msrs->values[slot].host & ~mask);
443 if (value == msrs->values[slot].curr)
444 return 0;
445 err = wrmsrl_safe(kvm_uret_msrs_list[slot], value);
446 if (err)
447 return 1;
448
449 msrs->values[slot].curr = value;
450 if (!msrs->registered) {
451 msrs->urn.on_user_return = kvm_on_user_return;
452 user_return_notifier_register(&msrs->urn);
453 msrs->registered = true;
454 }
455 return 0;
456 }
457 EXPORT_SYMBOL_GPL(kvm_set_user_return_msr);
458
drop_user_return_notifiers(void)459 static void drop_user_return_notifiers(void)
460 {
461 unsigned int cpu = smp_processor_id();
462 struct kvm_user_return_msrs *msrs = per_cpu_ptr(user_return_msrs, cpu);
463
464 if (msrs->registered)
465 kvm_on_user_return(&msrs->urn);
466 }
467
kvm_get_apic_base(struct kvm_vcpu * vcpu)468 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
469 {
470 return vcpu->arch.apic_base;
471 }
472
kvm_get_apic_mode(struct kvm_vcpu * vcpu)473 enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu)
474 {
475 return kvm_apic_mode(kvm_get_apic_base(vcpu));
476 }
477 EXPORT_SYMBOL_GPL(kvm_get_apic_mode);
478
kvm_set_apic_base(struct kvm_vcpu * vcpu,struct msr_data * msr_info)479 int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
480 {
481 enum lapic_mode old_mode = kvm_get_apic_mode(vcpu);
482 enum lapic_mode new_mode = kvm_apic_mode(msr_info->data);
483 u64 reserved_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu) | 0x2ff |
484 (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) ? 0 : X2APIC_ENABLE);
485
486 if ((msr_info->data & reserved_bits) != 0 || new_mode == LAPIC_MODE_INVALID)
487 return 1;
488 if (!msr_info->host_initiated) {
489 if (old_mode == LAPIC_MODE_X2APIC && new_mode == LAPIC_MODE_XAPIC)
490 return 1;
491 if (old_mode == LAPIC_MODE_DISABLED && new_mode == LAPIC_MODE_X2APIC)
492 return 1;
493 }
494
495 kvm_lapic_set_base(vcpu, msr_info->data);
496 kvm_recalculate_apic_map(vcpu->kvm);
497 return 0;
498 }
499
500 /*
501 * Handle a fault on a hardware virtualization (VMX or SVM) instruction.
502 *
503 * Hardware virtualization extension instructions may fault if a reboot turns
504 * off virtualization while processes are running. Usually after catching the
505 * fault we just panic; during reboot instead the instruction is ignored.
506 */
kvm_spurious_fault(void)507 noinstr void kvm_spurious_fault(void)
508 {
509 /* Fault while not rebooting. We want the trace. */
510 BUG_ON(!kvm_rebooting);
511 }
512 EXPORT_SYMBOL_GPL(kvm_spurious_fault);
513
514 #define EXCPT_BENIGN 0
515 #define EXCPT_CONTRIBUTORY 1
516 #define EXCPT_PF 2
517
exception_class(int vector)518 static int exception_class(int vector)
519 {
520 switch (vector) {
521 case PF_VECTOR:
522 return EXCPT_PF;
523 case DE_VECTOR:
524 case TS_VECTOR:
525 case NP_VECTOR:
526 case SS_VECTOR:
527 case GP_VECTOR:
528 return EXCPT_CONTRIBUTORY;
529 default:
530 break;
531 }
532 return EXCPT_BENIGN;
533 }
534
535 #define EXCPT_FAULT 0
536 #define EXCPT_TRAP 1
537 #define EXCPT_ABORT 2
538 #define EXCPT_INTERRUPT 3
539 #define EXCPT_DB 4
540
exception_type(int vector)541 static int exception_type(int vector)
542 {
543 unsigned int mask;
544
545 if (WARN_ON(vector > 31 || vector == NMI_VECTOR))
546 return EXCPT_INTERRUPT;
547
548 mask = 1 << vector;
549
550 /*
551 * #DBs can be trap-like or fault-like, the caller must check other CPU
552 * state, e.g. DR6, to determine whether a #DB is a trap or fault.
553 */
554 if (mask & (1 << DB_VECTOR))
555 return EXCPT_DB;
556
557 if (mask & ((1 << BP_VECTOR) | (1 << OF_VECTOR)))
558 return EXCPT_TRAP;
559
560 if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR)))
561 return EXCPT_ABORT;
562
563 /* Reserved exceptions will result in fault */
564 return EXCPT_FAULT;
565 }
566
kvm_deliver_exception_payload(struct kvm_vcpu * vcpu,struct kvm_queued_exception * ex)567 void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu,
568 struct kvm_queued_exception *ex)
569 {
570 if (!ex->has_payload)
571 return;
572
573 switch (ex->vector) {
574 case DB_VECTOR:
575 /*
576 * "Certain debug exceptions may clear bit 0-3. The
577 * remaining contents of the DR6 register are never
578 * cleared by the processor".
579 */
580 vcpu->arch.dr6 &= ~DR_TRAP_BITS;
581 /*
582 * In order to reflect the #DB exception payload in guest
583 * dr6, three components need to be considered: active low
584 * bit, FIXED_1 bits and active high bits (e.g. DR6_BD,
585 * DR6_BS and DR6_BT)
586 * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits.
587 * In the target guest dr6:
588 * FIXED_1 bits should always be set.
589 * Active low bits should be cleared if 1-setting in payload.
590 * Active high bits should be set if 1-setting in payload.
591 *
592 * Note, the payload is compatible with the pending debug
593 * exceptions/exit qualification under VMX, that active_low bits
594 * are active high in payload.
595 * So they need to be flipped for DR6.
596 */
597 vcpu->arch.dr6 |= DR6_ACTIVE_LOW;
598 vcpu->arch.dr6 |= ex->payload;
599 vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW;
600
601 /*
602 * The #DB payload is defined as compatible with the 'pending
603 * debug exceptions' field under VMX, not DR6. While bit 12 is
604 * defined in the 'pending debug exceptions' field (enabled
605 * breakpoint), it is reserved and must be zero in DR6.
606 */
607 vcpu->arch.dr6 &= ~BIT(12);
608 break;
609 case PF_VECTOR:
610 vcpu->arch.cr2 = ex->payload;
611 break;
612 }
613
614 ex->has_payload = false;
615 ex->payload = 0;
616 }
617 EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload);
618
kvm_queue_exception_vmexit(struct kvm_vcpu * vcpu,unsigned int vector,bool has_error_code,u32 error_code,bool has_payload,unsigned long payload)619 static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector,
620 bool has_error_code, u32 error_code,
621 bool has_payload, unsigned long payload)
622 {
623 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;
624
625 ex->vector = vector;
626 ex->injected = false;
627 ex->pending = true;
628 ex->has_error_code = has_error_code;
629 ex->error_code = error_code;
630 ex->has_payload = has_payload;
631 ex->payload = payload;
632 }
633
634 /* Forcibly leave the nested mode in cases like a vCPU reset */
kvm_leave_nested(struct kvm_vcpu * vcpu)635 static void kvm_leave_nested(struct kvm_vcpu *vcpu)
636 {
637 kvm_x86_ops.nested_ops->leave_nested(vcpu);
638 }
639
kvm_multiple_exception(struct kvm_vcpu * vcpu,unsigned nr,bool has_error,u32 error_code,bool has_payload,unsigned long payload,bool reinject)640 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
641 unsigned nr, bool has_error, u32 error_code,
642 bool has_payload, unsigned long payload, bool reinject)
643 {
644 u32 prev_nr;
645 int class1, class2;
646
647 kvm_make_request(KVM_REQ_EVENT, vcpu);
648
649 /*
650 * If the exception is destined for L2 and isn't being reinjected,
651 * morph it to a VM-Exit if L1 wants to intercept the exception. A
652 * previously injected exception is not checked because it was checked
653 * when it was original queued, and re-checking is incorrect if _L1_
654 * injected the exception, in which case it's exempt from interception.
655 */
656 if (!reinject && is_guest_mode(vcpu) &&
657 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {
658 kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,
659 has_payload, payload);
660 return;
661 }
662
663 if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
664 queue:
665 if (reinject) {
666 /*
667 * On VM-Entry, an exception can be pending if and only
668 * if event injection was blocked by nested_run_pending.
669 * In that case, however, vcpu_enter_guest() requests an
670 * immediate exit, and the guest shouldn't proceed far
671 * enough to need reinjection.
672 */
673 WARN_ON_ONCE(kvm_is_exception_pending(vcpu));
674 vcpu->arch.exception.injected = true;
675 if (WARN_ON_ONCE(has_payload)) {
676 /*
677 * A reinjected event has already
678 * delivered its payload.
679 */
680 has_payload = false;
681 payload = 0;
682 }
683 } else {
684 vcpu->arch.exception.pending = true;
685 vcpu->arch.exception.injected = false;
686 }
687 vcpu->arch.exception.has_error_code = has_error;
688 vcpu->arch.exception.vector = nr;
689 vcpu->arch.exception.error_code = error_code;
690 vcpu->arch.exception.has_payload = has_payload;
691 vcpu->arch.exception.payload = payload;
692 if (!is_guest_mode(vcpu))
693 kvm_deliver_exception_payload(vcpu,
694 &vcpu->arch.exception);
695 return;
696 }
697
698 /* to check exception */
699 prev_nr = vcpu->arch.exception.vector;
700 if (prev_nr == DF_VECTOR) {
701 /* triple fault -> shutdown */
702 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
703 return;
704 }
705 class1 = exception_class(prev_nr);
706 class2 = exception_class(nr);
707 if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) ||
708 (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
709 /*
710 * Synthesize #DF. Clear the previously injected or pending
711 * exception so as not to incorrectly trigger shutdown.
712 */
713 vcpu->arch.exception.injected = false;
714 vcpu->arch.exception.pending = false;
715
716 kvm_queue_exception_e(vcpu, DF_VECTOR, 0);
717 } else {
718 /* replace previous exception with a new one in a hope
719 that instruction re-execution will regenerate lost
720 exception */
721 goto queue;
722 }
723 }
724
kvm_queue_exception(struct kvm_vcpu * vcpu,unsigned nr)725 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
726 {
727 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
728 }
729 EXPORT_SYMBOL_GPL(kvm_queue_exception);
730
kvm_requeue_exception(struct kvm_vcpu * vcpu,unsigned nr)731 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
732 {
733 kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
734 }
735 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
736
kvm_queue_exception_p(struct kvm_vcpu * vcpu,unsigned nr,unsigned long payload)737 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
738 unsigned long payload)
739 {
740 kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
741 }
742 EXPORT_SYMBOL_GPL(kvm_queue_exception_p);
743
kvm_queue_exception_e_p(struct kvm_vcpu * vcpu,unsigned nr,u32 error_code,unsigned long payload)744 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
745 u32 error_code, unsigned long payload)
746 {
747 kvm_multiple_exception(vcpu, nr, true, error_code,
748 true, payload, false);
749 }
750
kvm_complete_insn_gp(struct kvm_vcpu * vcpu,int err)751 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
752 {
753 if (err)
754 kvm_inject_gp(vcpu, 0);
755 else
756 return kvm_skip_emulated_instruction(vcpu);
757
758 return 1;
759 }
760 EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);
761
complete_emulated_insn_gp(struct kvm_vcpu * vcpu,int err)762 static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
763 {
764 if (err) {
765 kvm_inject_gp(vcpu, 0);
766 return 1;
767 }
768
769 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
770 EMULTYPE_COMPLETE_USER_EXIT);
771 }
772
kvm_inject_page_fault(struct kvm_vcpu * vcpu,struct x86_exception * fault)773 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
774 {
775 ++vcpu->stat.pf_guest;
776
777 /*
778 * Async #PF in L2 is always forwarded to L1 as a VM-Exit regardless of
779 * whether or not L1 wants to intercept "regular" #PF.
780 */
781 if (is_guest_mode(vcpu) && fault->async_page_fault)
782 kvm_queue_exception_vmexit(vcpu, PF_VECTOR,
783 true, fault->error_code,
784 true, fault->address);
785 else
786 kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code,
787 fault->address);
788 }
789
kvm_inject_emulated_page_fault(struct kvm_vcpu * vcpu,struct x86_exception * fault)790 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
791 struct x86_exception *fault)
792 {
793 struct kvm_mmu *fault_mmu;
794 WARN_ON_ONCE(fault->vector != PF_VECTOR);
795
796 fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu :
797 vcpu->arch.walk_mmu;
798
799 /*
800 * Invalidate the TLB entry for the faulting address, if it exists,
801 * else the access will fault indefinitely (and to emulate hardware).
802 */
803 if ((fault->error_code & PFERR_PRESENT_MASK) &&
804 !(fault->error_code & PFERR_RSVD_MASK))
805 kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,
806 KVM_MMU_ROOT_CURRENT);
807
808 fault_mmu->inject_page_fault(vcpu, fault);
809 }
810 EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault);
811
kvm_inject_nmi(struct kvm_vcpu * vcpu)812 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
813 {
814 atomic_inc(&vcpu->arch.nmi_queued);
815 kvm_make_request(KVM_REQ_NMI, vcpu);
816 }
817
kvm_queue_exception_e(struct kvm_vcpu * vcpu,unsigned nr,u32 error_code)818 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
819 {
820 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
821 }
822 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
823
kvm_requeue_exception_e(struct kvm_vcpu * vcpu,unsigned nr,u32 error_code)824 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
825 {
826 kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
827 }
828 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
829
830 /*
831 * Checks if cpl <= required_cpl; if true, return true. Otherwise queue
832 * a #GP and return false.
833 */
kvm_require_cpl(struct kvm_vcpu * vcpu,int required_cpl)834 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
835 {
836 if (static_call(kvm_x86_get_cpl)(vcpu) <= required_cpl)
837 return true;
838 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
839 return false;
840 }
841
kvm_require_dr(struct kvm_vcpu * vcpu,int dr)842 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
843 {
844 if ((dr != 4 && dr != 5) || !kvm_is_cr4_bit_set(vcpu, X86_CR4_DE))
845 return true;
846
847 kvm_queue_exception(vcpu, UD_VECTOR);
848 return false;
849 }
850 EXPORT_SYMBOL_GPL(kvm_require_dr);
851
pdptr_rsvd_bits(struct kvm_vcpu * vcpu)852 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
853 {
854 return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2);
855 }
856
857 /*
858 * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise.
859 */
load_pdptrs(struct kvm_vcpu * vcpu,unsigned long cr3)860 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
861 {
862 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
863 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
864 gpa_t real_gpa;
865 int i;
866 int ret;
867 u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];
868
869 /*
870 * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated
871 * to an L1 GPA.
872 */
873 real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),
874 PFERR_USER_MASK | PFERR_WRITE_MASK, NULL);
875 if (real_gpa == INVALID_GPA)
876 return 0;
877
878 /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */
879 ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte,
880 cr3 & GENMASK(11, 5), sizeof(pdpte));
881 if (ret < 0)
882 return 0;
883
884 for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
885 if ((pdpte[i] & PT_PRESENT_MASK) &&
886 (pdpte[i] & pdptr_rsvd_bits(vcpu))) {
887 return 0;
888 }
889 }
890
891 /*
892 * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled.
893 * Shadow page roots need to be reconstructed instead.
894 */
895 if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)))
896 kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);
897
898 memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
899 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
900 kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
901 vcpu->arch.pdptrs_from_userspace = false;
902
903 return 1;
904 }
905 EXPORT_SYMBOL_GPL(load_pdptrs);
906
kvm_is_valid_cr0(struct kvm_vcpu * vcpu,unsigned long cr0)907 static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
908 {
909 #ifdef CONFIG_X86_64
910 if (cr0 & 0xffffffff00000000UL)
911 return false;
912 #endif
913
914 if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
915 return false;
916
917 if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
918 return false;
919
920 return static_call(kvm_x86_is_valid_cr0)(vcpu, cr0);
921 }
922
kvm_post_set_cr0(struct kvm_vcpu * vcpu,unsigned long old_cr0,unsigned long cr0)923 void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
924 {
925 /*
926 * CR0.WP is incorporated into the MMU role, but only for non-nested,
927 * indirect shadow MMUs. If paging is disabled, no updates are needed
928 * as there are no permission bits to emulate. If TDP is enabled, the
929 * MMU's metadata needs to be updated, e.g. so that emulating guest
930 * translations does the right thing, but there's no need to unload the
931 * root as CR0.WP doesn't affect SPTEs.
932 */
933 if ((cr0 ^ old_cr0) == X86_CR0_WP) {
934 if (!(cr0 & X86_CR0_PG))
935 return;
936
937 if (tdp_enabled) {
938 kvm_init_mmu(vcpu);
939 return;
940 }
941 }
942
943 if ((cr0 ^ old_cr0) & X86_CR0_PG) {
944 kvm_clear_async_pf_completion_queue(vcpu);
945 kvm_async_pf_hash_reset(vcpu);
946
947 /*
948 * Clearing CR0.PG is defined to flush the TLB from the guest's
949 * perspective.
950 */
951 if (!(cr0 & X86_CR0_PG))
952 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
953 }
954
955 if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
956 kvm_mmu_reset_context(vcpu);
957
958 if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
959 kvm_mmu_honors_guest_mtrrs(vcpu->kvm) &&
960 !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
961 kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
962 }
963 EXPORT_SYMBOL_GPL(kvm_post_set_cr0);
964
kvm_set_cr0(struct kvm_vcpu * vcpu,unsigned long cr0)965 int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
966 {
967 unsigned long old_cr0 = kvm_read_cr0(vcpu);
968
969 if (!kvm_is_valid_cr0(vcpu, cr0))
970 return 1;
971
972 cr0 |= X86_CR0_ET;
973
974 /* Write to CR0 reserved bits are ignored, even on Intel. */
975 cr0 &= ~CR0_RESERVED_BITS;
976
977 #ifdef CONFIG_X86_64
978 if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) &&
979 (cr0 & X86_CR0_PG)) {
980 int cs_db, cs_l;
981
982 if (!is_pae(vcpu))
983 return 1;
984 static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
985 if (cs_l)
986 return 1;
987 }
988 #endif
989 if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) &&
990 is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) &&
991 !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
992 return 1;
993
994 if (!(cr0 & X86_CR0_PG) &&
995 (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)))
996 return 1;
997
998 static_call(kvm_x86_set_cr0)(vcpu, cr0);
999
1000 kvm_post_set_cr0(vcpu, old_cr0, cr0);
1001
1002 return 0;
1003 }
1004 EXPORT_SYMBOL_GPL(kvm_set_cr0);
1005
kvm_lmsw(struct kvm_vcpu * vcpu,unsigned long msw)1006 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
1007 {
1008 (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
1009 }
1010 EXPORT_SYMBOL_GPL(kvm_lmsw);
1011
kvm_load_guest_xsave_state(struct kvm_vcpu * vcpu)1012 void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu)
1013 {
1014 if (vcpu->arch.guest_state_protected)
1015 return;
1016
1017 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {
1018
1019 if (vcpu->arch.xcr0 != host_xcr0)
1020 xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
1021
1022 if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&
1023 vcpu->arch.ia32_xss != host_xss)
1024 wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss);
1025 }
1026
1027 if (cpu_feature_enabled(X86_FEATURE_PKU) &&
1028 vcpu->arch.pkru != vcpu->arch.host_pkru &&
1029 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
1030 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE)))
1031 write_pkru(vcpu->arch.pkru);
1032 }
1033 EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state);
1034
kvm_load_host_xsave_state(struct kvm_vcpu * vcpu)1035 void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu)
1036 {
1037 if (vcpu->arch.guest_state_protected)
1038 return;
1039
1040 if (cpu_feature_enabled(X86_FEATURE_PKU) &&
1041 ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) ||
1042 kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE))) {
1043 vcpu->arch.pkru = rdpkru();
1044 if (vcpu->arch.pkru != vcpu->arch.host_pkru)
1045 write_pkru(vcpu->arch.host_pkru);
1046 }
1047
1048 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE)) {
1049
1050 if (vcpu->arch.xcr0 != host_xcr0)
1051 xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
1052
1053 if (guest_can_use(vcpu, X86_FEATURE_XSAVES) &&
1054 vcpu->arch.ia32_xss != host_xss)
1055 wrmsrl(MSR_IA32_XSS, host_xss);
1056 }
1057
1058 }
1059 EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state);
1060
1061 #ifdef CONFIG_X86_64
kvm_guest_supported_xfd(struct kvm_vcpu * vcpu)1062 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu)
1063 {
1064 return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC;
1065 }
1066 #endif
1067
__kvm_set_xcr(struct kvm_vcpu * vcpu,u32 index,u64 xcr)1068 static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
1069 {
1070 u64 xcr0 = xcr;
1071 u64 old_xcr0 = vcpu->arch.xcr0;
1072 u64 valid_bits;
1073
1074 /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now */
1075 if (index != XCR_XFEATURE_ENABLED_MASK)
1076 return 1;
1077 if (!(xcr0 & XFEATURE_MASK_FP))
1078 return 1;
1079 if ((xcr0 & XFEATURE_MASK_YMM) && !(xcr0 & XFEATURE_MASK_SSE))
1080 return 1;
1081
1082 /*
1083 * Do not allow the guest to set bits that we do not support
1084 * saving. However, xcr0 bit 0 is always set, even if the
1085 * emulated CPU does not support XSAVE (see kvm_vcpu_reset()).
1086 */
1087 valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP;
1088 if (xcr0 & ~valid_bits)
1089 return 1;
1090
1091 if ((!(xcr0 & XFEATURE_MASK_BNDREGS)) !=
1092 (!(xcr0 & XFEATURE_MASK_BNDCSR)))
1093 return 1;
1094
1095 if (xcr0 & XFEATURE_MASK_AVX512) {
1096 if (!(xcr0 & XFEATURE_MASK_YMM))
1097 return 1;
1098 if ((xcr0 & XFEATURE_MASK_AVX512) != XFEATURE_MASK_AVX512)
1099 return 1;
1100 }
1101
1102 if ((xcr0 & XFEATURE_MASK_XTILE) &&
1103 ((xcr0 & XFEATURE_MASK_XTILE) != XFEATURE_MASK_XTILE))
1104 return 1;
1105
1106 vcpu->arch.xcr0 = xcr0;
1107
1108 if ((xcr0 ^ old_xcr0) & XFEATURE_MASK_EXTEND)
1109 kvm_update_cpuid_runtime(vcpu);
1110 return 0;
1111 }
1112
kvm_emulate_xsetbv(struct kvm_vcpu * vcpu)1113 int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu)
1114 {
1115 /* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */
1116 if (static_call(kvm_x86_get_cpl)(vcpu) != 0 ||
1117 __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) {
1118 kvm_inject_gp(vcpu, 0);
1119 return 1;
1120 }
1121
1122 return kvm_skip_emulated_instruction(vcpu);
1123 }
1124 EXPORT_SYMBOL_GPL(kvm_emulate_xsetbv);
1125
__kvm_is_valid_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1126 bool __kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1127 {
1128 if (cr4 & cr4_reserved_bits)
1129 return false;
1130
1131 if (cr4 & vcpu->arch.cr4_guest_rsvd_bits)
1132 return false;
1133
1134 return true;
1135 }
1136 EXPORT_SYMBOL_GPL(__kvm_is_valid_cr4);
1137
kvm_is_valid_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1138 static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1139 {
1140 return __kvm_is_valid_cr4(vcpu, cr4) &&
1141 static_call(kvm_x86_is_valid_cr4)(vcpu, cr4);
1142 }
1143
kvm_post_set_cr4(struct kvm_vcpu * vcpu,unsigned long old_cr4,unsigned long cr4)1144 void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4)
1145 {
1146 if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS)
1147 kvm_mmu_reset_context(vcpu);
1148
1149 /*
1150 * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB
1151 * according to the SDM; however, stale prev_roots could be reused
1152 * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we
1153 * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST
1154 * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed,
1155 * so fall through.
1156 */
1157 if (!tdp_enabled &&
1158 (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE))
1159 kvm_mmu_unload(vcpu);
1160
1161 /*
1162 * The TLB has to be flushed for all PCIDs if any of the following
1163 * (architecturally required) changes happen:
1164 * - CR4.PCIDE is changed from 1 to 0
1165 * - CR4.PGE is toggled
1166 *
1167 * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT.
1168 */
1169 if (((cr4 ^ old_cr4) & X86_CR4_PGE) ||
1170 (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
1171 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1172
1173 /*
1174 * The TLB has to be flushed for the current PCID if any of the
1175 * following (architecturally required) changes happen:
1176 * - CR4.SMEP is changed from 0 to 1
1177 * - CR4.PAE is toggled
1178 */
1179 else if (((cr4 ^ old_cr4) & X86_CR4_PAE) ||
1180 ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP)))
1181 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1182
1183 }
1184 EXPORT_SYMBOL_GPL(kvm_post_set_cr4);
1185
kvm_set_cr4(struct kvm_vcpu * vcpu,unsigned long cr4)1186 int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1187 {
1188 unsigned long old_cr4 = kvm_read_cr4(vcpu);
1189
1190 if (!kvm_is_valid_cr4(vcpu, cr4))
1191 return 1;
1192
1193 if (is_long_mode(vcpu)) {
1194 if (!(cr4 & X86_CR4_PAE))
1195 return 1;
1196 if ((cr4 ^ old_cr4) & X86_CR4_LA57)
1197 return 1;
1198 } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
1199 && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS)
1200 && !load_pdptrs(vcpu, kvm_read_cr3(vcpu)))
1201 return 1;
1202
1203 if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
1204 /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
1205 if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
1206 return 1;
1207 }
1208
1209 static_call(kvm_x86_set_cr4)(vcpu, cr4);
1210
1211 kvm_post_set_cr4(vcpu, old_cr4, cr4);
1212
1213 return 0;
1214 }
1215 EXPORT_SYMBOL_GPL(kvm_set_cr4);
1216
kvm_invalidate_pcid(struct kvm_vcpu * vcpu,unsigned long pcid)1217 static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid)
1218 {
1219 struct kvm_mmu *mmu = vcpu->arch.mmu;
1220 unsigned long roots_to_free = 0;
1221 int i;
1222
1223 /*
1224 * MOV CR3 and INVPCID are usually not intercepted when using TDP, but
1225 * this is reachable when running EPT=1 and unrestricted_guest=0, and
1226 * also via the emulator. KVM's TDP page tables are not in the scope of
1227 * the invalidation, but the guest's TLB entries need to be flushed as
1228 * the CPU may have cached entries in its TLB for the target PCID.
1229 */
1230 if (unlikely(tdp_enabled)) {
1231 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
1232 return;
1233 }
1234
1235 /*
1236 * If neither the current CR3 nor any of the prev_roots use the given
1237 * PCID, then nothing needs to be done here because a resync will
1238 * happen anyway before switching to any other CR3.
1239 */
1240 if (kvm_get_active_pcid(vcpu) == pcid) {
1241 kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
1242 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
1243 }
1244
1245 /*
1246 * If PCID is disabled, there is no need to free prev_roots even if the
1247 * PCIDs for them are also 0, because MOV to CR3 always flushes the TLB
1248 * with PCIDE=0.
1249 */
1250 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))
1251 return;
1252
1253 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
1254 if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid)
1255 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
1256
1257 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free);
1258 }
1259
kvm_set_cr3(struct kvm_vcpu * vcpu,unsigned long cr3)1260 int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1261 {
1262 bool skip_tlb_flush = false;
1263 unsigned long pcid = 0;
1264 #ifdef CONFIG_X86_64
1265 if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) {
1266 skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH;
1267 cr3 &= ~X86_CR3_PCID_NOFLUSH;
1268 pcid = cr3 & X86_CR3_PCID_MASK;
1269 }
1270 #endif
1271
1272 /* PDPTRs are always reloaded for PAE paging. */
1273 if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu))
1274 goto handle_tlb_flush;
1275
1276 /*
1277 * Do not condition the GPA check on long mode, this helper is used to
1278 * stuff CR3, e.g. for RSM emulation, and there is no guarantee that
1279 * the current vCPU mode is accurate.
1280 */
1281 if (!kvm_vcpu_is_legal_cr3(vcpu, cr3))
1282 return 1;
1283
1284 if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3))
1285 return 1;
1286
1287 if (cr3 != kvm_read_cr3(vcpu))
1288 kvm_mmu_new_pgd(vcpu, cr3);
1289
1290 vcpu->arch.cr3 = cr3;
1291 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
1292 /* Do not call post_set_cr3, we do not get here for confidential guests. */
1293
1294 handle_tlb_flush:
1295 /*
1296 * A load of CR3 that flushes the TLB flushes only the current PCID,
1297 * even if PCID is disabled, in which case PCID=0 is flushed. It's a
1298 * moot point in the end because _disabling_ PCID will flush all PCIDs,
1299 * and it's impossible to use a non-zero PCID when PCID is disabled,
1300 * i.e. only PCID=0 can be relevant.
1301 */
1302 if (!skip_tlb_flush)
1303 kvm_invalidate_pcid(vcpu, pcid);
1304
1305 return 0;
1306 }
1307 EXPORT_SYMBOL_GPL(kvm_set_cr3);
1308
kvm_set_cr8(struct kvm_vcpu * vcpu,unsigned long cr8)1309 int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
1310 {
1311 if (cr8 & CR8_RESERVED_BITS)
1312 return 1;
1313 if (lapic_in_kernel(vcpu))
1314 kvm_lapic_set_tpr(vcpu, cr8);
1315 else
1316 vcpu->arch.cr8 = cr8;
1317 return 0;
1318 }
1319 EXPORT_SYMBOL_GPL(kvm_set_cr8);
1320
kvm_get_cr8(struct kvm_vcpu * vcpu)1321 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
1322 {
1323 if (lapic_in_kernel(vcpu))
1324 return kvm_lapic_get_cr8(vcpu);
1325 else
1326 return vcpu->arch.cr8;
1327 }
1328 EXPORT_SYMBOL_GPL(kvm_get_cr8);
1329
kvm_update_dr0123(struct kvm_vcpu * vcpu)1330 static void kvm_update_dr0123(struct kvm_vcpu *vcpu)
1331 {
1332 int i;
1333
1334 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
1335 for (i = 0; i < KVM_NR_DB_REGS; i++)
1336 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
1337 }
1338 }
1339
kvm_update_dr7(struct kvm_vcpu * vcpu)1340 void kvm_update_dr7(struct kvm_vcpu *vcpu)
1341 {
1342 unsigned long dr7;
1343
1344 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1345 dr7 = vcpu->arch.guest_debug_dr7;
1346 else
1347 dr7 = vcpu->arch.dr7;
1348 static_call(kvm_x86_set_dr7)(vcpu, dr7);
1349 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
1350 if (dr7 & DR7_BP_EN_MASK)
1351 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
1352 }
1353 EXPORT_SYMBOL_GPL(kvm_update_dr7);
1354
kvm_dr6_fixed(struct kvm_vcpu * vcpu)1355 static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu)
1356 {
1357 u64 fixed = DR6_FIXED_1;
1358
1359 if (!guest_cpuid_has(vcpu, X86_FEATURE_RTM))
1360 fixed |= DR6_RTM;
1361
1362 if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))
1363 fixed |= DR6_BUS_LOCK;
1364 return fixed;
1365 }
1366
kvm_set_dr(struct kvm_vcpu * vcpu,int dr,unsigned long val)1367 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
1368 {
1369 size_t size = ARRAY_SIZE(vcpu->arch.db);
1370
1371 switch (dr) {
1372 case 0 ... 3:
1373 vcpu->arch.db[array_index_nospec(dr, size)] = val;
1374 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
1375 vcpu->arch.eff_db[dr] = val;
1376 break;
1377 case 4:
1378 case 6:
1379 if (!kvm_dr6_valid(val))
1380 return 1; /* #GP */
1381 vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu);
1382 break;
1383 case 5:
1384 default: /* 7 */
1385 if (!kvm_dr7_valid(val))
1386 return 1; /* #GP */
1387 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
1388 kvm_update_dr7(vcpu);
1389 break;
1390 }
1391
1392 return 0;
1393 }
1394 EXPORT_SYMBOL_GPL(kvm_set_dr);
1395
kvm_get_dr(struct kvm_vcpu * vcpu,int dr)1396 unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr)
1397 {
1398 size_t size = ARRAY_SIZE(vcpu->arch.db);
1399
1400 switch (dr) {
1401 case 0 ... 3:
1402 return vcpu->arch.db[array_index_nospec(dr, size)];
1403 case 4:
1404 case 6:
1405 return vcpu->arch.dr6;
1406 case 5:
1407 default: /* 7 */
1408 return vcpu->arch.dr7;
1409 }
1410 }
1411 EXPORT_SYMBOL_GPL(kvm_get_dr);
1412
kvm_emulate_rdpmc(struct kvm_vcpu * vcpu)1413 int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu)
1414 {
1415 u32 ecx = kvm_rcx_read(vcpu);
1416 u64 data;
1417
1418 if (kvm_pmu_rdpmc(vcpu, ecx, &data)) {
1419 kvm_inject_gp(vcpu, 0);
1420 return 1;
1421 }
1422
1423 kvm_rax_write(vcpu, (u32)data);
1424 kvm_rdx_write(vcpu, data >> 32);
1425 return kvm_skip_emulated_instruction(vcpu);
1426 }
1427 EXPORT_SYMBOL_GPL(kvm_emulate_rdpmc);
1428
1429 /*
1430 * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) track
1431 * the set of MSRs that KVM exposes to userspace through KVM_GET_MSRS,
1432 * KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. msrs_to_save holds MSRs that
1433 * require host support, i.e. should be probed via RDMSR. emulated_msrs holds
1434 * MSRs that KVM emulates without strictly requiring host support.
1435 * msr_based_features holds MSRs that enumerate features, i.e. are effectively
1436 * CPUID leafs. Note, msr_based_features isn't mutually exclusive with
1437 * msrs_to_save and emulated_msrs.
1438 */
1439
1440 static const u32 msrs_to_save_base[] = {
1441 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1442 MSR_STAR,
1443 #ifdef CONFIG_X86_64
1444 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1445 #endif
1446 MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
1447 MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
1448 MSR_IA32_SPEC_CTRL, MSR_IA32_TSX_CTRL,
1449 MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH,
1450 MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK,
1451 MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B,
1452 MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B,
1453 MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B,
1454 MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B,
1455 MSR_IA32_UMWAIT_CONTROL,
1456
1457 MSR_IA32_XFD, MSR_IA32_XFD_ERR,
1458 };
1459
1460 static const u32 msrs_to_save_pmu[] = {
1461 MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1,
1462 MSR_ARCH_PERFMON_FIXED_CTR0 + 2,
1463 MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS,
1464 MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
1465 MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG,
1466
1467 /* This part of MSRs should match KVM_INTEL_PMC_MAX_GENERIC. */
1468 MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1,
1469 MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3,
1470 MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5,
1471 MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7,
1472 MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1,
1473 MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3,
1474 MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5,
1475 MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7,
1476
1477 MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3,
1478 MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3,
1479
1480 /* This part of MSRs should match KVM_AMD_PMC_MAX_GENERIC. */
1481 MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2,
1482 MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5,
1483 MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2,
1484 MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5,
1485
1486 MSR_AMD64_PERF_CNTR_GLOBAL_CTL,
1487 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS,
1488 MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
1489 };
1490
1491 static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) +
1492 ARRAY_SIZE(msrs_to_save_pmu)];
1493 static unsigned num_msrs_to_save;
1494
1495 static const u32 emulated_msrs_all[] = {
1496 MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
1497 MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
1498
1499 #ifdef CONFIG_KVM_HYPERV
1500 HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
1501 HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
1502 HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY,
1503 HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2,
1504 HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL,
1505 HV_X64_MSR_RESET,
1506 HV_X64_MSR_VP_INDEX,
1507 HV_X64_MSR_VP_RUNTIME,
1508 HV_X64_MSR_SCONTROL,
1509 HV_X64_MSR_STIMER0_CONFIG,
1510 HV_X64_MSR_VP_ASSIST_PAGE,
1511 HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL,
1512 HV_X64_MSR_TSC_EMULATION_STATUS, HV_X64_MSR_TSC_INVARIANT_CONTROL,
1513 HV_X64_MSR_SYNDBG_OPTIONS,
1514 HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS,
1515 HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER,
1516 HV_X64_MSR_SYNDBG_PENDING_BUFFER,
1517 #endif
1518
1519 MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
1520 MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK,
1521
1522 MSR_IA32_TSC_ADJUST,
1523 MSR_IA32_TSC_DEADLINE,
1524 MSR_IA32_ARCH_CAPABILITIES,
1525 MSR_IA32_PERF_CAPABILITIES,
1526 MSR_IA32_MISC_ENABLE,
1527 MSR_IA32_MCG_STATUS,
1528 MSR_IA32_MCG_CTL,
1529 MSR_IA32_MCG_EXT_CTL,
1530 MSR_IA32_SMBASE,
1531 MSR_SMI_COUNT,
1532 MSR_PLATFORM_INFO,
1533 MSR_MISC_FEATURES_ENABLES,
1534 MSR_AMD64_VIRT_SPEC_CTRL,
1535 MSR_AMD64_TSC_RATIO,
1536 MSR_IA32_POWER_CTL,
1537 MSR_IA32_UCODE_REV,
1538
1539 /*
1540 * KVM always supports the "true" VMX control MSRs, even if the host
1541 * does not. The VMX MSRs as a whole are considered "emulated" as KVM
1542 * doesn't strictly require them to exist in the host (ignoring that
1543 * KVM would refuse to load in the first place if the core set of MSRs
1544 * aren't supported).
1545 */
1546 MSR_IA32_VMX_BASIC,
1547 MSR_IA32_VMX_TRUE_PINBASED_CTLS,
1548 MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
1549 MSR_IA32_VMX_TRUE_EXIT_CTLS,
1550 MSR_IA32_VMX_TRUE_ENTRY_CTLS,
1551 MSR_IA32_VMX_MISC,
1552 MSR_IA32_VMX_CR0_FIXED0,
1553 MSR_IA32_VMX_CR4_FIXED0,
1554 MSR_IA32_VMX_VMCS_ENUM,
1555 MSR_IA32_VMX_PROCBASED_CTLS2,
1556 MSR_IA32_VMX_EPT_VPID_CAP,
1557 MSR_IA32_VMX_VMFUNC,
1558
1559 MSR_K7_HWCR,
1560 MSR_KVM_POLL_CONTROL,
1561 };
1562
1563 static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)];
1564 static unsigned num_emulated_msrs;
1565
1566 /*
1567 * List of MSRs that control the existence of MSR-based features, i.e. MSRs
1568 * that are effectively CPUID leafs. VMX MSRs are also included in the set of
1569 * feature MSRs, but are handled separately to allow expedited lookups.
1570 */
1571 static const u32 msr_based_features_all_except_vmx[] = {
1572 MSR_AMD64_DE_CFG,
1573 MSR_IA32_UCODE_REV,
1574 MSR_IA32_ARCH_CAPABILITIES,
1575 MSR_IA32_PERF_CAPABILITIES,
1576 };
1577
1578 static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all_except_vmx) +
1579 (KVM_LAST_EMULATED_VMX_MSR - KVM_FIRST_EMULATED_VMX_MSR + 1)];
1580 static unsigned int num_msr_based_features;
1581
1582 /*
1583 * All feature MSRs except uCode revID, which tracks the currently loaded uCode
1584 * patch, are immutable once the vCPU model is defined.
1585 */
kvm_is_immutable_feature_msr(u32 msr)1586 static bool kvm_is_immutable_feature_msr(u32 msr)
1587 {
1588 int i;
1589
1590 if (msr >= KVM_FIRST_EMULATED_VMX_MSR && msr <= KVM_LAST_EMULATED_VMX_MSR)
1591 return true;
1592
1593 for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) {
1594 if (msr == msr_based_features_all_except_vmx[i])
1595 return msr != MSR_IA32_UCODE_REV;
1596 }
1597
1598 return false;
1599 }
1600
1601 /*
1602 * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM
1603 * does not yet virtualize. These include:
1604 * 10 - MISC_PACKAGE_CTRLS
1605 * 11 - ENERGY_FILTERING_CTL
1606 * 12 - DOITM
1607 * 18 - FB_CLEAR_CTRL
1608 * 21 - XAPIC_DISABLE_STATUS
1609 * 23 - OVERCLOCKING_STATUS
1610 */
1611
1612 #define KVM_SUPPORTED_ARCH_CAP \
1613 (ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \
1614 ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \
1615 ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \
1616 ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \
1617 ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO | ARCH_CAP_GDS_NO | \
1618 ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO)
1619
kvm_get_arch_capabilities(void)1620 static u64 kvm_get_arch_capabilities(void)
1621 {
1622 u64 data = host_arch_capabilities & KVM_SUPPORTED_ARCH_CAP;
1623
1624 /*
1625 * If nx_huge_pages is enabled, KVM's shadow paging will ensure that
1626 * the nested hypervisor runs with NX huge pages. If it is not,
1627 * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other
1628 * L1 guests, so it need not worry about its own (L2) guests.
1629 */
1630 data |= ARCH_CAP_PSCHANGE_MC_NO;
1631
1632 /*
1633 * If we're doing cache flushes (either "always" or "cond")
1634 * we will do one whenever the guest does a vmlaunch/vmresume.
1635 * If an outer hypervisor is doing the cache flush for us
1636 * (ARCH_CAP_SKIP_VMENTRY_L1DFLUSH), we can safely pass that
1637 * capability to the guest too, and if EPT is disabled we're not
1638 * vulnerable. Overall, only VMENTER_L1D_FLUSH_NEVER will
1639 * require a nested hypervisor to do a flush of its own.
1640 */
1641 if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER)
1642 data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH;
1643
1644 if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
1645 data |= ARCH_CAP_RDCL_NO;
1646 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1647 data |= ARCH_CAP_SSB_NO;
1648 if (!boot_cpu_has_bug(X86_BUG_MDS))
1649 data |= ARCH_CAP_MDS_NO;
1650 if (!boot_cpu_has_bug(X86_BUG_RFDS))
1651 data |= ARCH_CAP_RFDS_NO;
1652
1653 if (!boot_cpu_has(X86_FEATURE_RTM)) {
1654 /*
1655 * If RTM=0 because the kernel has disabled TSX, the host might
1656 * have TAA_NO or TSX_CTRL. Clear TAA_NO (the guest sees RTM=0
1657 * and therefore knows that there cannot be TAA) but keep
1658 * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts,
1659 * and we want to allow migrating those guests to tsx=off hosts.
1660 */
1661 data &= ~ARCH_CAP_TAA_NO;
1662 } else if (!boot_cpu_has_bug(X86_BUG_TAA)) {
1663 data |= ARCH_CAP_TAA_NO;
1664 } else {
1665 /*
1666 * Nothing to do here; we emulate TSX_CTRL if present on the
1667 * host so the guest can choose between disabling TSX or
1668 * using VERW to clear CPU buffers.
1669 */
1670 }
1671
1672 if (!boot_cpu_has_bug(X86_BUG_GDS) || gds_ucode_mitigated())
1673 data |= ARCH_CAP_GDS_NO;
1674
1675 return data;
1676 }
1677
kvm_get_msr_feature(struct kvm_msr_entry * msr)1678 static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
1679 {
1680 switch (msr->index) {
1681 case MSR_IA32_ARCH_CAPABILITIES:
1682 msr->data = kvm_get_arch_capabilities();
1683 break;
1684 case MSR_IA32_PERF_CAPABILITIES:
1685 msr->data = kvm_caps.supported_perf_cap;
1686 break;
1687 case MSR_IA32_UCODE_REV:
1688 rdmsrl_safe(msr->index, &msr->data);
1689 break;
1690 default:
1691 return static_call(kvm_x86_get_msr_feature)(msr);
1692 }
1693 return 0;
1694 }
1695
do_get_msr_feature(struct kvm_vcpu * vcpu,unsigned index,u64 * data)1696 static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1697 {
1698 struct kvm_msr_entry msr;
1699 int r;
1700
1701 /* Unconditionally clear the output for simplicity */
1702 msr.data = 0;
1703 msr.index = index;
1704 r = kvm_get_msr_feature(&msr);
1705
1706 if (r == KVM_MSR_RET_INVALID && kvm_msr_ignored_check(index, 0, false))
1707 r = 0;
1708
1709 *data = msr.data;
1710
1711 return r;
1712 }
1713
__kvm_valid_efer(struct kvm_vcpu * vcpu,u64 efer)1714 static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1715 {
1716 if (efer & EFER_AUTOIBRS && !guest_cpuid_has(vcpu, X86_FEATURE_AUTOIBRS))
1717 return false;
1718
1719 if (efer & EFER_FFXSR && !guest_cpuid_has(vcpu, X86_FEATURE_FXSR_OPT))
1720 return false;
1721
1722 if (efer & EFER_SVME && !guest_cpuid_has(vcpu, X86_FEATURE_SVM))
1723 return false;
1724
1725 if (efer & (EFER_LME | EFER_LMA) &&
1726 !guest_cpuid_has(vcpu, X86_FEATURE_LM))
1727 return false;
1728
1729 if (efer & EFER_NX && !guest_cpuid_has(vcpu, X86_FEATURE_NX))
1730 return false;
1731
1732 return true;
1733
1734 }
kvm_valid_efer(struct kvm_vcpu * vcpu,u64 efer)1735 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
1736 {
1737 if (efer & efer_reserved_bits)
1738 return false;
1739
1740 return __kvm_valid_efer(vcpu, efer);
1741 }
1742 EXPORT_SYMBOL_GPL(kvm_valid_efer);
1743
set_efer(struct kvm_vcpu * vcpu,struct msr_data * msr_info)1744 static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
1745 {
1746 u64 old_efer = vcpu->arch.efer;
1747 u64 efer = msr_info->data;
1748 int r;
1749
1750 if (efer & efer_reserved_bits)
1751 return 1;
1752
1753 if (!msr_info->host_initiated) {
1754 if (!__kvm_valid_efer(vcpu, efer))
1755 return 1;
1756
1757 if (is_paging(vcpu) &&
1758 (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
1759 return 1;
1760 }
1761
1762 efer &= ~EFER_LMA;
1763 efer |= vcpu->arch.efer & EFER_LMA;
1764
1765 r = static_call(kvm_x86_set_efer)(vcpu, efer);
1766 if (r) {
1767 WARN_ON(r > 0);
1768 return r;
1769 }
1770
1771 if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS)
1772 kvm_mmu_reset_context(vcpu);
1773
1774 if (!static_cpu_has(X86_FEATURE_XSAVES) &&
1775 (efer & EFER_SVME))
1776 kvm_hv_xsaves_xsavec_maybe_warn(vcpu);
1777
1778 return 0;
1779 }
1780
kvm_enable_efer_bits(u64 mask)1781 void kvm_enable_efer_bits(u64 mask)
1782 {
1783 efer_reserved_bits &= ~mask;
1784 }
1785 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
1786
kvm_msr_allowed(struct kvm_vcpu * vcpu,u32 index,u32 type)1787 bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type)
1788 {
1789 struct kvm_x86_msr_filter *msr_filter;
1790 struct msr_bitmap_range *ranges;
1791 struct kvm *kvm = vcpu->kvm;
1792 bool allowed;
1793 int idx;
1794 u32 i;
1795
1796 /* x2APIC MSRs do not support filtering. */
1797 if (index >= 0x800 && index <= 0x8ff)
1798 return true;
1799
1800 idx = srcu_read_lock(&kvm->srcu);
1801
1802 msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu);
1803 if (!msr_filter) {
1804 allowed = true;
1805 goto out;
1806 }
1807
1808 allowed = msr_filter->default_allow;
1809 ranges = msr_filter->ranges;
1810
1811 for (i = 0; i < msr_filter->count; i++) {
1812 u32 start = ranges[i].base;
1813 u32 end = start + ranges[i].nmsrs;
1814 u32 flags = ranges[i].flags;
1815 unsigned long *bitmap = ranges[i].bitmap;
1816
1817 if ((index >= start) && (index < end) && (flags & type)) {
1818 allowed = test_bit(index - start, bitmap);
1819 break;
1820 }
1821 }
1822
1823 out:
1824 srcu_read_unlock(&kvm->srcu, idx);
1825
1826 return allowed;
1827 }
1828 EXPORT_SYMBOL_GPL(kvm_msr_allowed);
1829
1830 /*
1831 * Write @data into the MSR specified by @index. Select MSR specific fault
1832 * checks are bypassed if @host_initiated is %true.
1833 * Returns 0 on success, non-0 otherwise.
1834 * Assumes vcpu_load() was already called.
1835 */
__kvm_set_msr(struct kvm_vcpu * vcpu,u32 index,u64 data,bool host_initiated)1836 static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data,
1837 bool host_initiated)
1838 {
1839 struct msr_data msr;
1840
1841 switch (index) {
1842 case MSR_FS_BASE:
1843 case MSR_GS_BASE:
1844 case MSR_KERNEL_GS_BASE:
1845 case MSR_CSTAR:
1846 case MSR_LSTAR:
1847 if (is_noncanonical_address(data, vcpu))
1848 return 1;
1849 break;
1850 case MSR_IA32_SYSENTER_EIP:
1851 case MSR_IA32_SYSENTER_ESP:
1852 /*
1853 * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if
1854 * non-canonical address is written on Intel but not on
1855 * AMD (which ignores the top 32-bits, because it does
1856 * not implement 64-bit SYSENTER).
1857 *
1858 * 64-bit code should hence be able to write a non-canonical
1859 * value on AMD. Making the address canonical ensures that
1860 * vmentry does not fail on Intel after writing a non-canonical
1861 * value, and that something deterministic happens if the guest
1862 * invokes 64-bit SYSENTER.
1863 */
1864 data = __canonical_address(data, vcpu_virt_addr_bits(vcpu));
1865 break;
1866 case MSR_TSC_AUX:
1867 if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
1868 return 1;
1869
1870 if (!host_initiated &&
1871 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1872 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1873 return 1;
1874
1875 /*
1876 * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has
1877 * incomplete and conflicting architectural behavior. Current
1878 * AMD CPUs completely ignore bits 63:32, i.e. they aren't
1879 * reserved and always read as zeros. Enforce Intel's reserved
1880 * bits check if and only if the guest CPU is Intel, and clear
1881 * the bits in all other cases. This ensures cross-vendor
1882 * migration will provide consistent behavior for the guest.
1883 */
1884 if (guest_cpuid_is_intel(vcpu) && (data >> 32) != 0)
1885 return 1;
1886
1887 data = (u32)data;
1888 break;
1889 }
1890
1891 msr.data = data;
1892 msr.index = index;
1893 msr.host_initiated = host_initiated;
1894
1895 return static_call(kvm_x86_set_msr)(vcpu, &msr);
1896 }
1897
kvm_set_msr_ignored_check(struct kvm_vcpu * vcpu,u32 index,u64 data,bool host_initiated)1898 static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu,
1899 u32 index, u64 data, bool host_initiated)
1900 {
1901 int ret = __kvm_set_msr(vcpu, index, data, host_initiated);
1902
1903 if (ret == KVM_MSR_RET_INVALID)
1904 if (kvm_msr_ignored_check(index, data, true))
1905 ret = 0;
1906
1907 return ret;
1908 }
1909
1910 /*
1911 * Read the MSR specified by @index into @data. Select MSR specific fault
1912 * checks are bypassed if @host_initiated is %true.
1913 * Returns 0 on success, non-0 otherwise.
1914 * Assumes vcpu_load() was already called.
1915 */
__kvm_get_msr(struct kvm_vcpu * vcpu,u32 index,u64 * data,bool host_initiated)1916 int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data,
1917 bool host_initiated)
1918 {
1919 struct msr_data msr;
1920 int ret;
1921
1922 switch (index) {
1923 case MSR_TSC_AUX:
1924 if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX))
1925 return 1;
1926
1927 if (!host_initiated &&
1928 !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP) &&
1929 !guest_cpuid_has(vcpu, X86_FEATURE_RDPID))
1930 return 1;
1931 break;
1932 }
1933
1934 msr.index = index;
1935 msr.host_initiated = host_initiated;
1936
1937 ret = static_call(kvm_x86_get_msr)(vcpu, &msr);
1938 if (!ret)
1939 *data = msr.data;
1940 return ret;
1941 }
1942
kvm_get_msr_ignored_check(struct kvm_vcpu * vcpu,u32 index,u64 * data,bool host_initiated)1943 static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
1944 u32 index, u64 *data, bool host_initiated)
1945 {
1946 int ret = __kvm_get_msr(vcpu, index, data, host_initiated);
1947
1948 if (ret == KVM_MSR_RET_INVALID) {
1949 /* Unconditionally clear *data for simplicity */
1950 *data = 0;
1951 if (kvm_msr_ignored_check(index, 0, false))
1952 ret = 0;
1953 }
1954
1955 return ret;
1956 }
1957
kvm_get_msr_with_filter(struct kvm_vcpu * vcpu,u32 index,u64 * data)1958 static int kvm_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1959 {
1960 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ))
1961 return KVM_MSR_RET_FILTERED;
1962 return kvm_get_msr_ignored_check(vcpu, index, data, false);
1963 }
1964
kvm_set_msr_with_filter(struct kvm_vcpu * vcpu,u32 index,u64 data)1965 static int kvm_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 index, u64 data)
1966 {
1967 if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE))
1968 return KVM_MSR_RET_FILTERED;
1969 return kvm_set_msr_ignored_check(vcpu, index, data, false);
1970 }
1971
kvm_get_msr(struct kvm_vcpu * vcpu,u32 index,u64 * data)1972 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data)
1973 {
1974 return kvm_get_msr_ignored_check(vcpu, index, data, false);
1975 }
1976 EXPORT_SYMBOL_GPL(kvm_get_msr);
1977
kvm_set_msr(struct kvm_vcpu * vcpu,u32 index,u64 data)1978 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
1979 {
1980 return kvm_set_msr_ignored_check(vcpu, index, data, false);
1981 }
1982 EXPORT_SYMBOL_GPL(kvm_set_msr);
1983
complete_userspace_rdmsr(struct kvm_vcpu * vcpu)1984 static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu)
1985 {
1986 if (!vcpu->run->msr.error) {
1987 kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
1988 kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
1989 }
1990 }
1991
complete_emulated_msr_access(struct kvm_vcpu * vcpu)1992 static int complete_emulated_msr_access(struct kvm_vcpu *vcpu)
1993 {
1994 return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error);
1995 }
1996
complete_emulated_rdmsr(struct kvm_vcpu * vcpu)1997 static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
1998 {
1999 complete_userspace_rdmsr(vcpu);
2000 return complete_emulated_msr_access(vcpu);
2001 }
2002
complete_fast_msr_access(struct kvm_vcpu * vcpu)2003 static int complete_fast_msr_access(struct kvm_vcpu *vcpu)
2004 {
2005 return static_call(kvm_x86_complete_emulated_msr)(vcpu, vcpu->run->msr.error);
2006 }
2007
complete_fast_rdmsr(struct kvm_vcpu * vcpu)2008 static int complete_fast_rdmsr(struct kvm_vcpu *vcpu)
2009 {
2010 complete_userspace_rdmsr(vcpu);
2011 return complete_fast_msr_access(vcpu);
2012 }
2013
kvm_msr_reason(int r)2014 static u64 kvm_msr_reason(int r)
2015 {
2016 switch (r) {
2017 case KVM_MSR_RET_INVALID:
2018 return KVM_MSR_EXIT_REASON_UNKNOWN;
2019 case KVM_MSR_RET_FILTERED:
2020 return KVM_MSR_EXIT_REASON_FILTER;
2021 default:
2022 return KVM_MSR_EXIT_REASON_INVAL;
2023 }
2024 }
2025
kvm_msr_user_space(struct kvm_vcpu * vcpu,u32 index,u32 exit_reason,u64 data,int (* completion)(struct kvm_vcpu * vcpu),int r)2026 static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index,
2027 u32 exit_reason, u64 data,
2028 int (*completion)(struct kvm_vcpu *vcpu),
2029 int r)
2030 {
2031 u64 msr_reason = kvm_msr_reason(r);
2032
2033 /* Check if the user wanted to know about this MSR fault */
2034 if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason))
2035 return 0;
2036
2037 vcpu->run->exit_reason = exit_reason;
2038 vcpu->run->msr.error = 0;
2039 memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad));
2040 vcpu->run->msr.reason = msr_reason;
2041 vcpu->run->msr.index = index;
2042 vcpu->run->msr.data = data;
2043 vcpu->arch.complete_userspace_io = completion;
2044
2045 return 1;
2046 }
2047
kvm_emulate_rdmsr(struct kvm_vcpu * vcpu)2048 int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
2049 {
2050 u32 ecx = kvm_rcx_read(vcpu);
2051 u64 data;
2052 int r;
2053
2054 r = kvm_get_msr_with_filter(vcpu, ecx, &data);
2055
2056 if (!r) {
2057 trace_kvm_msr_read(ecx, data);
2058
2059 kvm_rax_write(vcpu, data & -1u);
2060 kvm_rdx_write(vcpu, (data >> 32) & -1u);
2061 } else {
2062 /* MSR read failed? See if we should ask user space */
2063 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_RDMSR, 0,
2064 complete_fast_rdmsr, r))
2065 return 0;
2066 trace_kvm_msr_read_ex(ecx);
2067 }
2068
2069 return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
2070 }
2071 EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
2072
kvm_emulate_wrmsr(struct kvm_vcpu * vcpu)2073 int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
2074 {
2075 u32 ecx = kvm_rcx_read(vcpu);
2076 u64 data = kvm_read_edx_eax(vcpu);
2077 int r;
2078
2079 r = kvm_set_msr_with_filter(vcpu, ecx, data);
2080
2081 if (!r) {
2082 trace_kvm_msr_write(ecx, data);
2083 } else {
2084 /* MSR write failed? See if we should ask user space */
2085 if (kvm_msr_user_space(vcpu, ecx, KVM_EXIT_X86_WRMSR, data,
2086 complete_fast_msr_access, r))
2087 return 0;
2088 /* Signal all other negative errors to userspace */
2089 if (r < 0)
2090 return r;
2091 trace_kvm_msr_write_ex(ecx, data);
2092 }
2093
2094 return static_call(kvm_x86_complete_emulated_msr)(vcpu, r);
2095 }
2096 EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
2097
kvm_emulate_as_nop(struct kvm_vcpu * vcpu)2098 int kvm_emulate_as_nop(struct kvm_vcpu *vcpu)
2099 {
2100 return kvm_skip_emulated_instruction(vcpu);
2101 }
2102
kvm_emulate_invd(struct kvm_vcpu * vcpu)2103 int kvm_emulate_invd(struct kvm_vcpu *vcpu)
2104 {
2105 /* Treat an INVD instruction as a NOP and just skip it. */
2106 return kvm_emulate_as_nop(vcpu);
2107 }
2108 EXPORT_SYMBOL_GPL(kvm_emulate_invd);
2109
kvm_handle_invalid_op(struct kvm_vcpu * vcpu)2110 int kvm_handle_invalid_op(struct kvm_vcpu *vcpu)
2111 {
2112 kvm_queue_exception(vcpu, UD_VECTOR);
2113 return 1;
2114 }
2115 EXPORT_SYMBOL_GPL(kvm_handle_invalid_op);
2116
2117
kvm_emulate_monitor_mwait(struct kvm_vcpu * vcpu,const char * insn)2118 static int kvm_emulate_monitor_mwait(struct kvm_vcpu *vcpu, const char *insn)
2119 {
2120 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS) &&
2121 !guest_cpuid_has(vcpu, X86_FEATURE_MWAIT))
2122 return kvm_handle_invalid_op(vcpu);
2123
2124 pr_warn_once("%s instruction emulated as NOP!\n", insn);
2125 return kvm_emulate_as_nop(vcpu);
2126 }
kvm_emulate_mwait(struct kvm_vcpu * vcpu)2127 int kvm_emulate_mwait(struct kvm_vcpu *vcpu)
2128 {
2129 return kvm_emulate_monitor_mwait(vcpu, "MWAIT");
2130 }
2131 EXPORT_SYMBOL_GPL(kvm_emulate_mwait);
2132
kvm_emulate_monitor(struct kvm_vcpu * vcpu)2133 int kvm_emulate_monitor(struct kvm_vcpu *vcpu)
2134 {
2135 return kvm_emulate_monitor_mwait(vcpu, "MONITOR");
2136 }
2137 EXPORT_SYMBOL_GPL(kvm_emulate_monitor);
2138
kvm_vcpu_exit_request(struct kvm_vcpu * vcpu)2139 static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
2140 {
2141 xfer_to_guest_mode_prepare();
2142 return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
2143 xfer_to_guest_mode_work_pending();
2144 }
2145
2146 /*
2147 * The fast path for frequent and performance sensitive wrmsr emulation,
2148 * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces
2149 * the latency of virtual IPI by avoiding the expensive bits of transitioning
2150 * from guest to host, e.g. reacquiring KVM's SRCU lock. In contrast to the
2151 * other cases which must be called after interrupts are enabled on the host.
2152 */
handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu * vcpu,u64 data)2153 static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data)
2154 {
2155 if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic))
2156 return 1;
2157
2158 if (((data & APIC_SHORT_MASK) == APIC_DEST_NOSHORT) &&
2159 ((data & APIC_DEST_MASK) == APIC_DEST_PHYSICAL) &&
2160 ((data & APIC_MODE_MASK) == APIC_DM_FIXED) &&
2161 ((u32)(data >> 32) != X2APIC_BROADCAST))
2162 return kvm_x2apic_icr_write(vcpu->arch.apic, data);
2163
2164 return 1;
2165 }
2166
handle_fastpath_set_tscdeadline(struct kvm_vcpu * vcpu,u64 data)2167 static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data)
2168 {
2169 if (!kvm_can_use_hv_timer(vcpu))
2170 return 1;
2171
2172 kvm_set_lapic_tscdeadline_msr(vcpu, data);
2173 return 0;
2174 }
2175
handle_fastpath_set_msr_irqoff(struct kvm_vcpu * vcpu)2176 fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu)
2177 {
2178 u32 msr = kvm_rcx_read(vcpu);
2179 u64 data;
2180 fastpath_t ret = EXIT_FASTPATH_NONE;
2181
2182 kvm_vcpu_srcu_read_lock(vcpu);
2183
2184 switch (msr) {
2185 case APIC_BASE_MSR + (APIC_ICR >> 4):
2186 data = kvm_read_edx_eax(vcpu);
2187 if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) {
2188 kvm_skip_emulated_instruction(vcpu);
2189 ret = EXIT_FASTPATH_EXIT_HANDLED;
2190 }
2191 break;
2192 case MSR_IA32_TSC_DEADLINE:
2193 data = kvm_read_edx_eax(vcpu);
2194 if (!handle_fastpath_set_tscdeadline(vcpu, data)) {
2195 kvm_skip_emulated_instruction(vcpu);
2196 ret = EXIT_FASTPATH_REENTER_GUEST;
2197 }
2198 break;
2199 default:
2200 break;
2201 }
2202
2203 if (ret != EXIT_FASTPATH_NONE)
2204 trace_kvm_msr_write(msr, data);
2205
2206 kvm_vcpu_srcu_read_unlock(vcpu);
2207
2208 return ret;
2209 }
2210 EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff);
2211
2212 /*
2213 * Adapt set_msr() to msr_io()'s calling convention
2214 */
do_get_msr(struct kvm_vcpu * vcpu,unsigned index,u64 * data)2215 static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2216 {
2217 return kvm_get_msr_ignored_check(vcpu, index, data, true);
2218 }
2219
do_set_msr(struct kvm_vcpu * vcpu,unsigned index,u64 * data)2220 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
2221 {
2222 u64 val;
2223
2224 /*
2225 * Disallow writes to immutable feature MSRs after KVM_RUN. KVM does
2226 * not support modifying the guest vCPU model on the fly, e.g. changing
2227 * the nVMX capabilities while L2 is running is nonsensical. Allow
2228 * writes of the same value, e.g. to allow userspace to blindly stuff
2229 * all MSRs when emulating RESET.
2230 */
2231 if (kvm_vcpu_has_run(vcpu) && kvm_is_immutable_feature_msr(index) &&
2232 (do_get_msr(vcpu, index, &val) || *data != val))
2233 return -EINVAL;
2234
2235 return kvm_set_msr_ignored_check(vcpu, index, *data, true);
2236 }
2237
2238 #ifdef CONFIG_X86_64
2239 struct pvclock_clock {
2240 int vclock_mode;
2241 u64 cycle_last;
2242 u64 mask;
2243 u32 mult;
2244 u32 shift;
2245 u64 base_cycles;
2246 u64 offset;
2247 };
2248
2249 struct pvclock_gtod_data {
2250 seqcount_t seq;
2251
2252 struct pvclock_clock clock; /* extract of a clocksource struct */
2253 struct pvclock_clock raw_clock; /* extract of a clocksource struct */
2254
2255 ktime_t offs_boot;
2256 u64 wall_time_sec;
2257 };
2258
2259 static struct pvclock_gtod_data pvclock_gtod_data;
2260
update_pvclock_gtod(struct timekeeper * tk)2261 static void update_pvclock_gtod(struct timekeeper *tk)
2262 {
2263 struct pvclock_gtod_data *vdata = &pvclock_gtod_data;
2264
2265 write_seqcount_begin(&vdata->seq);
2266
2267 /* copy pvclock gtod data */
2268 vdata->clock.vclock_mode = tk->tkr_mono.clock->vdso_clock_mode;
2269 vdata->clock.cycle_last = tk->tkr_mono.cycle_last;
2270 vdata->clock.mask = tk->tkr_mono.mask;
2271 vdata->clock.mult = tk->tkr_mono.mult;
2272 vdata->clock.shift = tk->tkr_mono.shift;
2273 vdata->clock.base_cycles = tk->tkr_mono.xtime_nsec;
2274 vdata->clock.offset = tk->tkr_mono.base;
2275
2276 vdata->raw_clock.vclock_mode = tk->tkr_raw.clock->vdso_clock_mode;
2277 vdata->raw_clock.cycle_last = tk->tkr_raw.cycle_last;
2278 vdata->raw_clock.mask = tk->tkr_raw.mask;
2279 vdata->raw_clock.mult = tk->tkr_raw.mult;
2280 vdata->raw_clock.shift = tk->tkr_raw.shift;
2281 vdata->raw_clock.base_cycles = tk->tkr_raw.xtime_nsec;
2282 vdata->raw_clock.offset = tk->tkr_raw.base;
2283
2284 vdata->wall_time_sec = tk->xtime_sec;
2285
2286 vdata->offs_boot = tk->offs_boot;
2287
2288 write_seqcount_end(&vdata->seq);
2289 }
2290
get_kvmclock_base_ns(void)2291 static s64 get_kvmclock_base_ns(void)
2292 {
2293 /* Count up from boot time, but with the frequency of the raw clock. */
2294 return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot));
2295 }
2296 #else
get_kvmclock_base_ns(void)2297 static s64 get_kvmclock_base_ns(void)
2298 {
2299 /* Master clock not used, so we can just use CLOCK_BOOTTIME. */
2300 return ktime_get_boottime_ns();
2301 }
2302 #endif
2303
kvm_write_wall_clock(struct kvm * kvm,gpa_t wall_clock,int sec_hi_ofs)2304 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs)
2305 {
2306 int version;
2307 int r;
2308 struct pvclock_wall_clock wc;
2309 u32 wc_sec_hi;
2310 u64 wall_nsec;
2311
2312 if (!wall_clock)
2313 return;
2314
2315 r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
2316 if (r)
2317 return;
2318
2319 if (version & 1)
2320 ++version; /* first time write, random junk */
2321
2322 ++version;
2323
2324 if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version)))
2325 return;
2326
2327 wall_nsec = kvm_get_wall_clock_epoch(kvm);
2328
2329 wc.nsec = do_div(wall_nsec, NSEC_PER_SEC);
2330 wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */
2331 wc.version = version;
2332
2333 kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
2334
2335 if (sec_hi_ofs) {
2336 wc_sec_hi = wall_nsec >> 32;
2337 kvm_write_guest(kvm, wall_clock + sec_hi_ofs,
2338 &wc_sec_hi, sizeof(wc_sec_hi));
2339 }
2340
2341 version++;
2342 kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
2343 }
2344
kvm_write_system_time(struct kvm_vcpu * vcpu,gpa_t system_time,bool old_msr,bool host_initiated)2345 static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time,
2346 bool old_msr, bool host_initiated)
2347 {
2348 struct kvm_arch *ka = &vcpu->kvm->arch;
2349
2350 if (vcpu->vcpu_id == 0 && !host_initiated) {
2351 if (ka->boot_vcpu_runs_old_kvmclock != old_msr)
2352 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2353
2354 ka->boot_vcpu_runs_old_kvmclock = old_msr;
2355 }
2356
2357 vcpu->arch.time = system_time;
2358 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
2359
2360 /* we verify if the enable bit is set... */
2361 if (system_time & 1)
2362 kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL,
2363 sizeof(struct pvclock_vcpu_time_info));
2364 else
2365 kvm_gpc_deactivate(&vcpu->arch.pv_time);
2366
2367 return;
2368 }
2369
div_frac(uint32_t dividend,uint32_t divisor)2370 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
2371 {
2372 do_shl32_div32(dividend, divisor);
2373 return dividend;
2374 }
2375
kvm_get_time_scale(uint64_t scaled_hz,uint64_t base_hz,s8 * pshift,u32 * pmultiplier)2376 static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz,
2377 s8 *pshift, u32 *pmultiplier)
2378 {
2379 uint64_t scaled64;
2380 int32_t shift = 0;
2381 uint64_t tps64;
2382 uint32_t tps32;
2383
2384 tps64 = base_hz;
2385 scaled64 = scaled_hz;
2386 while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
2387 tps64 >>= 1;
2388 shift--;
2389 }
2390
2391 tps32 = (uint32_t)tps64;
2392 while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
2393 if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
2394 scaled64 >>= 1;
2395 else
2396 tps32 <<= 1;
2397 shift++;
2398 }
2399
2400 *pshift = shift;
2401 *pmultiplier = div_frac(scaled64, tps32);
2402 }
2403
2404 #ifdef CONFIG_X86_64
2405 static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
2406 #endif
2407
2408 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
2409 static unsigned long max_tsc_khz;
2410
adjust_tsc_khz(u32 khz,s32 ppm)2411 static u32 adjust_tsc_khz(u32 khz, s32 ppm)
2412 {
2413 u64 v = (u64)khz * (1000000 + ppm);
2414 do_div(v, 1000000);
2415 return v;
2416 }
2417
2418 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier);
2419
set_tsc_khz(struct kvm_vcpu * vcpu,u32 user_tsc_khz,bool scale)2420 static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale)
2421 {
2422 u64 ratio;
2423
2424 /* Guest TSC same frequency as host TSC? */
2425 if (!scale) {
2426 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
2427 return 0;
2428 }
2429
2430 /* TSC scaling supported? */
2431 if (!kvm_caps.has_tsc_control) {
2432 if (user_tsc_khz > tsc_khz) {
2433 vcpu->arch.tsc_catchup = 1;
2434 vcpu->arch.tsc_always_catchup = 1;
2435 return 0;
2436 } else {
2437 pr_warn_ratelimited("user requested TSC rate below hardware speed\n");
2438 return -1;
2439 }
2440 }
2441
2442 /* TSC scaling required - calculate ratio */
2443 ratio = mul_u64_u32_div(1ULL << kvm_caps.tsc_scaling_ratio_frac_bits,
2444 user_tsc_khz, tsc_khz);
2445
2446 if (ratio == 0 || ratio >= kvm_caps.max_tsc_scaling_ratio) {
2447 pr_warn_ratelimited("Invalid TSC scaling ratio - virtual-tsc-khz=%u\n",
2448 user_tsc_khz);
2449 return -1;
2450 }
2451
2452 kvm_vcpu_write_tsc_multiplier(vcpu, ratio);
2453 return 0;
2454 }
2455
kvm_set_tsc_khz(struct kvm_vcpu * vcpu,u32 user_tsc_khz)2456 static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz)
2457 {
2458 u32 thresh_lo, thresh_hi;
2459 int use_scaling = 0;
2460
2461 /* tsc_khz can be zero if TSC calibration fails */
2462 if (user_tsc_khz == 0) {
2463 /* set tsc_scaling_ratio to a safe value */
2464 kvm_vcpu_write_tsc_multiplier(vcpu, kvm_caps.default_tsc_scaling_ratio);
2465 return -1;
2466 }
2467
2468 /* Compute a scale to convert nanoseconds in TSC cycles */
2469 kvm_get_time_scale(user_tsc_khz * 1000LL, NSEC_PER_SEC,
2470 &vcpu->arch.virtual_tsc_shift,
2471 &vcpu->arch.virtual_tsc_mult);
2472 vcpu->arch.virtual_tsc_khz = user_tsc_khz;
2473
2474 /*
2475 * Compute the variation in TSC rate which is acceptable
2476 * within the range of tolerance and decide if the
2477 * rate being applied is within that bounds of the hardware
2478 * rate. If so, no scaling or compensation need be done.
2479 */
2480 thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
2481 thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
2482 if (user_tsc_khz < thresh_lo || user_tsc_khz > thresh_hi) {
2483 pr_debug("requested TSC rate %u falls outside tolerance [%u,%u]\n",
2484 user_tsc_khz, thresh_lo, thresh_hi);
2485 use_scaling = 1;
2486 }
2487 return set_tsc_khz(vcpu, user_tsc_khz, use_scaling);
2488 }
2489
compute_guest_tsc(struct kvm_vcpu * vcpu,s64 kernel_ns)2490 static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
2491 {
2492 u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
2493 vcpu->arch.virtual_tsc_mult,
2494 vcpu->arch.virtual_tsc_shift);
2495 tsc += vcpu->arch.this_tsc_write;
2496 return tsc;
2497 }
2498
2499 #ifdef CONFIG_X86_64
gtod_is_based_on_tsc(int mode)2500 static inline bool gtod_is_based_on_tsc(int mode)
2501 {
2502 return mode == VDSO_CLOCKMODE_TSC || mode == VDSO_CLOCKMODE_HVCLOCK;
2503 }
2504 #endif
2505
kvm_track_tsc_matching(struct kvm_vcpu * vcpu,bool new_generation)2506 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu, bool new_generation)
2507 {
2508 #ifdef CONFIG_X86_64
2509 struct kvm_arch *ka = &vcpu->kvm->arch;
2510 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2511
2512 /*
2513 * To use the masterclock, the host clocksource must be based on TSC
2514 * and all vCPUs must have matching TSCs. Note, the count for matching
2515 * vCPUs doesn't include the reference vCPU, hence "+1".
2516 */
2517 bool use_master_clock = (ka->nr_vcpus_matched_tsc + 1 ==
2518 atomic_read(&vcpu->kvm->online_vcpus)) &&
2519 gtod_is_based_on_tsc(gtod->clock.vclock_mode);
2520
2521 /*
2522 * Request a masterclock update if the masterclock needs to be toggled
2523 * on/off, or when starting a new generation and the masterclock is
2524 * enabled (compute_guest_tsc() requires the masterclock snapshot to be
2525 * taken _after_ the new generation is created).
2526 */
2527 if ((ka->use_master_clock && new_generation) ||
2528 (ka->use_master_clock != use_master_clock))
2529 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
2530
2531 trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
2532 atomic_read(&vcpu->kvm->online_vcpus),
2533 ka->use_master_clock, gtod->clock.vclock_mode);
2534 #endif
2535 }
2536
2537 /*
2538 * Multiply tsc by a fixed point number represented by ratio.
2539 *
2540 * The most significant 64-N bits (mult) of ratio represent the
2541 * integral part of the fixed point number; the remaining N bits
2542 * (frac) represent the fractional part, ie. ratio represents a fixed
2543 * point number (mult + frac * 2^(-N)).
2544 *
2545 * N equals to kvm_caps.tsc_scaling_ratio_frac_bits.
2546 */
__scale_tsc(u64 ratio,u64 tsc)2547 static inline u64 __scale_tsc(u64 ratio, u64 tsc)
2548 {
2549 return mul_u64_u64_shr(tsc, ratio, kvm_caps.tsc_scaling_ratio_frac_bits);
2550 }
2551
kvm_scale_tsc(u64 tsc,u64 ratio)2552 u64 kvm_scale_tsc(u64 tsc, u64 ratio)
2553 {
2554 u64 _tsc = tsc;
2555
2556 if (ratio != kvm_caps.default_tsc_scaling_ratio)
2557 _tsc = __scale_tsc(ratio, tsc);
2558
2559 return _tsc;
2560 }
2561
kvm_compute_l1_tsc_offset(struct kvm_vcpu * vcpu,u64 target_tsc)2562 static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc)
2563 {
2564 u64 tsc;
2565
2566 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio);
2567
2568 return target_tsc - tsc;
2569 }
2570
kvm_read_l1_tsc(struct kvm_vcpu * vcpu,u64 host_tsc)2571 u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
2572 {
2573 return vcpu->arch.l1_tsc_offset +
2574 kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio);
2575 }
2576 EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);
2577
kvm_calc_nested_tsc_offset(u64 l1_offset,u64 l2_offset,u64 l2_multiplier)2578 u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier)
2579 {
2580 u64 nested_offset;
2581
2582 if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio)
2583 nested_offset = l1_offset;
2584 else
2585 nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier,
2586 kvm_caps.tsc_scaling_ratio_frac_bits);
2587
2588 nested_offset += l2_offset;
2589 return nested_offset;
2590 }
2591 EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_offset);
2592
kvm_calc_nested_tsc_multiplier(u64 l1_multiplier,u64 l2_multiplier)2593 u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier)
2594 {
2595 if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio)
2596 return mul_u64_u64_shr(l1_multiplier, l2_multiplier,
2597 kvm_caps.tsc_scaling_ratio_frac_bits);
2598
2599 return l1_multiplier;
2600 }
2601 EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_multiplier);
2602
kvm_vcpu_write_tsc_offset(struct kvm_vcpu * vcpu,u64 l1_offset)2603 static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset)
2604 {
2605 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
2606 vcpu->arch.l1_tsc_offset,
2607 l1_offset);
2608
2609 vcpu->arch.l1_tsc_offset = l1_offset;
2610
2611 /*
2612 * If we are here because L1 chose not to trap WRMSR to TSC then
2613 * according to the spec this should set L1's TSC (as opposed to
2614 * setting L1's offset for L2).
2615 */
2616 if (is_guest_mode(vcpu))
2617 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
2618 l1_offset,
2619 static_call(kvm_x86_get_l2_tsc_offset)(vcpu),
2620 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2621 else
2622 vcpu->arch.tsc_offset = l1_offset;
2623
2624 static_call(kvm_x86_write_tsc_offset)(vcpu);
2625 }
2626
kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu * vcpu,u64 l1_multiplier)2627 static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier)
2628 {
2629 vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier;
2630
2631 /* Userspace is changing the multiplier while L2 is active */
2632 if (is_guest_mode(vcpu))
2633 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier(
2634 l1_multiplier,
2635 static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu));
2636 else
2637 vcpu->arch.tsc_scaling_ratio = l1_multiplier;
2638
2639 if (kvm_caps.has_tsc_control)
2640 static_call(kvm_x86_write_tsc_multiplier)(vcpu);
2641 }
2642
kvm_check_tsc_unstable(void)2643 static inline bool kvm_check_tsc_unstable(void)
2644 {
2645 #ifdef CONFIG_X86_64
2646 /*
2647 * TSC is marked unstable when we're running on Hyper-V,
2648 * 'TSC page' clocksource is good.
2649 */
2650 if (pvclock_gtod_data.clock.vclock_mode == VDSO_CLOCKMODE_HVCLOCK)
2651 return false;
2652 #endif
2653 return check_tsc_unstable();
2654 }
2655
2656 /*
2657 * Infers attempts to synchronize the guest's tsc from host writes. Sets the
2658 * offset for the vcpu and tracks the TSC matching generation that the vcpu
2659 * participates in.
2660 */
__kvm_synchronize_tsc(struct kvm_vcpu * vcpu,u64 offset,u64 tsc,u64 ns,bool matched)2661 static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc,
2662 u64 ns, bool matched)
2663 {
2664 struct kvm *kvm = vcpu->kvm;
2665
2666 lockdep_assert_held(&kvm->arch.tsc_write_lock);
2667
2668 /*
2669 * We also track th most recent recorded KHZ, write and time to
2670 * allow the matching interval to be extended at each write.
2671 */
2672 kvm->arch.last_tsc_nsec = ns;
2673 kvm->arch.last_tsc_write = tsc;
2674 kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;
2675 kvm->arch.last_tsc_offset = offset;
2676
2677 vcpu->arch.last_guest_tsc = tsc;
2678
2679 kvm_vcpu_write_tsc_offset(vcpu, offset);
2680
2681 if (!matched) {
2682 /*
2683 * We split periods of matched TSC writes into generations.
2684 * For each generation, we track the original measured
2685 * nanosecond time, offset, and write, so if TSCs are in
2686 * sync, we can match exact offset, and if not, we can match
2687 * exact software computation in compute_guest_tsc()
2688 *
2689 * These values are tracked in kvm->arch.cur_xxx variables.
2690 */
2691 kvm->arch.cur_tsc_generation++;
2692 kvm->arch.cur_tsc_nsec = ns;
2693 kvm->arch.cur_tsc_write = tsc;
2694 kvm->arch.cur_tsc_offset = offset;
2695 kvm->arch.nr_vcpus_matched_tsc = 0;
2696 } else if (vcpu->arch.this_tsc_generation != kvm->arch.cur_tsc_generation) {
2697 kvm->arch.nr_vcpus_matched_tsc++;
2698 }
2699
2700 /* Keep track of which generation this VCPU has synchronized to */
2701 vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
2702 vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
2703 vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;
2704
2705 kvm_track_tsc_matching(vcpu, !matched);
2706 }
2707
kvm_synchronize_tsc(struct kvm_vcpu * vcpu,u64 * user_value)2708 static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value)
2709 {
2710 u64 data = user_value ? *user_value : 0;
2711 struct kvm *kvm = vcpu->kvm;
2712 u64 offset, ns, elapsed;
2713 unsigned long flags;
2714 bool matched = false;
2715 bool synchronizing = false;
2716
2717 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
2718 offset = kvm_compute_l1_tsc_offset(vcpu, data);
2719 ns = get_kvmclock_base_ns();
2720 elapsed = ns - kvm->arch.last_tsc_nsec;
2721
2722 if (vcpu->arch.virtual_tsc_khz) {
2723 if (data == 0) {
2724 /*
2725 * Force synchronization when creating a vCPU, or when
2726 * userspace explicitly writes a zero value.
2727 */
2728 synchronizing = true;
2729 } else if (kvm->arch.user_set_tsc) {
2730 u64 tsc_exp = kvm->arch.last_tsc_write +
2731 nsec_to_cycles(vcpu, elapsed);
2732 u64 tsc_hz = vcpu->arch.virtual_tsc_khz * 1000LL;
2733 /*
2734 * Here lies UAPI baggage: when a user-initiated TSC write has
2735 * a small delta (1 second) of virtual cycle time against the
2736 * previously set vCPU, we assume that they were intended to be
2737 * in sync and the delta was only due to the racy nature of the
2738 * legacy API.
2739 *
2740 * This trick falls down when restoring a guest which genuinely
2741 * has been running for less time than the 1 second of imprecision
2742 * which we allow for in the legacy API. In this case, the first
2743 * value written by userspace (on any vCPU) should not be subject
2744 * to this 'correction' to make it sync up with values that only
2745 * come from the kernel's default vCPU creation. Make the 1-second
2746 * slop hack only trigger if the user_set_tsc flag is already set.
2747 */
2748 synchronizing = data < tsc_exp + tsc_hz &&
2749 data + tsc_hz > tsc_exp;
2750 }
2751 }
2752
2753 if (user_value)
2754 kvm->arch.user_set_tsc = true;
2755
2756 /*
2757 * For a reliable TSC, we can match TSC offsets, and for an unstable
2758 * TSC, we add elapsed time in this computation. We could let the
2759 * compensation code attempt to catch up if we fall behind, but
2760 * it's better to try to match offsets from the beginning.
2761 */
2762 if (synchronizing &&
2763 vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
2764 if (!kvm_check_tsc_unstable()) {
2765 offset = kvm->arch.cur_tsc_offset;
2766 } else {
2767 u64 delta = nsec_to_cycles(vcpu, elapsed);
2768 data += delta;
2769 offset = kvm_compute_l1_tsc_offset(vcpu, data);
2770 }
2771 matched = true;
2772 }
2773
2774 __kvm_synchronize_tsc(vcpu, offset, data, ns, matched);
2775 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
2776 }
2777
adjust_tsc_offset_guest(struct kvm_vcpu * vcpu,s64 adjustment)2778 static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu,
2779 s64 adjustment)
2780 {
2781 u64 tsc_offset = vcpu->arch.l1_tsc_offset;
2782 kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment);
2783 }
2784
adjust_tsc_offset_host(struct kvm_vcpu * vcpu,s64 adjustment)2785 static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment)
2786 {
2787 if (vcpu->arch.l1_tsc_scaling_ratio != kvm_caps.default_tsc_scaling_ratio)
2788 WARN_ON(adjustment < 0);
2789 adjustment = kvm_scale_tsc((u64) adjustment,
2790 vcpu->arch.l1_tsc_scaling_ratio);
2791 adjust_tsc_offset_guest(vcpu, adjustment);
2792 }
2793
2794 #ifdef CONFIG_X86_64
2795
read_tsc(void)2796 static u64 read_tsc(void)
2797 {
2798 u64 ret = (u64)rdtsc_ordered();
2799 u64 last = pvclock_gtod_data.clock.cycle_last;
2800
2801 if (likely(ret >= last))
2802 return ret;
2803
2804 /*
2805 * GCC likes to generate cmov here, but this branch is extremely
2806 * predictable (it's just a function of time and the likely is
2807 * very likely) and there's a data dependence, so force GCC
2808 * to generate a branch instead. I don't barrier() because
2809 * we don't actually need a barrier, and if this function
2810 * ever gets inlined it will generate worse code.
2811 */
2812 asm volatile ("");
2813 return last;
2814 }
2815
vgettsc(struct pvclock_clock * clock,u64 * tsc_timestamp,int * mode)2816 static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp,
2817 int *mode)
2818 {
2819 u64 tsc_pg_val;
2820 long v;
2821
2822 switch (clock->vclock_mode) {
2823 case VDSO_CLOCKMODE_HVCLOCK:
2824 if (hv_read_tsc_page_tsc(hv_get_tsc_page(),
2825 tsc_timestamp, &tsc_pg_val)) {
2826 /* TSC page valid */
2827 *mode = VDSO_CLOCKMODE_HVCLOCK;
2828 v = (tsc_pg_val - clock->cycle_last) &
2829 clock->mask;
2830 } else {
2831 /* TSC page invalid */
2832 *mode = VDSO_CLOCKMODE_NONE;
2833 }
2834 break;
2835 case VDSO_CLOCKMODE_TSC:
2836 *mode = VDSO_CLOCKMODE_TSC;
2837 *tsc_timestamp = read_tsc();
2838 v = (*tsc_timestamp - clock->cycle_last) &
2839 clock->mask;
2840 break;
2841 default:
2842 *mode = VDSO_CLOCKMODE_NONE;
2843 }
2844
2845 if (*mode == VDSO_CLOCKMODE_NONE)
2846 *tsc_timestamp = v = 0;
2847
2848 return v * clock->mult;
2849 }
2850
2851 /*
2852 * As with get_kvmclock_base_ns(), this counts from boot time, at the
2853 * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot).
2854 */
do_kvmclock_base(s64 * t,u64 * tsc_timestamp)2855 static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp)
2856 {
2857 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2858 unsigned long seq;
2859 int mode;
2860 u64 ns;
2861
2862 do {
2863 seq = read_seqcount_begin(>od->seq);
2864 ns = gtod->raw_clock.base_cycles;
2865 ns += vgettsc(>od->raw_clock, tsc_timestamp, &mode);
2866 ns >>= gtod->raw_clock.shift;
2867 ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot));
2868 } while (unlikely(read_seqcount_retry(>od->seq, seq)));
2869 *t = ns;
2870
2871 return mode;
2872 }
2873
2874 /*
2875 * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with
2876 * no boot time offset.
2877 */
do_monotonic(s64 * t,u64 * tsc_timestamp)2878 static int do_monotonic(s64 *t, u64 *tsc_timestamp)
2879 {
2880 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2881 unsigned long seq;
2882 int mode;
2883 u64 ns;
2884
2885 do {
2886 seq = read_seqcount_begin(>od->seq);
2887 ns = gtod->clock.base_cycles;
2888 ns += vgettsc(>od->clock, tsc_timestamp, &mode);
2889 ns >>= gtod->clock.shift;
2890 ns += ktime_to_ns(gtod->clock.offset);
2891 } while (unlikely(read_seqcount_retry(>od->seq, seq)));
2892 *t = ns;
2893
2894 return mode;
2895 }
2896
do_realtime(struct timespec64 * ts,u64 * tsc_timestamp)2897 static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp)
2898 {
2899 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
2900 unsigned long seq;
2901 int mode;
2902 u64 ns;
2903
2904 do {
2905 seq = read_seqcount_begin(>od->seq);
2906 ts->tv_sec = gtod->wall_time_sec;
2907 ns = gtod->clock.base_cycles;
2908 ns += vgettsc(>od->clock, tsc_timestamp, &mode);
2909 ns >>= gtod->clock.shift;
2910 } while (unlikely(read_seqcount_retry(>od->seq, seq)));
2911
2912 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
2913 ts->tv_nsec = ns;
2914
2915 return mode;
2916 }
2917
2918 /*
2919 * Calculates the kvmclock_base_ns (CLOCK_MONOTONIC_RAW + boot time) and
2920 * reports the TSC value from which it do so. Returns true if host is
2921 * using TSC based clocksource.
2922 */
kvm_get_time_and_clockread(s64 * kernel_ns,u64 * tsc_timestamp)2923 static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2924 {
2925 /* checked again under seqlock below */
2926 if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2927 return false;
2928
2929 return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns,
2930 tsc_timestamp));
2931 }
2932
2933 /*
2934 * Calculates CLOCK_MONOTONIC and reports the TSC value from which it did
2935 * so. Returns true if host is using TSC based clocksource.
2936 */
kvm_get_monotonic_and_clockread(s64 * kernel_ns,u64 * tsc_timestamp)2937 bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
2938 {
2939 /* checked again under seqlock below */
2940 if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2941 return false;
2942
2943 return gtod_is_based_on_tsc(do_monotonic(kernel_ns,
2944 tsc_timestamp));
2945 }
2946
2947 /*
2948 * Calculates CLOCK_REALTIME and reports the TSC value from which it did
2949 * so. Returns true if host is using TSC based clocksource.
2950 *
2951 * DO NOT USE this for anything related to migration. You want CLOCK_TAI
2952 * for that.
2953 */
kvm_get_walltime_and_clockread(struct timespec64 * ts,u64 * tsc_timestamp)2954 static bool kvm_get_walltime_and_clockread(struct timespec64 *ts,
2955 u64 *tsc_timestamp)
2956 {
2957 /* checked again under seqlock below */
2958 if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
2959 return false;
2960
2961 return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
2962 }
2963 #endif
2964
2965 /*
2966 *
2967 * Assuming a stable TSC across physical CPUS, and a stable TSC
2968 * across virtual CPUs, the following condition is possible.
2969 * Each numbered line represents an event visible to both
2970 * CPUs at the next numbered event.
2971 *
2972 * "timespecX" represents host monotonic time. "tscX" represents
2973 * RDTSC value.
2974 *
2975 * VCPU0 on CPU0 | VCPU1 on CPU1
2976 *
2977 * 1. read timespec0,tsc0
2978 * 2. | timespec1 = timespec0 + N
2979 * | tsc1 = tsc0 + M
2980 * 3. transition to guest | transition to guest
2981 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
2982 * 5. | ret1 = timespec1 + (rdtsc - tsc1)
2983 * | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
2984 *
2985 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
2986 *
2987 * - ret0 < ret1
2988 * - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
2989 * ...
2990 * - 0 < N - M => M < N
2991 *
2992 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
2993 * always the case (the difference between two distinct xtime instances
2994 * might be smaller then the difference between corresponding TSC reads,
2995 * when updating guest vcpus pvclock areas).
2996 *
2997 * To avoid that problem, do not allow visibility of distinct
2998 * system_timestamp/tsc_timestamp values simultaneously: use a master
2999 * copy of host monotonic time values. Update that master copy
3000 * in lockstep.
3001 *
3002 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
3003 *
3004 */
3005
pvclock_update_vm_gtod_copy(struct kvm * kvm)3006 static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
3007 {
3008 #ifdef CONFIG_X86_64
3009 struct kvm_arch *ka = &kvm->arch;
3010 int vclock_mode;
3011 bool host_tsc_clocksource, vcpus_matched;
3012
3013 lockdep_assert_held(&kvm->arch.tsc_write_lock);
3014 vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
3015 atomic_read(&kvm->online_vcpus));
3016
3017 /*
3018 * If the host uses TSC clock, then passthrough TSC as stable
3019 * to the guest.
3020 */
3021 host_tsc_clocksource = kvm_get_time_and_clockread(
3022 &ka->master_kernel_ns,
3023 &ka->master_cycle_now);
3024
3025 ka->use_master_clock = host_tsc_clocksource && vcpus_matched
3026 && !ka->backwards_tsc_observed
3027 && !ka->boot_vcpu_runs_old_kvmclock;
3028
3029 if (ka->use_master_clock)
3030 atomic_set(&kvm_guest_has_master_clock, 1);
3031
3032 vclock_mode = pvclock_gtod_data.clock.vclock_mode;
3033 trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
3034 vcpus_matched);
3035 #endif
3036 }
3037
kvm_make_mclock_inprogress_request(struct kvm * kvm)3038 static void kvm_make_mclock_inprogress_request(struct kvm *kvm)
3039 {
3040 kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
3041 }
3042
__kvm_start_pvclock_update(struct kvm * kvm)3043 static void __kvm_start_pvclock_update(struct kvm *kvm)
3044 {
3045 raw_spin_lock_irq(&kvm->arch.tsc_write_lock);
3046 write_seqcount_begin(&kvm->arch.pvclock_sc);
3047 }
3048
kvm_start_pvclock_update(struct kvm * kvm)3049 static void kvm_start_pvclock_update(struct kvm *kvm)
3050 {
3051 kvm_make_mclock_inprogress_request(kvm);
3052
3053 /* no guest entries from this point */
3054 __kvm_start_pvclock_update(kvm);
3055 }
3056
kvm_end_pvclock_update(struct kvm * kvm)3057 static void kvm_end_pvclock_update(struct kvm *kvm)
3058 {
3059 struct kvm_arch *ka = &kvm->arch;
3060 struct kvm_vcpu *vcpu;
3061 unsigned long i;
3062
3063 write_seqcount_end(&ka->pvclock_sc);
3064 raw_spin_unlock_irq(&ka->tsc_write_lock);
3065 kvm_for_each_vcpu(i, vcpu, kvm)
3066 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3067
3068 /* guest entries allowed */
3069 kvm_for_each_vcpu(i, vcpu, kvm)
3070 kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
3071 }
3072
kvm_update_masterclock(struct kvm * kvm)3073 static void kvm_update_masterclock(struct kvm *kvm)
3074 {
3075 kvm_hv_request_tsc_page_update(kvm);
3076 kvm_start_pvclock_update(kvm);
3077 pvclock_update_vm_gtod_copy(kvm);
3078 kvm_end_pvclock_update(kvm);
3079 }
3080
3081 /*
3082 * Use the kernel's tsc_khz directly if the TSC is constant, otherwise use KVM's
3083 * per-CPU value (which may be zero if a CPU is going offline). Note, tsc_khz
3084 * can change during boot even if the TSC is constant, as it's possible for KVM
3085 * to be loaded before TSC calibration completes. Ideally, KVM would get a
3086 * notification when calibration completes, but practically speaking calibration
3087 * will complete before userspace is alive enough to create VMs.
3088 */
get_cpu_tsc_khz(void)3089 static unsigned long get_cpu_tsc_khz(void)
3090 {
3091 if (static_cpu_has(X86_FEATURE_CONSTANT_TSC))
3092 return tsc_khz;
3093 else
3094 return __this_cpu_read(cpu_tsc_khz);
3095 }
3096
3097 /* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */
__get_kvmclock(struct kvm * kvm,struct kvm_clock_data * data)3098 static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
3099 {
3100 struct kvm_arch *ka = &kvm->arch;
3101 struct pvclock_vcpu_time_info hv_clock;
3102
3103 /* both __this_cpu_read() and rdtsc() should be on the same cpu */
3104 get_cpu();
3105
3106 data->flags = 0;
3107 if (ka->use_master_clock &&
3108 (static_cpu_has(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) {
3109 #ifdef CONFIG_X86_64
3110 struct timespec64 ts;
3111
3112 if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) {
3113 data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec;
3114 data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC;
3115 } else
3116 #endif
3117 data->host_tsc = rdtsc();
3118
3119 data->flags |= KVM_CLOCK_TSC_STABLE;
3120 hv_clock.tsc_timestamp = ka->master_cycle_now;
3121 hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
3122 kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL,
3123 &hv_clock.tsc_shift,
3124 &hv_clock.tsc_to_system_mul);
3125 data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc);
3126 } else {
3127 data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset;
3128 }
3129
3130 put_cpu();
3131 }
3132
get_kvmclock(struct kvm * kvm,struct kvm_clock_data * data)3133 static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data)
3134 {
3135 struct kvm_arch *ka = &kvm->arch;
3136 unsigned seq;
3137
3138 do {
3139 seq = read_seqcount_begin(&ka->pvclock_sc);
3140 __get_kvmclock(kvm, data);
3141 } while (read_seqcount_retry(&ka->pvclock_sc, seq));
3142 }
3143
get_kvmclock_ns(struct kvm * kvm)3144 u64 get_kvmclock_ns(struct kvm *kvm)
3145 {
3146 struct kvm_clock_data data;
3147
3148 get_kvmclock(kvm, &data);
3149 return data.clock;
3150 }
3151
kvm_setup_guest_pvclock(struct kvm_vcpu * v,struct gfn_to_pfn_cache * gpc,unsigned int offset,bool force_tsc_unstable)3152 static void kvm_setup_guest_pvclock(struct kvm_vcpu *v,
3153 struct gfn_to_pfn_cache *gpc,
3154 unsigned int offset,
3155 bool force_tsc_unstable)
3156 {
3157 struct kvm_vcpu_arch *vcpu = &v->arch;
3158 struct pvclock_vcpu_time_info *guest_hv_clock;
3159 unsigned long flags;
3160
3161 read_lock_irqsave(&gpc->lock, flags);
3162 while (!kvm_gpc_check(gpc, offset + sizeof(*guest_hv_clock))) {
3163 read_unlock_irqrestore(&gpc->lock, flags);
3164
3165 if (kvm_gpc_refresh(gpc, offset + sizeof(*guest_hv_clock)))
3166 return;
3167
3168 read_lock_irqsave(&gpc->lock, flags);
3169 }
3170
3171 guest_hv_clock = (void *)(gpc->khva + offset);
3172
3173 /*
3174 * This VCPU is paused, but it's legal for a guest to read another
3175 * VCPU's kvmclock, so we really have to follow the specification where
3176 * it says that version is odd if data is being modified, and even after
3177 * it is consistent.
3178 */
3179
3180 guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1;
3181 smp_wmb();
3182
3183 /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
3184 vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED);
3185
3186 if (vcpu->pvclock_set_guest_stopped_request) {
3187 vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED;
3188 vcpu->pvclock_set_guest_stopped_request = false;
3189 }
3190
3191 memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock));
3192
3193 if (force_tsc_unstable)
3194 guest_hv_clock->flags &= ~PVCLOCK_TSC_STABLE_BIT;
3195
3196 smp_wmb();
3197
3198 guest_hv_clock->version = ++vcpu->hv_clock.version;
3199
3200 kvm_gpc_mark_dirty_in_slot(gpc);
3201 read_unlock_irqrestore(&gpc->lock, flags);
3202
3203 trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock);
3204 }
3205
kvm_guest_time_update(struct kvm_vcpu * v)3206 static int kvm_guest_time_update(struct kvm_vcpu *v)
3207 {
3208 unsigned long flags, tgt_tsc_khz;
3209 unsigned seq;
3210 struct kvm_vcpu_arch *vcpu = &v->arch;
3211 struct kvm_arch *ka = &v->kvm->arch;
3212 s64 kernel_ns;
3213 u64 tsc_timestamp, host_tsc;
3214 u8 pvclock_flags;
3215 bool use_master_clock;
3216 #ifdef CONFIG_KVM_XEN
3217 /*
3218 * For Xen guests we may need to override PVCLOCK_TSC_STABLE_BIT as unless
3219 * explicitly told to use TSC as its clocksource Xen will not set this bit.
3220 * This default behaviour led to bugs in some guest kernels which cause
3221 * problems if they observe PVCLOCK_TSC_STABLE_BIT in the pvclock flags.
3222 */
3223 bool xen_pvclock_tsc_unstable =
3224 ka->xen_hvm_config.flags & KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE;
3225 #endif
3226
3227 kernel_ns = 0;
3228 host_tsc = 0;
3229
3230 /*
3231 * If the host uses TSC clock, then passthrough TSC as stable
3232 * to the guest.
3233 */
3234 do {
3235 seq = read_seqcount_begin(&ka->pvclock_sc);
3236 use_master_clock = ka->use_master_clock;
3237 if (use_master_clock) {
3238 host_tsc = ka->master_cycle_now;
3239 kernel_ns = ka->master_kernel_ns;
3240 }
3241 } while (read_seqcount_retry(&ka->pvclock_sc, seq));
3242
3243 /* Keep irq disabled to prevent changes to the clock */
3244 local_irq_save(flags);
3245 tgt_tsc_khz = get_cpu_tsc_khz();
3246 if (unlikely(tgt_tsc_khz == 0)) {
3247 local_irq_restore(flags);
3248 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
3249 return 1;
3250 }
3251 if (!use_master_clock) {
3252 host_tsc = rdtsc();
3253 kernel_ns = get_kvmclock_base_ns();
3254 }
3255
3256 tsc_timestamp = kvm_read_l1_tsc(v, host_tsc);
3257
3258 /*
3259 * We may have to catch up the TSC to match elapsed wall clock
3260 * time for two reasons, even if kvmclock is used.
3261 * 1) CPU could have been running below the maximum TSC rate
3262 * 2) Broken TSC compensation resets the base at each VCPU
3263 * entry to avoid unknown leaps of TSC even when running
3264 * again on the same CPU. This may cause apparent elapsed
3265 * time to disappear, and the guest to stand still or run
3266 * very slowly.
3267 */
3268 if (vcpu->tsc_catchup) {
3269 u64 tsc = compute_guest_tsc(v, kernel_ns);
3270 if (tsc > tsc_timestamp) {
3271 adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
3272 tsc_timestamp = tsc;
3273 }
3274 }
3275
3276 local_irq_restore(flags);
3277
3278 /* With all the info we got, fill in the values */
3279
3280 if (kvm_caps.has_tsc_control)
3281 tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz,
3282 v->arch.l1_tsc_scaling_ratio);
3283
3284 if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) {
3285 kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL,
3286 &vcpu->hv_clock.tsc_shift,
3287 &vcpu->hv_clock.tsc_to_system_mul);
3288 vcpu->hw_tsc_khz = tgt_tsc_khz;
3289 kvm_xen_update_tsc_info(v);
3290 }
3291
3292 vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
3293 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
3294 vcpu->last_guest_tsc = tsc_timestamp;
3295
3296 /* If the host uses TSC clocksource, then it is stable */
3297 pvclock_flags = 0;
3298 if (use_master_clock)
3299 pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;
3300
3301 vcpu->hv_clock.flags = pvclock_flags;
3302
3303 if (vcpu->pv_time.active)
3304 kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0, false);
3305 #ifdef CONFIG_KVM_XEN
3306 if (vcpu->xen.vcpu_info_cache.active)
3307 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache,
3308 offsetof(struct compat_vcpu_info, time),
3309 xen_pvclock_tsc_unstable);
3310 if (vcpu->xen.vcpu_time_info_cache.active)
3311 kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0,
3312 xen_pvclock_tsc_unstable);
3313 #endif
3314 kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock);
3315 return 0;
3316 }
3317
3318 /*
3319 * The pvclock_wall_clock ABI tells the guest the wall clock time at
3320 * which it started (i.e. its epoch, when its kvmclock was zero).
3321 *
3322 * In fact those clocks are subtly different; wall clock frequency is
3323 * adjusted by NTP and has leap seconds, while the kvmclock is a
3324 * simple function of the TSC without any such adjustment.
3325 *
3326 * Perhaps the ABI should have exposed CLOCK_TAI and a ratio between
3327 * that and kvmclock, but even that would be subject to change over
3328 * time.
3329 *
3330 * Attempt to calculate the epoch at a given moment using the *same*
3331 * TSC reading via kvm_get_walltime_and_clockread() to obtain both
3332 * wallclock and kvmclock times, and subtracting one from the other.
3333 *
3334 * Fall back to using their values at slightly different moments by
3335 * calling ktime_get_real_ns() and get_kvmclock_ns() separately.
3336 */
kvm_get_wall_clock_epoch(struct kvm * kvm)3337 uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm)
3338 {
3339 #ifdef CONFIG_X86_64
3340 struct pvclock_vcpu_time_info hv_clock;
3341 struct kvm_arch *ka = &kvm->arch;
3342 unsigned long seq, local_tsc_khz;
3343 struct timespec64 ts;
3344 uint64_t host_tsc;
3345
3346 do {
3347 seq = read_seqcount_begin(&ka->pvclock_sc);
3348
3349 local_tsc_khz = 0;
3350 if (!ka->use_master_clock)
3351 break;
3352
3353 /*
3354 * The TSC read and the call to get_cpu_tsc_khz() must happen
3355 * on the same CPU.
3356 */
3357 get_cpu();
3358
3359 local_tsc_khz = get_cpu_tsc_khz();
3360
3361 if (local_tsc_khz &&
3362 !kvm_get_walltime_and_clockread(&ts, &host_tsc))
3363 local_tsc_khz = 0; /* Fall back to old method */
3364
3365 put_cpu();
3366
3367 /*
3368 * These values must be snapshotted within the seqcount loop.
3369 * After that, it's just mathematics which can happen on any
3370 * CPU at any time.
3371 */
3372 hv_clock.tsc_timestamp = ka->master_cycle_now;
3373 hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset;
3374
3375 } while (read_seqcount_retry(&ka->pvclock_sc, seq));
3376
3377 /*
3378 * If the conditions were right, and obtaining the wallclock+TSC was
3379 * successful, calculate the KVM clock at the corresponding time and
3380 * subtract one from the other to get the guest's epoch in nanoseconds
3381 * since 1970-01-01.
3382 */
3383 if (local_tsc_khz) {
3384 kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC,
3385 &hv_clock.tsc_shift,
3386 &hv_clock.tsc_to_system_mul);
3387 return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec -
3388 __pvclock_read_cycles(&hv_clock, host_tsc);
3389 }
3390 #endif
3391 return ktime_get_real_ns() - get_kvmclock_ns(kvm);
3392 }
3393
3394 /*
3395 * kvmclock updates which are isolated to a given vcpu, such as
3396 * vcpu->cpu migration, should not allow system_timestamp from
3397 * the rest of the vcpus to remain static. Otherwise ntp frequency
3398 * correction applies to one vcpu's system_timestamp but not
3399 * the others.
3400 *
3401 * So in those cases, request a kvmclock update for all vcpus.
3402 * We need to rate-limit these requests though, as they can
3403 * considerably slow guests that have a large number of vcpus.
3404 * The time for a remote vcpu to update its kvmclock is bound
3405 * by the delay we use to rate-limit the updates.
3406 */
3407
3408 #define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)
3409
kvmclock_update_fn(struct work_struct * work)3410 static void kvmclock_update_fn(struct work_struct *work)
3411 {
3412 unsigned long i;
3413 struct delayed_work *dwork = to_delayed_work(work);
3414 struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3415 kvmclock_update_work);
3416 struct kvm *kvm = container_of(ka, struct kvm, arch);
3417 struct kvm_vcpu *vcpu;
3418
3419 kvm_for_each_vcpu(i, vcpu, kvm) {
3420 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3421 kvm_vcpu_kick(vcpu);
3422 }
3423 }
3424
kvm_gen_kvmclock_update(struct kvm_vcpu * v)3425 static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
3426 {
3427 struct kvm *kvm = v->kvm;
3428
3429 kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
3430 schedule_delayed_work(&kvm->arch.kvmclock_update_work,
3431 KVMCLOCK_UPDATE_DELAY);
3432 }
3433
3434 #define KVMCLOCK_SYNC_PERIOD (300 * HZ)
3435
kvmclock_sync_fn(struct work_struct * work)3436 static void kvmclock_sync_fn(struct work_struct *work)
3437 {
3438 struct delayed_work *dwork = to_delayed_work(work);
3439 struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
3440 kvmclock_sync_work);
3441 struct kvm *kvm = container_of(ka, struct kvm, arch);
3442
3443 schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
3444 schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
3445 KVMCLOCK_SYNC_PERIOD);
3446 }
3447
3448 /* These helpers are safe iff @msr is known to be an MCx bank MSR. */
is_mci_control_msr(u32 msr)3449 static bool is_mci_control_msr(u32 msr)
3450 {
3451 return (msr & 3) == 0;
3452 }
is_mci_status_msr(u32 msr)3453 static bool is_mci_status_msr(u32 msr)
3454 {
3455 return (msr & 3) == 1;
3456 }
3457
3458 /*
3459 * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP.
3460 */
can_set_mci_status(struct kvm_vcpu * vcpu)3461 static bool can_set_mci_status(struct kvm_vcpu *vcpu)
3462 {
3463 /* McStatusWrEn enabled? */
3464 if (guest_cpuid_is_amd_compatible(vcpu))
3465 return !!(vcpu->arch.msr_hwcr & BIT_ULL(18));
3466
3467 return false;
3468 }
3469
set_msr_mce(struct kvm_vcpu * vcpu,struct msr_data * msr_info)3470 static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3471 {
3472 u64 mcg_cap = vcpu->arch.mcg_cap;
3473 unsigned bank_num = mcg_cap & 0xff;
3474 u32 msr = msr_info->index;
3475 u64 data = msr_info->data;
3476 u32 offset, last_msr;
3477
3478 switch (msr) {
3479 case MSR_IA32_MCG_STATUS:
3480 vcpu->arch.mcg_status = data;
3481 break;
3482 case MSR_IA32_MCG_CTL:
3483 if (!(mcg_cap & MCG_CTL_P) &&
3484 (data || !msr_info->host_initiated))
3485 return 1;
3486 if (data != 0 && data != ~(u64)0)
3487 return 1;
3488 vcpu->arch.mcg_ctl = data;
3489 break;
3490 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
3491 last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
3492 if (msr > last_msr)
3493 return 1;
3494
3495 if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated))
3496 return 1;
3497 /* An attempt to write a 1 to a reserved bit raises #GP */
3498 if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK))
3499 return 1;
3500 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
3501 last_msr + 1 - MSR_IA32_MC0_CTL2);
3502 vcpu->arch.mci_ctl2_banks[offset] = data;
3503 break;
3504 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
3505 last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
3506 if (msr > last_msr)
3507 return 1;
3508
3509 /*
3510 * Only 0 or all 1s can be written to IA32_MCi_CTL, all other
3511 * values are architecturally undefined. But, some Linux
3512 * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB
3513 * issue on AMD K8s, allow bit 10 to be clear when setting all
3514 * other bits in order to avoid an uncaught #GP in the guest.
3515 *
3516 * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable,
3517 * single-bit ECC data errors.
3518 */
3519 if (is_mci_control_msr(msr) &&
3520 data != 0 && (data | (1 << 10) | 1) != ~(u64)0)
3521 return 1;
3522
3523 /*
3524 * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR.
3525 * AMD-based CPUs allow non-zero values, but if and only if
3526 * HWCR[McStatusWrEn] is set.
3527 */
3528 if (!msr_info->host_initiated && is_mci_status_msr(msr) &&
3529 data != 0 && !can_set_mci_status(vcpu))
3530 return 1;
3531
3532 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
3533 last_msr + 1 - MSR_IA32_MC0_CTL);
3534 vcpu->arch.mce_banks[offset] = data;
3535 break;
3536 default:
3537 return 1;
3538 }
3539 return 0;
3540 }
3541
kvm_pv_async_pf_enabled(struct kvm_vcpu * vcpu)3542 static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
3543 {
3544 u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
3545
3546 return (vcpu->arch.apf.msr_en_val & mask) == mask;
3547 }
3548
kvm_pv_enable_async_pf(struct kvm_vcpu * vcpu,u64 data)3549 static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
3550 {
3551 gpa_t gpa = data & ~0x3f;
3552
3553 /* Bits 4:5 are reserved, Should be zero */
3554 if (data & 0x30)
3555 return 1;
3556
3557 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) &&
3558 (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT))
3559 return 1;
3560
3561 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) &&
3562 (data & KVM_ASYNC_PF_DELIVERY_AS_INT))
3563 return 1;
3564
3565 if (!lapic_in_kernel(vcpu))
3566 return data ? 1 : 0;
3567
3568 vcpu->arch.apf.msr_en_val = data;
3569
3570 if (!kvm_pv_async_pf_enabled(vcpu)) {
3571 kvm_clear_async_pf_completion_queue(vcpu);
3572 kvm_async_pf_hash_reset(vcpu);
3573 return 0;
3574 }
3575
3576 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
3577 sizeof(u64)))
3578 return 1;
3579
3580 vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
3581 vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
3582
3583 kvm_async_pf_wakeup_all(vcpu);
3584
3585 return 0;
3586 }
3587
kvm_pv_enable_async_pf_int(struct kvm_vcpu * vcpu,u64 data)3588 static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data)
3589 {
3590 /* Bits 8-63 are reserved */
3591 if (data >> 8)
3592 return 1;
3593
3594 if (!lapic_in_kernel(vcpu))
3595 return 1;
3596
3597 vcpu->arch.apf.msr_int_val = data;
3598
3599 vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK;
3600
3601 return 0;
3602 }
3603
kvmclock_reset(struct kvm_vcpu * vcpu)3604 static void kvmclock_reset(struct kvm_vcpu *vcpu)
3605 {
3606 kvm_gpc_deactivate(&vcpu->arch.pv_time);
3607 vcpu->arch.time = 0;
3608 }
3609
kvm_vcpu_flush_tlb_all(struct kvm_vcpu * vcpu)3610 static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu)
3611 {
3612 ++vcpu->stat.tlb_flush;
3613 static_call(kvm_x86_flush_tlb_all)(vcpu);
3614
3615 /* Flushing all ASIDs flushes the current ASID... */
3616 kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
3617 }
3618
kvm_vcpu_flush_tlb_guest(struct kvm_vcpu * vcpu)3619 static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
3620 {
3621 ++vcpu->stat.tlb_flush;
3622
3623 if (!tdp_enabled) {
3624 /*
3625 * A TLB flush on behalf of the guest is equivalent to
3626 * INVPCID(all), toggling CR4.PGE, etc., which requires
3627 * a forced sync of the shadow page tables. Ensure all the
3628 * roots are synced and the guest TLB in hardware is clean.
3629 */
3630 kvm_mmu_sync_roots(vcpu);
3631 kvm_mmu_sync_prev_roots(vcpu);
3632 }
3633
3634 static_call(kvm_x86_flush_tlb_guest)(vcpu);
3635
3636 /*
3637 * Flushing all "guest" TLB is always a superset of Hyper-V's fine
3638 * grained flushing.
3639 */
3640 kvm_hv_vcpu_purge_flush_tlb(vcpu);
3641 }
3642
3643
kvm_vcpu_flush_tlb_current(struct kvm_vcpu * vcpu)3644 static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu)
3645 {
3646 ++vcpu->stat.tlb_flush;
3647 static_call(kvm_x86_flush_tlb_current)(vcpu);
3648 }
3649
3650 /*
3651 * Service "local" TLB flush requests, which are specific to the current MMU
3652 * context. In addition to the generic event handling in vcpu_enter_guest(),
3653 * TLB flushes that are targeted at an MMU context also need to be serviced
3654 * prior before nested VM-Enter/VM-Exit.
3655 */
kvm_service_local_tlb_flush_requests(struct kvm_vcpu * vcpu)3656 void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu)
3657 {
3658 if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu))
3659 kvm_vcpu_flush_tlb_current(vcpu);
3660
3661 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu))
3662 kvm_vcpu_flush_tlb_guest(vcpu);
3663 }
3664 EXPORT_SYMBOL_GPL(kvm_service_local_tlb_flush_requests);
3665
record_steal_time(struct kvm_vcpu * vcpu)3666 static void record_steal_time(struct kvm_vcpu *vcpu)
3667 {
3668 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
3669 struct kvm_steal_time __user *st;
3670 struct kvm_memslots *slots;
3671 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
3672 u64 steal;
3673 u32 version;
3674
3675 if (kvm_xen_msr_enabled(vcpu->kvm)) {
3676 kvm_xen_runstate_set_running(vcpu);
3677 return;
3678 }
3679
3680 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
3681 return;
3682
3683 if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
3684 return;
3685
3686 slots = kvm_memslots(vcpu->kvm);
3687
3688 if (unlikely(slots->generation != ghc->generation ||
3689 gpa != ghc->gpa ||
3690 kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {
3691 /* We rely on the fact that it fits in a single page. */
3692 BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);
3693
3694 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st)) ||
3695 kvm_is_error_hva(ghc->hva) || !ghc->memslot)
3696 return;
3697 }
3698
3699 st = (struct kvm_steal_time __user *)ghc->hva;
3700 /*
3701 * Doing a TLB flush here, on the guest's behalf, can avoid
3702 * expensive IPIs.
3703 */
3704 if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
3705 u8 st_preempted = 0;
3706 int err = -EFAULT;
3707
3708 if (!user_access_begin(st, sizeof(*st)))
3709 return;
3710
3711 asm volatile("1: xchgb %0, %2\n"
3712 "xor %1, %1\n"
3713 "2:\n"
3714 _ASM_EXTABLE_UA(1b, 2b)
3715 : "+q" (st_preempted),
3716 "+&r" (err),
3717 "+m" (st->preempted));
3718 if (err)
3719 goto out;
3720
3721 user_access_end();
3722
3723 vcpu->arch.st.preempted = 0;
3724
3725 trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
3726 st_preempted & KVM_VCPU_FLUSH_TLB);
3727 if (st_preempted & KVM_VCPU_FLUSH_TLB)
3728 kvm_vcpu_flush_tlb_guest(vcpu);
3729
3730 if (!user_access_begin(st, sizeof(*st)))
3731 goto dirty;
3732 } else {
3733 if (!user_access_begin(st, sizeof(*st)))
3734 return;
3735
3736 unsafe_put_user(0, &st->preempted, out);
3737 vcpu->arch.st.preempted = 0;
3738 }
3739
3740 unsafe_get_user(version, &st->version, out);
3741 if (version & 1)
3742 version += 1; /* first time write, random junk */
3743
3744 version += 1;
3745 unsafe_put_user(version, &st->version, out);
3746
3747 smp_wmb();
3748
3749 unsafe_get_user(steal, &st->steal, out);
3750 steal += current->sched_info.run_delay -
3751 vcpu->arch.st.last_steal;
3752 vcpu->arch.st.last_steal = current->sched_info.run_delay;
3753 unsafe_put_user(steal, &st->steal, out);
3754
3755 version += 1;
3756 unsafe_put_user(version, &st->version, out);
3757
3758 out:
3759 user_access_end();
3760 dirty:
3761 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
3762 }
3763
kvm_is_msr_to_save(u32 msr_index)3764 static bool kvm_is_msr_to_save(u32 msr_index)
3765 {
3766 unsigned int i;
3767
3768 for (i = 0; i < num_msrs_to_save; i++) {
3769 if (msrs_to_save[i] == msr_index)
3770 return true;
3771 }
3772
3773 return false;
3774 }
3775
kvm_set_msr_common(struct kvm_vcpu * vcpu,struct msr_data * msr_info)3776 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3777 {
3778 u32 msr = msr_info->index;
3779 u64 data = msr_info->data;
3780
3781 if (msr && msr == vcpu->kvm->arch.xen_hvm_config.msr)
3782 return kvm_xen_write_hypercall_page(vcpu, data);
3783
3784 switch (msr) {
3785 case MSR_AMD64_NB_CFG:
3786 case MSR_IA32_UCODE_WRITE:
3787 case MSR_VM_HSAVE_PA:
3788 case MSR_AMD64_PATCH_LOADER:
3789 case MSR_AMD64_BU_CFG2:
3790 case MSR_AMD64_DC_CFG:
3791 case MSR_AMD64_TW_CFG:
3792 case MSR_F15H_EX_CFG:
3793 break;
3794
3795 case MSR_IA32_UCODE_REV:
3796 if (msr_info->host_initiated)
3797 vcpu->arch.microcode_version = data;
3798 break;
3799 case MSR_IA32_ARCH_CAPABILITIES:
3800 if (!msr_info->host_initiated)
3801 return 1;
3802 vcpu->arch.arch_capabilities = data;
3803 break;
3804 case MSR_IA32_PERF_CAPABILITIES:
3805 if (!msr_info->host_initiated)
3806 return 1;
3807 if (data & ~kvm_caps.supported_perf_cap)
3808 return 1;
3809
3810 /*
3811 * Note, this is not just a performance optimization! KVM
3812 * disallows changing feature MSRs after the vCPU has run; PMU
3813 * refresh will bug the VM if called after the vCPU has run.
3814 */
3815 if (vcpu->arch.perf_capabilities == data)
3816 break;
3817
3818 vcpu->arch.perf_capabilities = data;
3819 kvm_pmu_refresh(vcpu);
3820 break;
3821 case MSR_IA32_PRED_CMD: {
3822 u64 reserved_bits = ~(PRED_CMD_IBPB | PRED_CMD_SBPB);
3823
3824 if (!msr_info->host_initiated) {
3825 if ((!guest_has_pred_cmd_msr(vcpu)))
3826 return 1;
3827
3828 if (!guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL) &&
3829 !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
3830 reserved_bits |= PRED_CMD_IBPB;
3831
3832 if (!guest_cpuid_has(vcpu, X86_FEATURE_SBPB))
3833 reserved_bits |= PRED_CMD_SBPB;
3834 }
3835
3836 if (!boot_cpu_has(X86_FEATURE_IBPB))
3837 reserved_bits |= PRED_CMD_IBPB;
3838
3839 if (!boot_cpu_has(X86_FEATURE_SBPB))
3840 reserved_bits |= PRED_CMD_SBPB;
3841
3842 if (data & reserved_bits)
3843 return 1;
3844
3845 if (!data)
3846 break;
3847
3848 wrmsrl(MSR_IA32_PRED_CMD, data);
3849 break;
3850 }
3851 case MSR_IA32_FLUSH_CMD:
3852 if (!msr_info->host_initiated &&
3853 !guest_cpuid_has(vcpu, X86_FEATURE_FLUSH_L1D))
3854 return 1;
3855
3856 if (!boot_cpu_has(X86_FEATURE_FLUSH_L1D) || (data & ~L1D_FLUSH))
3857 return 1;
3858 if (!data)
3859 break;
3860
3861 wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
3862 break;
3863 case MSR_EFER:
3864 return set_efer(vcpu, msr_info);
3865 case MSR_K7_HWCR:
3866 data &= ~(u64)0x40; /* ignore flush filter disable */
3867 data &= ~(u64)0x100; /* ignore ignne emulation enable */
3868 data &= ~(u64)0x8; /* ignore TLB cache disable */
3869
3870 /*
3871 * Allow McStatusWrEn and TscFreqSel. (Linux guests from v3.2
3872 * through at least v6.6 whine if TscFreqSel is clear,
3873 * depending on F/M/S.
3874 */
3875 if (data & ~(BIT_ULL(18) | BIT_ULL(24))) {
3876 kvm_pr_unimpl_wrmsr(vcpu, msr, data);
3877 return 1;
3878 }
3879 vcpu->arch.msr_hwcr = data;
3880 break;
3881 case MSR_FAM10H_MMIO_CONF_BASE:
3882 if (data != 0) {
3883 kvm_pr_unimpl_wrmsr(vcpu, msr, data);
3884 return 1;
3885 }
3886 break;
3887 case MSR_IA32_CR_PAT:
3888 if (!kvm_pat_valid(data))
3889 return 1;
3890
3891 vcpu->arch.pat = data;
3892 break;
3893 case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000:
3894 case MSR_MTRRdefType:
3895 return kvm_mtrr_set_msr(vcpu, msr, data);
3896 case MSR_IA32_APICBASE:
3897 return kvm_set_apic_base(vcpu, msr_info);
3898 case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
3899 return kvm_x2apic_msr_write(vcpu, msr, data);
3900 case MSR_IA32_TSC_DEADLINE:
3901 kvm_set_lapic_tscdeadline_msr(vcpu, data);
3902 break;
3903 case MSR_IA32_TSC_ADJUST:
3904 if (guest_cpuid_has(vcpu, X86_FEATURE_TSC_ADJUST)) {
3905 if (!msr_info->host_initiated) {
3906 s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
3907 adjust_tsc_offset_guest(vcpu, adj);
3908 /* Before back to guest, tsc_timestamp must be adjusted
3909 * as well, otherwise guest's percpu pvclock time could jump.
3910 */
3911 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
3912 }
3913 vcpu->arch.ia32_tsc_adjust_msr = data;
3914 }
3915 break;
3916 case MSR_IA32_MISC_ENABLE: {
3917 u64 old_val = vcpu->arch.ia32_misc_enable_msr;
3918
3919 if (!msr_info->host_initiated) {
3920 /* RO bits */
3921 if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK)
3922 return 1;
3923
3924 /* R bits, i.e. writes are ignored, but don't fault. */
3925 data = data & ~MSR_IA32_MISC_ENABLE_EMON;
3926 data |= old_val & MSR_IA32_MISC_ENABLE_EMON;
3927 }
3928
3929 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) &&
3930 ((old_val ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) {
3931 if (!guest_cpuid_has(vcpu, X86_FEATURE_XMM3))
3932 return 1;
3933 vcpu->arch.ia32_misc_enable_msr = data;
3934 kvm_update_cpuid_runtime(vcpu);
3935 } else {
3936 vcpu->arch.ia32_misc_enable_msr = data;
3937 }
3938 break;
3939 }
3940 case MSR_IA32_SMBASE:
3941 if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated)
3942 return 1;
3943 vcpu->arch.smbase = data;
3944 break;
3945 case MSR_IA32_POWER_CTL:
3946 vcpu->arch.msr_ia32_power_ctl = data;
3947 break;
3948 case MSR_IA32_TSC:
3949 if (msr_info->host_initiated) {
3950 kvm_synchronize_tsc(vcpu, &data);
3951 } else {
3952 u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset;
3953 adjust_tsc_offset_guest(vcpu, adj);
3954 vcpu->arch.ia32_tsc_adjust_msr += adj;
3955 }
3956 break;
3957 case MSR_IA32_XSS:
3958 if (!msr_info->host_initiated &&
3959 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
3960 return 1;
3961 /*
3962 * KVM supports exposing PT to the guest, but does not support
3963 * IA32_XSS[bit 8]. Guests have to use RDMSR/WRMSR rather than
3964 * XSAVES/XRSTORS to save/restore PT MSRs.
3965 */
3966 if (data & ~kvm_caps.supported_xss)
3967 return 1;
3968 vcpu->arch.ia32_xss = data;
3969 kvm_update_cpuid_runtime(vcpu);
3970 break;
3971 case MSR_SMI_COUNT:
3972 if (!msr_info->host_initiated)
3973 return 1;
3974 vcpu->arch.smi_count = data;
3975 break;
3976 case MSR_KVM_WALL_CLOCK_NEW:
3977 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3978 return 1;
3979
3980 vcpu->kvm->arch.wall_clock = data;
3981 kvm_write_wall_clock(vcpu->kvm, data, 0);
3982 break;
3983 case MSR_KVM_WALL_CLOCK:
3984 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3985 return 1;
3986
3987 vcpu->kvm->arch.wall_clock = data;
3988 kvm_write_wall_clock(vcpu->kvm, data, 0);
3989 break;
3990 case MSR_KVM_SYSTEM_TIME_NEW:
3991 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
3992 return 1;
3993
3994 kvm_write_system_time(vcpu, data, false, msr_info->host_initiated);
3995 break;
3996 case MSR_KVM_SYSTEM_TIME:
3997 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
3998 return 1;
3999
4000 kvm_write_system_time(vcpu, data, true, msr_info->host_initiated);
4001 break;
4002 case MSR_KVM_ASYNC_PF_EN:
4003 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
4004 return 1;
4005
4006 if (kvm_pv_enable_async_pf(vcpu, data))
4007 return 1;
4008 break;
4009 case MSR_KVM_ASYNC_PF_INT:
4010 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4011 return 1;
4012
4013 if (kvm_pv_enable_async_pf_int(vcpu, data))
4014 return 1;
4015 break;
4016 case MSR_KVM_ASYNC_PF_ACK:
4017 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4018 return 1;
4019 if (data & 0x1) {
4020 vcpu->arch.apf.pageready_pending = false;
4021 kvm_check_async_pf_completion(vcpu);
4022 }
4023 break;
4024 case MSR_KVM_STEAL_TIME:
4025 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
4026 return 1;
4027
4028 if (unlikely(!sched_info_on()))
4029 return 1;
4030
4031 if (data & KVM_STEAL_RESERVED_MASK)
4032 return 1;
4033
4034 vcpu->arch.st.msr_val = data;
4035
4036 if (!(data & KVM_MSR_ENABLED))
4037 break;
4038
4039 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
4040
4041 break;
4042 case MSR_KVM_PV_EOI_EN:
4043 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
4044 return 1;
4045
4046 if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8)))
4047 return 1;
4048 break;
4049
4050 case MSR_KVM_POLL_CONTROL:
4051 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
4052 return 1;
4053
4054 /* only enable bit supported */
4055 if (data & (-1ULL << 1))
4056 return 1;
4057
4058 vcpu->arch.msr_kvm_poll_control = data;
4059 break;
4060
4061 case MSR_IA32_MCG_CTL:
4062 case MSR_IA32_MCG_STATUS:
4063 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
4064 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
4065 return set_msr_mce(vcpu, msr_info);
4066
4067 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
4068 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
4069 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
4070 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
4071 if (kvm_pmu_is_valid_msr(vcpu, msr))
4072 return kvm_pmu_set_msr(vcpu, msr_info);
4073
4074 if (data)
4075 kvm_pr_unimpl_wrmsr(vcpu, msr, data);
4076 break;
4077 case MSR_K7_CLK_CTL:
4078 /*
4079 * Ignore all writes to this no longer documented MSR.
4080 * Writes are only relevant for old K7 processors,
4081 * all pre-dating SVM, but a recommended workaround from
4082 * AMD for these chips. It is possible to specify the
4083 * affected processor models on the command line, hence
4084 * the need to ignore the workaround.
4085 */
4086 break;
4087 #ifdef CONFIG_KVM_HYPERV
4088 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
4089 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
4090 case HV_X64_MSR_SYNDBG_OPTIONS:
4091 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
4092 case HV_X64_MSR_CRASH_CTL:
4093 case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
4094 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
4095 case HV_X64_MSR_TSC_EMULATION_CONTROL:
4096 case HV_X64_MSR_TSC_EMULATION_STATUS:
4097 case HV_X64_MSR_TSC_INVARIANT_CONTROL:
4098 return kvm_hv_set_msr_common(vcpu, msr, data,
4099 msr_info->host_initiated);
4100 #endif
4101 case MSR_IA32_BBL_CR_CTL3:
4102 /* Drop writes to this legacy MSR -- see rdmsr
4103 * counterpart for further detail.
4104 */
4105 kvm_pr_unimpl_wrmsr(vcpu, msr, data);
4106 break;
4107 case MSR_AMD64_OSVW_ID_LENGTH:
4108 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4109 return 1;
4110 vcpu->arch.osvw.length = data;
4111 break;
4112 case MSR_AMD64_OSVW_STATUS:
4113 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4114 return 1;
4115 vcpu->arch.osvw.status = data;
4116 break;
4117 case MSR_PLATFORM_INFO:
4118 if (!msr_info->host_initiated ||
4119 (!(data & MSR_PLATFORM_INFO_CPUID_FAULT) &&
4120 cpuid_fault_enabled(vcpu)))
4121 return 1;
4122 vcpu->arch.msr_platform_info = data;
4123 break;
4124 case MSR_MISC_FEATURES_ENABLES:
4125 if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT ||
4126 (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
4127 !supports_cpuid_fault(vcpu)))
4128 return 1;
4129 vcpu->arch.msr_misc_features_enables = data;
4130 break;
4131 #ifdef CONFIG_X86_64
4132 case MSR_IA32_XFD:
4133 if (!msr_info->host_initiated &&
4134 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4135 return 1;
4136
4137 if (data & ~kvm_guest_supported_xfd(vcpu))
4138 return 1;
4139
4140 fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data);
4141 break;
4142 case MSR_IA32_XFD_ERR:
4143 if (!msr_info->host_initiated &&
4144 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4145 return 1;
4146
4147 if (data & ~kvm_guest_supported_xfd(vcpu))
4148 return 1;
4149
4150 vcpu->arch.guest_fpu.xfd_err = data;
4151 break;
4152 #endif
4153 default:
4154 if (kvm_pmu_is_valid_msr(vcpu, msr))
4155 return kvm_pmu_set_msr(vcpu, msr_info);
4156
4157 /*
4158 * Userspace is allowed to write '0' to MSRs that KVM reports
4159 * as to-be-saved, even if an MSRs isn't fully supported.
4160 */
4161 if (msr_info->host_initiated && !data &&
4162 kvm_is_msr_to_save(msr))
4163 break;
4164
4165 return KVM_MSR_RET_INVALID;
4166 }
4167 return 0;
4168 }
4169 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
4170
get_msr_mce(struct kvm_vcpu * vcpu,u32 msr,u64 * pdata,bool host)4171 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)
4172 {
4173 u64 data;
4174 u64 mcg_cap = vcpu->arch.mcg_cap;
4175 unsigned bank_num = mcg_cap & 0xff;
4176 u32 offset, last_msr;
4177
4178 switch (msr) {
4179 case MSR_IA32_P5_MC_ADDR:
4180 case MSR_IA32_P5_MC_TYPE:
4181 data = 0;
4182 break;
4183 case MSR_IA32_MCG_CAP:
4184 data = vcpu->arch.mcg_cap;
4185 break;
4186 case MSR_IA32_MCG_CTL:
4187 if (!(mcg_cap & MCG_CTL_P) && !host)
4188 return 1;
4189 data = vcpu->arch.mcg_ctl;
4190 break;
4191 case MSR_IA32_MCG_STATUS:
4192 data = vcpu->arch.mcg_status;
4193 break;
4194 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
4195 last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1;
4196 if (msr > last_msr)
4197 return 1;
4198
4199 if (!(mcg_cap & MCG_CMCI_P) && !host)
4200 return 1;
4201 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2,
4202 last_msr + 1 - MSR_IA32_MC0_CTL2);
4203 data = vcpu->arch.mci_ctl2_banks[offset];
4204 break;
4205 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
4206 last_msr = MSR_IA32_MCx_CTL(bank_num) - 1;
4207 if (msr > last_msr)
4208 return 1;
4209
4210 offset = array_index_nospec(msr - MSR_IA32_MC0_CTL,
4211 last_msr + 1 - MSR_IA32_MC0_CTL);
4212 data = vcpu->arch.mce_banks[offset];
4213 break;
4214 default:
4215 return 1;
4216 }
4217 *pdata = data;
4218 return 0;
4219 }
4220
kvm_get_msr_common(struct kvm_vcpu * vcpu,struct msr_data * msr_info)4221 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
4222 {
4223 switch (msr_info->index) {
4224 case MSR_IA32_PLATFORM_ID:
4225 case MSR_IA32_EBL_CR_POWERON:
4226 case MSR_IA32_LASTBRANCHFROMIP:
4227 case MSR_IA32_LASTBRANCHTOIP:
4228 case MSR_IA32_LASTINTFROMIP:
4229 case MSR_IA32_LASTINTTOIP:
4230 case MSR_AMD64_SYSCFG:
4231 case MSR_K8_TSEG_ADDR:
4232 case MSR_K8_TSEG_MASK:
4233 case MSR_VM_HSAVE_PA:
4234 case MSR_K8_INT_PENDING_MSG:
4235 case MSR_AMD64_NB_CFG:
4236 case MSR_FAM10H_MMIO_CONF_BASE:
4237 case MSR_AMD64_BU_CFG2:
4238 case MSR_IA32_PERF_CTL:
4239 case MSR_AMD64_DC_CFG:
4240 case MSR_AMD64_TW_CFG:
4241 case MSR_F15H_EX_CFG:
4242 /*
4243 * Intel Sandy Bridge CPUs must support the RAPL (running average power
4244 * limit) MSRs. Just return 0, as we do not want to expose the host
4245 * data here. Do not conditionalize this on CPUID, as KVM does not do
4246 * so for existing CPU-specific MSRs.
4247 */
4248 case MSR_RAPL_POWER_UNIT:
4249 case MSR_PP0_ENERGY_STATUS: /* Power plane 0 (core) */
4250 case MSR_PP1_ENERGY_STATUS: /* Power plane 1 (graphics uncore) */
4251 case MSR_PKG_ENERGY_STATUS: /* Total package */
4252 case MSR_DRAM_ENERGY_STATUS: /* DRAM controller */
4253 msr_info->data = 0;
4254 break;
4255 case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3:
4256 case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3:
4257 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1:
4258 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1:
4259 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
4260 return kvm_pmu_get_msr(vcpu, msr_info);
4261 msr_info->data = 0;
4262 break;
4263 case MSR_IA32_UCODE_REV:
4264 msr_info->data = vcpu->arch.microcode_version;
4265 break;
4266 case MSR_IA32_ARCH_CAPABILITIES:
4267 if (!msr_info->host_initiated &&
4268 !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
4269 return 1;
4270 msr_info->data = vcpu->arch.arch_capabilities;
4271 break;
4272 case MSR_IA32_PERF_CAPABILITIES:
4273 if (!msr_info->host_initiated &&
4274 !guest_cpuid_has(vcpu, X86_FEATURE_PDCM))
4275 return 1;
4276 msr_info->data = vcpu->arch.perf_capabilities;
4277 break;
4278 case MSR_IA32_POWER_CTL:
4279 msr_info->data = vcpu->arch.msr_ia32_power_ctl;
4280 break;
4281 case MSR_IA32_TSC: {
4282 /*
4283 * Intel SDM states that MSR_IA32_TSC read adds the TSC offset
4284 * even when not intercepted. AMD manual doesn't explicitly
4285 * state this but appears to behave the same.
4286 *
4287 * On userspace reads and writes, however, we unconditionally
4288 * return L1's TSC value to ensure backwards-compatible
4289 * behavior for migration.
4290 */
4291 u64 offset, ratio;
4292
4293 if (msr_info->host_initiated) {
4294 offset = vcpu->arch.l1_tsc_offset;
4295 ratio = vcpu->arch.l1_tsc_scaling_ratio;
4296 } else {
4297 offset = vcpu->arch.tsc_offset;
4298 ratio = vcpu->arch.tsc_scaling_ratio;
4299 }
4300
4301 msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset;
4302 break;
4303 }
4304 case MSR_IA32_CR_PAT:
4305 msr_info->data = vcpu->arch.pat;
4306 break;
4307 case MSR_MTRRcap:
4308 case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000:
4309 case MSR_MTRRdefType:
4310 return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data);
4311 case 0xcd: /* fsb frequency */
4312 msr_info->data = 3;
4313 break;
4314 /*
4315 * MSR_EBC_FREQUENCY_ID
4316 * Conservative value valid for even the basic CPU models.
4317 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
4318 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
4319 * and 266MHz for model 3, or 4. Set Core Clock
4320 * Frequency to System Bus Frequency Ratio to 1 (bits
4321 * 31:24) even though these are only valid for CPU
4322 * models > 2, however guests may end up dividing or
4323 * multiplying by zero otherwise.
4324 */
4325 case MSR_EBC_FREQUENCY_ID:
4326 msr_info->data = 1 << 24;
4327 break;
4328 case MSR_IA32_APICBASE:
4329 msr_info->data = kvm_get_apic_base(vcpu);
4330 break;
4331 case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
4332 return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data);
4333 case MSR_IA32_TSC_DEADLINE:
4334 msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu);
4335 break;
4336 case MSR_IA32_TSC_ADJUST:
4337 msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
4338 break;
4339 case MSR_IA32_MISC_ENABLE:
4340 msr_info->data = vcpu->arch.ia32_misc_enable_msr;
4341 break;
4342 case MSR_IA32_SMBASE:
4343 if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated)
4344 return 1;
4345 msr_info->data = vcpu->arch.smbase;
4346 break;
4347 case MSR_SMI_COUNT:
4348 msr_info->data = vcpu->arch.smi_count;
4349 break;
4350 case MSR_IA32_PERF_STATUS:
4351 /* TSC increment by tick */
4352 msr_info->data = 1000ULL;
4353 /* CPU multiplier */
4354 msr_info->data |= (((uint64_t)4ULL) << 40);
4355 break;
4356 case MSR_EFER:
4357 msr_info->data = vcpu->arch.efer;
4358 break;
4359 case MSR_KVM_WALL_CLOCK:
4360 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
4361 return 1;
4362
4363 msr_info->data = vcpu->kvm->arch.wall_clock;
4364 break;
4365 case MSR_KVM_WALL_CLOCK_NEW:
4366 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
4367 return 1;
4368
4369 msr_info->data = vcpu->kvm->arch.wall_clock;
4370 break;
4371 case MSR_KVM_SYSTEM_TIME:
4372 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE))
4373 return 1;
4374
4375 msr_info->data = vcpu->arch.time;
4376 break;
4377 case MSR_KVM_SYSTEM_TIME_NEW:
4378 if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2))
4379 return 1;
4380
4381 msr_info->data = vcpu->arch.time;
4382 break;
4383 case MSR_KVM_ASYNC_PF_EN:
4384 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF))
4385 return 1;
4386
4387 msr_info->data = vcpu->arch.apf.msr_en_val;
4388 break;
4389 case MSR_KVM_ASYNC_PF_INT:
4390 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4391 return 1;
4392
4393 msr_info->data = vcpu->arch.apf.msr_int_val;
4394 break;
4395 case MSR_KVM_ASYNC_PF_ACK:
4396 if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT))
4397 return 1;
4398
4399 msr_info->data = 0;
4400 break;
4401 case MSR_KVM_STEAL_TIME:
4402 if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME))
4403 return 1;
4404
4405 msr_info->data = vcpu->arch.st.msr_val;
4406 break;
4407 case MSR_KVM_PV_EOI_EN:
4408 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI))
4409 return 1;
4410
4411 msr_info->data = vcpu->arch.pv_eoi.msr_val;
4412 break;
4413 case MSR_KVM_POLL_CONTROL:
4414 if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL))
4415 return 1;
4416
4417 msr_info->data = vcpu->arch.msr_kvm_poll_control;
4418 break;
4419 case MSR_IA32_P5_MC_ADDR:
4420 case MSR_IA32_P5_MC_TYPE:
4421 case MSR_IA32_MCG_CAP:
4422 case MSR_IA32_MCG_CTL:
4423 case MSR_IA32_MCG_STATUS:
4424 case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1:
4425 case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
4426 return get_msr_mce(vcpu, msr_info->index, &msr_info->data,
4427 msr_info->host_initiated);
4428 case MSR_IA32_XSS:
4429 if (!msr_info->host_initiated &&
4430 !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))
4431 return 1;
4432 msr_info->data = vcpu->arch.ia32_xss;
4433 break;
4434 case MSR_K7_CLK_CTL:
4435 /*
4436 * Provide expected ramp-up count for K7. All other
4437 * are set to zero, indicating minimum divisors for
4438 * every field.
4439 *
4440 * This prevents guest kernels on AMD host with CPU
4441 * type 6, model 8 and higher from exploding due to
4442 * the rdmsr failing.
4443 */
4444 msr_info->data = 0x20000000;
4445 break;
4446 #ifdef CONFIG_KVM_HYPERV
4447 case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
4448 case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER:
4449 case HV_X64_MSR_SYNDBG_OPTIONS:
4450 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
4451 case HV_X64_MSR_CRASH_CTL:
4452 case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT:
4453 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
4454 case HV_X64_MSR_TSC_EMULATION_CONTROL:
4455 case HV_X64_MSR_TSC_EMULATION_STATUS:
4456 case HV_X64_MSR_TSC_INVARIANT_CONTROL:
4457 return kvm_hv_get_msr_common(vcpu,
4458 msr_info->index, &msr_info->data,
4459 msr_info->host_initiated);
4460 #endif
4461 case MSR_IA32_BBL_CR_CTL3:
4462 /* This legacy MSR exists but isn't fully documented in current
4463 * silicon. It is however accessed by winxp in very narrow
4464 * scenarios where it sets bit #19, itself documented as
4465 * a "reserved" bit. Best effort attempt to source coherent
4466 * read data here should the balance of the register be
4467 * interpreted by the guest:
4468 *
4469 * L2 cache control register 3: 64GB range, 256KB size,
4470 * enabled, latency 0x1, configured
4471 */
4472 msr_info->data = 0xbe702111;
4473 break;
4474 case MSR_AMD64_OSVW_ID_LENGTH:
4475 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4476 return 1;
4477 msr_info->data = vcpu->arch.osvw.length;
4478 break;
4479 case MSR_AMD64_OSVW_STATUS:
4480 if (!guest_cpuid_has(vcpu, X86_FEATURE_OSVW))
4481 return 1;
4482 msr_info->data = vcpu->arch.osvw.status;
4483 break;
4484 case MSR_PLATFORM_INFO:
4485 if (!msr_info->host_initiated &&
4486 !vcpu->kvm->arch.guest_can_read_msr_platform_info)
4487 return 1;
4488 msr_info->data = vcpu->arch.msr_platform_info;
4489 break;
4490 case MSR_MISC_FEATURES_ENABLES:
4491 msr_info->data = vcpu->arch.msr_misc_features_enables;
4492 break;
4493 case MSR_K7_HWCR:
4494 msr_info->data = vcpu->arch.msr_hwcr;
4495 break;
4496 #ifdef CONFIG_X86_64
4497 case MSR_IA32_XFD:
4498 if (!msr_info->host_initiated &&
4499 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4500 return 1;
4501
4502 msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd;
4503 break;
4504 case MSR_IA32_XFD_ERR:
4505 if (!msr_info->host_initiated &&
4506 !guest_cpuid_has(vcpu, X86_FEATURE_XFD))
4507 return 1;
4508
4509 msr_info->data = vcpu->arch.guest_fpu.xfd_err;
4510 break;
4511 #endif
4512 default:
4513 if (kvm_pmu_is_valid_msr(vcpu, msr_info->index))
4514 return kvm_pmu_get_msr(vcpu, msr_info);
4515
4516 /*
4517 * Userspace is allowed to read MSRs that KVM reports as
4518 * to-be-saved, even if an MSR isn't fully supported.
4519 */
4520 if (msr_info->host_initiated &&
4521 kvm_is_msr_to_save(msr_info->index)) {
4522 msr_info->data = 0;
4523 break;
4524 }
4525
4526 return KVM_MSR_RET_INVALID;
4527 }
4528 return 0;
4529 }
4530 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
4531
4532 /*
4533 * Read or write a bunch of msrs. All parameters are kernel addresses.
4534 *
4535 * @return number of msrs set successfully.
4536 */
__msr_io(struct kvm_vcpu * vcpu,struct kvm_msrs * msrs,struct kvm_msr_entry * entries,int (* do_msr)(struct kvm_vcpu * vcpu,unsigned index,u64 * data))4537 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
4538 struct kvm_msr_entry *entries,
4539 int (*do_msr)(struct kvm_vcpu *vcpu,
4540 unsigned index, u64 *data))
4541 {
4542 int i;
4543
4544 for (i = 0; i < msrs->nmsrs; ++i)
4545 if (do_msr(vcpu, entries[i].index, &entries[i].data))
4546 break;
4547
4548 return i;
4549 }
4550
4551 /*
4552 * Read or write a bunch of msrs. Parameters are user addresses.
4553 *
4554 * @return number of msrs set successfully.
4555 */
msr_io(struct kvm_vcpu * vcpu,struct kvm_msrs __user * user_msrs,int (* do_msr)(struct kvm_vcpu * vcpu,unsigned index,u64 * data),int writeback)4556 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
4557 int (*do_msr)(struct kvm_vcpu *vcpu,
4558 unsigned index, u64 *data),
4559 int writeback)
4560 {
4561 struct kvm_msrs msrs;
4562 struct kvm_msr_entry *entries;
4563 unsigned size;
4564 int r;
4565
4566 r = -EFAULT;
4567 if (copy_from_user(&msrs, user_msrs, sizeof(msrs)))
4568 goto out;
4569
4570 r = -E2BIG;
4571 if (msrs.nmsrs >= MAX_IO_MSRS)
4572 goto out;
4573
4574 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
4575 entries = memdup_user(user_msrs->entries, size);
4576 if (IS_ERR(entries)) {
4577 r = PTR_ERR(entries);
4578 goto out;
4579 }
4580
4581 r = __msr_io(vcpu, &msrs, entries, do_msr);
4582
4583 if (writeback && copy_to_user(user_msrs->entries, entries, size))
4584 r = -EFAULT;
4585
4586 kfree(entries);
4587 out:
4588 return r;
4589 }
4590
kvm_can_mwait_in_guest(void)4591 static inline bool kvm_can_mwait_in_guest(void)
4592 {
4593 return boot_cpu_has(X86_FEATURE_MWAIT) &&
4594 !boot_cpu_has_bug(X86_BUG_MONITOR) &&
4595 boot_cpu_has(X86_FEATURE_ARAT);
4596 }
4597
4598 #ifdef CONFIG_KVM_HYPERV
kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu * vcpu,struct kvm_cpuid2 __user * cpuid_arg)4599 static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,
4600 struct kvm_cpuid2 __user *cpuid_arg)
4601 {
4602 struct kvm_cpuid2 cpuid;
4603 int r;
4604
4605 r = -EFAULT;
4606 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4607 return r;
4608
4609 r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);
4610 if (r)
4611 return r;
4612
4613 r = -EFAULT;
4614 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4615 return r;
4616
4617 return 0;
4618 }
4619 #endif
4620
kvm_is_vm_type_supported(unsigned long type)4621 static bool kvm_is_vm_type_supported(unsigned long type)
4622 {
4623 return type < 32 && (kvm_caps.supported_vm_types & BIT(type));
4624 }
4625
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)4626 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
4627 {
4628 int r = 0;
4629
4630 switch (ext) {
4631 case KVM_CAP_IRQCHIP:
4632 case KVM_CAP_HLT:
4633 case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
4634 case KVM_CAP_SET_TSS_ADDR:
4635 case KVM_CAP_EXT_CPUID:
4636 case KVM_CAP_EXT_EMUL_CPUID:
4637 case KVM_CAP_CLOCKSOURCE:
4638 case KVM_CAP_PIT:
4639 case KVM_CAP_NOP_IO_DELAY:
4640 case KVM_CAP_MP_STATE:
4641 case KVM_CAP_SYNC_MMU:
4642 case KVM_CAP_USER_NMI:
4643 case KVM_CAP_REINJECT_CONTROL:
4644 case KVM_CAP_IRQ_INJECT_STATUS:
4645 case KVM_CAP_IOEVENTFD:
4646 case KVM_CAP_IOEVENTFD_NO_LENGTH:
4647 case KVM_CAP_PIT2:
4648 case KVM_CAP_PIT_STATE2:
4649 case KVM_CAP_SET_IDENTITY_MAP_ADDR:
4650 case KVM_CAP_VCPU_EVENTS:
4651 #ifdef CONFIG_KVM_HYPERV
4652 case KVM_CAP_HYPERV:
4653 case KVM_CAP_HYPERV_VAPIC:
4654 case KVM_CAP_HYPERV_SPIN:
4655 case KVM_CAP_HYPERV_TIME:
4656 case KVM_CAP_HYPERV_SYNIC:
4657 case KVM_CAP_HYPERV_SYNIC2:
4658 case KVM_CAP_HYPERV_VP_INDEX:
4659 case KVM_CAP_HYPERV_EVENTFD:
4660 case KVM_CAP_HYPERV_TLBFLUSH:
4661 case KVM_CAP_HYPERV_SEND_IPI:
4662 case KVM_CAP_HYPERV_CPUID:
4663 case KVM_CAP_HYPERV_ENFORCE_CPUID:
4664 case KVM_CAP_SYS_HYPERV_CPUID:
4665 #endif
4666 case KVM_CAP_PCI_SEGMENT:
4667 case KVM_CAP_DEBUGREGS:
4668 case KVM_CAP_X86_ROBUST_SINGLESTEP:
4669 case KVM_CAP_XSAVE:
4670 case KVM_CAP_ASYNC_PF:
4671 case KVM_CAP_ASYNC_PF_INT:
4672 case KVM_CAP_GET_TSC_KHZ:
4673 case KVM_CAP_KVMCLOCK_CTRL:
4674 case KVM_CAP_READONLY_MEM:
4675 case KVM_CAP_IOAPIC_POLARITY_IGNORED:
4676 case KVM_CAP_TSC_DEADLINE_TIMER:
4677 case KVM_CAP_DISABLE_QUIRKS:
4678 case KVM_CAP_SET_BOOT_CPU_ID:
4679 case KVM_CAP_SPLIT_IRQCHIP:
4680 case KVM_CAP_IMMEDIATE_EXIT:
4681 case KVM_CAP_PMU_EVENT_FILTER:
4682 case KVM_CAP_PMU_EVENT_MASKED_EVENTS:
4683 case KVM_CAP_GET_MSR_FEATURES:
4684 case KVM_CAP_MSR_PLATFORM_INFO:
4685 case KVM_CAP_EXCEPTION_PAYLOAD:
4686 case KVM_CAP_X86_TRIPLE_FAULT_EVENT:
4687 case KVM_CAP_SET_GUEST_DEBUG:
4688 case KVM_CAP_LAST_CPU:
4689 case KVM_CAP_X86_USER_SPACE_MSR:
4690 case KVM_CAP_X86_MSR_FILTER:
4691 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
4692 #ifdef CONFIG_X86_SGX_KVM
4693 case KVM_CAP_SGX_ATTRIBUTE:
4694 #endif
4695 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM:
4696 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
4697 case KVM_CAP_SREGS2:
4698 case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
4699 case KVM_CAP_VCPU_ATTRIBUTES:
4700 case KVM_CAP_SYS_ATTRIBUTES:
4701 case KVM_CAP_VAPIC:
4702 case KVM_CAP_ENABLE_CAP:
4703 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES:
4704 case KVM_CAP_IRQFD_RESAMPLE:
4705 case KVM_CAP_MEMORY_FAULT_INFO:
4706 r = 1;
4707 break;
4708 case KVM_CAP_EXIT_HYPERCALL:
4709 r = KVM_EXIT_HYPERCALL_VALID_MASK;
4710 break;
4711 case KVM_CAP_SET_GUEST_DEBUG2:
4712 return KVM_GUESTDBG_VALID_MASK;
4713 #ifdef CONFIG_KVM_XEN
4714 case KVM_CAP_XEN_HVM:
4715 r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR |
4716 KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL |
4717 KVM_XEN_HVM_CONFIG_SHARED_INFO |
4718 KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL |
4719 KVM_XEN_HVM_CONFIG_EVTCHN_SEND |
4720 KVM_XEN_HVM_CONFIG_PVCLOCK_TSC_UNSTABLE |
4721 KVM_XEN_HVM_CONFIG_SHARED_INFO_HVA;
4722 if (sched_info_on())
4723 r |= KVM_XEN_HVM_CONFIG_RUNSTATE |
4724 KVM_XEN_HVM_CONFIG_RUNSTATE_UPDATE_FLAG;
4725 break;
4726 #endif
4727 case KVM_CAP_SYNC_REGS:
4728 r = KVM_SYNC_X86_VALID_FIELDS;
4729 break;
4730 case KVM_CAP_ADJUST_CLOCK:
4731 r = KVM_CLOCK_VALID_FLAGS;
4732 break;
4733 case KVM_CAP_X86_DISABLE_EXITS:
4734 r = KVM_X86_DISABLE_EXITS_PAUSE;
4735
4736 if (!mitigate_smt_rsb) {
4737 r |= KVM_X86_DISABLE_EXITS_HLT |
4738 KVM_X86_DISABLE_EXITS_CSTATE;
4739
4740 if (kvm_can_mwait_in_guest())
4741 r |= KVM_X86_DISABLE_EXITS_MWAIT;
4742 }
4743 break;
4744 case KVM_CAP_X86_SMM:
4745 if (!IS_ENABLED(CONFIG_KVM_SMM))
4746 break;
4747
4748 /* SMBASE is usually relocated above 1M on modern chipsets,
4749 * and SMM handlers might indeed rely on 4G segment limits,
4750 * so do not report SMM to be available if real mode is
4751 * emulated via vm86 mode. Still, do not go to great lengths
4752 * to avoid userspace's usage of the feature, because it is a
4753 * fringe case that is not enabled except via specific settings
4754 * of the module parameters.
4755 */
4756 r = static_call(kvm_x86_has_emulated_msr)(kvm, MSR_IA32_SMBASE);
4757 break;
4758 case KVM_CAP_NR_VCPUS:
4759 r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS);
4760 break;
4761 case KVM_CAP_MAX_VCPUS:
4762 r = KVM_MAX_VCPUS;
4763 break;
4764 case KVM_CAP_MAX_VCPU_ID:
4765 r = KVM_MAX_VCPU_IDS;
4766 break;
4767 case KVM_CAP_PV_MMU: /* obsolete */
4768 r = 0;
4769 break;
4770 case KVM_CAP_MCE:
4771 r = KVM_MAX_MCE_BANKS;
4772 break;
4773 case KVM_CAP_XCRS:
4774 r = boot_cpu_has(X86_FEATURE_XSAVE);
4775 break;
4776 case KVM_CAP_TSC_CONTROL:
4777 case KVM_CAP_VM_TSC_CONTROL:
4778 r = kvm_caps.has_tsc_control;
4779 break;
4780 case KVM_CAP_X2APIC_API:
4781 r = KVM_X2APIC_API_VALID_FLAGS;
4782 break;
4783 case KVM_CAP_NESTED_STATE:
4784 r = kvm_x86_ops.nested_ops->get_state ?
4785 kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0;
4786 break;
4787 #ifdef CONFIG_KVM_HYPERV
4788 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
4789 r = kvm_x86_ops.enable_l2_tlb_flush != NULL;
4790 break;
4791 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
4792 r = kvm_x86_ops.nested_ops->enable_evmcs != NULL;
4793 break;
4794 #endif
4795 case KVM_CAP_SMALLER_MAXPHYADDR:
4796 r = (int) allow_smaller_maxphyaddr;
4797 break;
4798 case KVM_CAP_STEAL_TIME:
4799 r = sched_info_on();
4800 break;
4801 case KVM_CAP_X86_BUS_LOCK_EXIT:
4802 if (kvm_caps.has_bus_lock_exit)
4803 r = KVM_BUS_LOCK_DETECTION_OFF |
4804 KVM_BUS_LOCK_DETECTION_EXIT;
4805 else
4806 r = 0;
4807 break;
4808 case KVM_CAP_XSAVE2: {
4809 r = xstate_required_size(kvm_get_filtered_xcr0(), false);
4810 if (r < sizeof(struct kvm_xsave))
4811 r = sizeof(struct kvm_xsave);
4812 break;
4813 }
4814 case KVM_CAP_PMU_CAPABILITY:
4815 r = enable_pmu ? KVM_CAP_PMU_VALID_MASK : 0;
4816 break;
4817 case KVM_CAP_DISABLE_QUIRKS2:
4818 r = KVM_X86_VALID_QUIRKS;
4819 break;
4820 case KVM_CAP_X86_NOTIFY_VMEXIT:
4821 r = kvm_caps.has_notify_vmexit;
4822 break;
4823 case KVM_CAP_VM_TYPES:
4824 r = kvm_caps.supported_vm_types;
4825 break;
4826 default:
4827 break;
4828 }
4829 return r;
4830 }
4831
__kvm_x86_dev_get_attr(struct kvm_device_attr * attr,u64 * val)4832 static int __kvm_x86_dev_get_attr(struct kvm_device_attr *attr, u64 *val)
4833 {
4834 if (attr->group) {
4835 if (kvm_x86_ops.dev_get_attr)
4836 return static_call(kvm_x86_dev_get_attr)(attr->group, attr->attr, val);
4837 return -ENXIO;
4838 }
4839
4840 switch (attr->attr) {
4841 case KVM_X86_XCOMP_GUEST_SUPP:
4842 *val = kvm_caps.supported_xcr0;
4843 return 0;
4844 default:
4845 return -ENXIO;
4846 }
4847 }
4848
kvm_x86_dev_get_attr(struct kvm_device_attr * attr)4849 static int kvm_x86_dev_get_attr(struct kvm_device_attr *attr)
4850 {
4851 u64 __user *uaddr = u64_to_user_ptr(attr->addr);
4852 int r;
4853 u64 val;
4854
4855 r = __kvm_x86_dev_get_attr(attr, &val);
4856 if (r < 0)
4857 return r;
4858
4859 if (put_user(val, uaddr))
4860 return -EFAULT;
4861
4862 return 0;
4863 }
4864
kvm_x86_dev_has_attr(struct kvm_device_attr * attr)4865 static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr)
4866 {
4867 u64 val;
4868
4869 return __kvm_x86_dev_get_attr(attr, &val);
4870 }
4871
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)4872 long kvm_arch_dev_ioctl(struct file *filp,
4873 unsigned int ioctl, unsigned long arg)
4874 {
4875 void __user *argp = (void __user *)arg;
4876 long r;
4877
4878 switch (ioctl) {
4879 case KVM_GET_MSR_INDEX_LIST: {
4880 struct kvm_msr_list __user *user_msr_list = argp;
4881 struct kvm_msr_list msr_list;
4882 unsigned n;
4883
4884 r = -EFAULT;
4885 if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
4886 goto out;
4887 n = msr_list.nmsrs;
4888 msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs;
4889 if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
4890 goto out;
4891 r = -E2BIG;
4892 if (n < msr_list.nmsrs)
4893 goto out;
4894 r = -EFAULT;
4895 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
4896 num_msrs_to_save * sizeof(u32)))
4897 goto out;
4898 if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
4899 &emulated_msrs,
4900 num_emulated_msrs * sizeof(u32)))
4901 goto out;
4902 r = 0;
4903 break;
4904 }
4905 case KVM_GET_SUPPORTED_CPUID:
4906 case KVM_GET_EMULATED_CPUID: {
4907 struct kvm_cpuid2 __user *cpuid_arg = argp;
4908 struct kvm_cpuid2 cpuid;
4909
4910 r = -EFAULT;
4911 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
4912 goto out;
4913
4914 r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
4915 ioctl);
4916 if (r)
4917 goto out;
4918
4919 r = -EFAULT;
4920 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
4921 goto out;
4922 r = 0;
4923 break;
4924 }
4925 case KVM_X86_GET_MCE_CAP_SUPPORTED:
4926 r = -EFAULT;
4927 if (copy_to_user(argp, &kvm_caps.supported_mce_cap,
4928 sizeof(kvm_caps.supported_mce_cap)))
4929 goto out;
4930 r = 0;
4931 break;
4932 case KVM_GET_MSR_FEATURE_INDEX_LIST: {
4933 struct kvm_msr_list __user *user_msr_list = argp;
4934 struct kvm_msr_list msr_list;
4935 unsigned int n;
4936
4937 r = -EFAULT;
4938 if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
4939 goto out;
4940 n = msr_list.nmsrs;
4941 msr_list.nmsrs = num_msr_based_features;
4942 if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
4943 goto out;
4944 r = -E2BIG;
4945 if (n < msr_list.nmsrs)
4946 goto out;
4947 r = -EFAULT;
4948 if (copy_to_user(user_msr_list->indices, &msr_based_features,
4949 num_msr_based_features * sizeof(u32)))
4950 goto out;
4951 r = 0;
4952 break;
4953 }
4954 case KVM_GET_MSRS:
4955 r = msr_io(NULL, argp, do_get_msr_feature, 1);
4956 break;
4957 #ifdef CONFIG_KVM_HYPERV
4958 case KVM_GET_SUPPORTED_HV_CPUID:
4959 r = kvm_ioctl_get_supported_hv_cpuid(NULL, argp);
4960 break;
4961 #endif
4962 case KVM_GET_DEVICE_ATTR: {
4963 struct kvm_device_attr attr;
4964 r = -EFAULT;
4965 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4966 break;
4967 r = kvm_x86_dev_get_attr(&attr);
4968 break;
4969 }
4970 case KVM_HAS_DEVICE_ATTR: {
4971 struct kvm_device_attr attr;
4972 r = -EFAULT;
4973 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
4974 break;
4975 r = kvm_x86_dev_has_attr(&attr);
4976 break;
4977 }
4978 default:
4979 r = -EINVAL;
4980 break;
4981 }
4982 out:
4983 return r;
4984 }
4985
wbinvd_ipi(void * garbage)4986 static void wbinvd_ipi(void *garbage)
4987 {
4988 wbinvd();
4989 }
4990
need_emulate_wbinvd(struct kvm_vcpu * vcpu)4991 static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
4992 {
4993 return kvm_arch_has_noncoherent_dma(vcpu->kvm);
4994 }
4995
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)4996 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
4997 {
4998 /* Address WBINVD may be executed by guest */
4999 if (need_emulate_wbinvd(vcpu)) {
5000 if (static_call(kvm_x86_has_wbinvd_exit)())
5001 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
5002 else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
5003 smp_call_function_single(vcpu->cpu,
5004 wbinvd_ipi, NULL, 1);
5005 }
5006
5007 static_call(kvm_x86_vcpu_load)(vcpu, cpu);
5008
5009 /* Save host pkru register if supported */
5010 vcpu->arch.host_pkru = read_pkru();
5011
5012 /* Apply any externally detected TSC adjustments (due to suspend) */
5013 if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
5014 adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
5015 vcpu->arch.tsc_offset_adjustment = 0;
5016 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5017 }
5018
5019 if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
5020 s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
5021 rdtsc() - vcpu->arch.last_host_tsc;
5022 if (tsc_delta < 0)
5023 mark_tsc_unstable("KVM discovered backwards TSC");
5024
5025 if (kvm_check_tsc_unstable()) {
5026 u64 offset = kvm_compute_l1_tsc_offset(vcpu,
5027 vcpu->arch.last_guest_tsc);
5028 kvm_vcpu_write_tsc_offset(vcpu, offset);
5029 vcpu->arch.tsc_catchup = 1;
5030 }
5031
5032 if (kvm_lapic_hv_timer_in_use(vcpu))
5033 kvm_lapic_restart_hv_timer(vcpu);
5034
5035 /*
5036 * On a host with synchronized TSC, there is no need to update
5037 * kvmclock on vcpu->cpu migration
5038 */
5039 if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
5040 kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
5041 if (vcpu->cpu != cpu)
5042 kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
5043 vcpu->cpu = cpu;
5044 }
5045
5046 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
5047 }
5048
kvm_steal_time_set_preempted(struct kvm_vcpu * vcpu)5049 static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
5050 {
5051 struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
5052 struct kvm_steal_time __user *st;
5053 struct kvm_memslots *slots;
5054 static const u8 preempted = KVM_VCPU_PREEMPTED;
5055 gpa_t gpa = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;
5056
5057 /*
5058 * The vCPU can be marked preempted if and only if the VM-Exit was on
5059 * an instruction boundary and will not trigger guest emulation of any
5060 * kind (see vcpu_run). Vendor specific code controls (conservatively)
5061 * when this is true, for example allowing the vCPU to be marked
5062 * preempted if and only if the VM-Exit was due to a host interrupt.
5063 */
5064 if (!vcpu->arch.at_instruction_boundary) {
5065 vcpu->stat.preemption_other++;
5066 return;
5067 }
5068
5069 vcpu->stat.preemption_reported++;
5070 if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
5071 return;
5072
5073 if (vcpu->arch.st.preempted)
5074 return;
5075
5076 /* This happens on process exit */
5077 if (unlikely(current->mm != vcpu->kvm->mm))
5078 return;
5079
5080 slots = kvm_memslots(vcpu->kvm);
5081
5082 if (unlikely(slots->generation != ghc->generation ||
5083 gpa != ghc->gpa ||
5084 kvm_is_error_hva(ghc->hva) || !ghc->memslot))
5085 return;
5086
5087 st = (struct kvm_steal_time __user *)ghc->hva;
5088 BUILD_BUG_ON(sizeof(st->preempted) != sizeof(preempted));
5089
5090 if (!copy_to_user_nofault(&st->preempted, &preempted, sizeof(preempted)))
5091 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED;
5092
5093 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
5094 }
5095
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)5096 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
5097 {
5098 int idx;
5099
5100 if (vcpu->preempted) {
5101 vcpu->arch.preempted_in_kernel = kvm_arch_vcpu_in_kernel(vcpu);
5102
5103 /*
5104 * Take the srcu lock as memslots will be accessed to check the gfn
5105 * cache generation against the memslots generation.
5106 */
5107 idx = srcu_read_lock(&vcpu->kvm->srcu);
5108 if (kvm_xen_msr_enabled(vcpu->kvm))
5109 kvm_xen_runstate_set_preempted(vcpu);
5110 else
5111 kvm_steal_time_set_preempted(vcpu);
5112 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5113 }
5114
5115 static_call(kvm_x86_vcpu_put)(vcpu);
5116 vcpu->arch.last_host_tsc = rdtsc();
5117 }
5118
kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu * vcpu,struct kvm_lapic_state * s)5119 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
5120 struct kvm_lapic_state *s)
5121 {
5122 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
5123
5124 return kvm_apic_get_state(vcpu, s);
5125 }
5126
kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu * vcpu,struct kvm_lapic_state * s)5127 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
5128 struct kvm_lapic_state *s)
5129 {
5130 int r;
5131
5132 r = kvm_apic_set_state(vcpu, s);
5133 if (r)
5134 return r;
5135 update_cr8_intercept(vcpu);
5136
5137 return 0;
5138 }
5139
kvm_cpu_accept_dm_intr(struct kvm_vcpu * vcpu)5140 static int kvm_cpu_accept_dm_intr(struct kvm_vcpu *vcpu)
5141 {
5142 /*
5143 * We can accept userspace's request for interrupt injection
5144 * as long as we have a place to store the interrupt number.
5145 * The actual injection will happen when the CPU is able to
5146 * deliver the interrupt.
5147 */
5148 if (kvm_cpu_has_extint(vcpu))
5149 return false;
5150
5151 /* Acknowledging ExtINT does not happen if LINT0 is masked. */
5152 return (!lapic_in_kernel(vcpu) ||
5153 kvm_apic_accept_pic_intr(vcpu));
5154 }
5155
kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu * vcpu)5156 static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu)
5157 {
5158 /*
5159 * Do not cause an interrupt window exit if an exception
5160 * is pending or an event needs reinjection; userspace
5161 * might want to inject the interrupt manually using KVM_SET_REGS
5162 * or KVM_SET_SREGS. For that to work, we must be at an
5163 * instruction boundary and with no events half-injected.
5164 */
5165 return (kvm_arch_interrupt_allowed(vcpu) &&
5166 kvm_cpu_accept_dm_intr(vcpu) &&
5167 !kvm_event_needs_reinjection(vcpu) &&
5168 !kvm_is_exception_pending(vcpu));
5169 }
5170
kvm_vcpu_ioctl_interrupt(struct kvm_vcpu * vcpu,struct kvm_interrupt * irq)5171 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
5172 struct kvm_interrupt *irq)
5173 {
5174 if (irq->irq >= KVM_NR_INTERRUPTS)
5175 return -EINVAL;
5176
5177 if (!irqchip_in_kernel(vcpu->kvm)) {
5178 kvm_queue_interrupt(vcpu, irq->irq, false);
5179 kvm_make_request(KVM_REQ_EVENT, vcpu);
5180 return 0;
5181 }
5182
5183 /*
5184 * With in-kernel LAPIC, we only use this to inject EXTINT, so
5185 * fail for in-kernel 8259.
5186 */
5187 if (pic_in_kernel(vcpu->kvm))
5188 return -ENXIO;
5189
5190 if (vcpu->arch.pending_external_vector != -1)
5191 return -EEXIST;
5192
5193 vcpu->arch.pending_external_vector = irq->irq;
5194 kvm_make_request(KVM_REQ_EVENT, vcpu);
5195 return 0;
5196 }
5197
kvm_vcpu_ioctl_nmi(struct kvm_vcpu * vcpu)5198 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
5199 {
5200 kvm_inject_nmi(vcpu);
5201
5202 return 0;
5203 }
5204
vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu * vcpu,struct kvm_tpr_access_ctl * tac)5205 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
5206 struct kvm_tpr_access_ctl *tac)
5207 {
5208 if (tac->flags)
5209 return -EINVAL;
5210 vcpu->arch.tpr_access_reporting = !!tac->enabled;
5211 return 0;
5212 }
5213
kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu * vcpu,u64 mcg_cap)5214 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
5215 u64 mcg_cap)
5216 {
5217 int r;
5218 unsigned bank_num = mcg_cap & 0xff, bank;
5219
5220 r = -EINVAL;
5221 if (!bank_num || bank_num > KVM_MAX_MCE_BANKS)
5222 goto out;
5223 if (mcg_cap & ~(kvm_caps.supported_mce_cap | 0xff | 0xff0000))
5224 goto out;
5225 r = 0;
5226 vcpu->arch.mcg_cap = mcg_cap;
5227 /* Init IA32_MCG_CTL to all 1s */
5228 if (mcg_cap & MCG_CTL_P)
5229 vcpu->arch.mcg_ctl = ~(u64)0;
5230 /* Init IA32_MCi_CTL to all 1s, IA32_MCi_CTL2 to all 0s */
5231 for (bank = 0; bank < bank_num; bank++) {
5232 vcpu->arch.mce_banks[bank*4] = ~(u64)0;
5233 if (mcg_cap & MCG_CMCI_P)
5234 vcpu->arch.mci_ctl2_banks[bank] = 0;
5235 }
5236
5237 kvm_apic_after_set_mcg_cap(vcpu);
5238
5239 static_call(kvm_x86_setup_mce)(vcpu);
5240 out:
5241 return r;
5242 }
5243
5244 /*
5245 * Validate this is an UCNA (uncorrectable no action) error by checking the
5246 * MCG_STATUS and MCi_STATUS registers:
5247 * - none of the bits for Machine Check Exceptions are set
5248 * - both the VAL (valid) and UC (uncorrectable) bits are set
5249 * MCI_STATUS_PCC - Processor Context Corrupted
5250 * MCI_STATUS_S - Signaled as a Machine Check Exception
5251 * MCI_STATUS_AR - Software recoverable Action Required
5252 */
is_ucna(struct kvm_x86_mce * mce)5253 static bool is_ucna(struct kvm_x86_mce *mce)
5254 {
5255 return !mce->mcg_status &&
5256 !(mce->status & (MCI_STATUS_PCC | MCI_STATUS_S | MCI_STATUS_AR)) &&
5257 (mce->status & MCI_STATUS_VAL) &&
5258 (mce->status & MCI_STATUS_UC);
5259 }
5260
kvm_vcpu_x86_set_ucna(struct kvm_vcpu * vcpu,struct kvm_x86_mce * mce,u64 * banks)5261 static int kvm_vcpu_x86_set_ucna(struct kvm_vcpu *vcpu, struct kvm_x86_mce *mce, u64* banks)
5262 {
5263 u64 mcg_cap = vcpu->arch.mcg_cap;
5264
5265 banks[1] = mce->status;
5266 banks[2] = mce->addr;
5267 banks[3] = mce->misc;
5268 vcpu->arch.mcg_status = mce->mcg_status;
5269
5270 if (!(mcg_cap & MCG_CMCI_P) ||
5271 !(vcpu->arch.mci_ctl2_banks[mce->bank] & MCI_CTL2_CMCI_EN))
5272 return 0;
5273
5274 if (lapic_in_kernel(vcpu))
5275 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTCMCI);
5276
5277 return 0;
5278 }
5279
kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu * vcpu,struct kvm_x86_mce * mce)5280 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
5281 struct kvm_x86_mce *mce)
5282 {
5283 u64 mcg_cap = vcpu->arch.mcg_cap;
5284 unsigned bank_num = mcg_cap & 0xff;
5285 u64 *banks = vcpu->arch.mce_banks;
5286
5287 if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
5288 return -EINVAL;
5289
5290 banks += array_index_nospec(4 * mce->bank, 4 * bank_num);
5291
5292 if (is_ucna(mce))
5293 return kvm_vcpu_x86_set_ucna(vcpu, mce, banks);
5294
5295 /*
5296 * if IA32_MCG_CTL is not all 1s, the uncorrected error
5297 * reporting is disabled
5298 */
5299 if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
5300 vcpu->arch.mcg_ctl != ~(u64)0)
5301 return 0;
5302 /*
5303 * if IA32_MCi_CTL is not all 1s, the uncorrected error
5304 * reporting is disabled for the bank
5305 */
5306 if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
5307 return 0;
5308 if (mce->status & MCI_STATUS_UC) {
5309 if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
5310 !kvm_is_cr4_bit_set(vcpu, X86_CR4_MCE)) {
5311 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5312 return 0;
5313 }
5314 if (banks[1] & MCI_STATUS_VAL)
5315 mce->status |= MCI_STATUS_OVER;
5316 banks[2] = mce->addr;
5317 banks[3] = mce->misc;
5318 vcpu->arch.mcg_status = mce->mcg_status;
5319 banks[1] = mce->status;
5320 kvm_queue_exception(vcpu, MC_VECTOR);
5321 } else if (!(banks[1] & MCI_STATUS_VAL)
5322 || !(banks[1] & MCI_STATUS_UC)) {
5323 if (banks[1] & MCI_STATUS_VAL)
5324 mce->status |= MCI_STATUS_OVER;
5325 banks[2] = mce->addr;
5326 banks[3] = mce->misc;
5327 banks[1] = mce->status;
5328 } else
5329 banks[1] |= MCI_STATUS_OVER;
5330 return 0;
5331 }
5332
kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)5333 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
5334 struct kvm_vcpu_events *events)
5335 {
5336 struct kvm_queued_exception *ex;
5337
5338 process_nmi(vcpu);
5339
5340 #ifdef CONFIG_KVM_SMM
5341 if (kvm_check_request(KVM_REQ_SMI, vcpu))
5342 process_smi(vcpu);
5343 #endif
5344
5345 /*
5346 * KVM's ABI only allows for one exception to be migrated. Luckily,
5347 * the only time there can be two queued exceptions is if there's a
5348 * non-exiting _injected_ exception, and a pending exiting exception.
5349 * In that case, ignore the VM-Exiting exception as it's an extension
5350 * of the injected exception.
5351 */
5352 if (vcpu->arch.exception_vmexit.pending &&
5353 !vcpu->arch.exception.pending &&
5354 !vcpu->arch.exception.injected)
5355 ex = &vcpu->arch.exception_vmexit;
5356 else
5357 ex = &vcpu->arch.exception;
5358
5359 /*
5360 * In guest mode, payload delivery should be deferred if the exception
5361 * will be intercepted by L1, e.g. KVM should not modifying CR2 if L1
5362 * intercepts #PF, ditto for DR6 and #DBs. If the per-VM capability,
5363 * KVM_CAP_EXCEPTION_PAYLOAD, is not set, userspace may or may not
5364 * propagate the payload and so it cannot be safely deferred. Deliver
5365 * the payload if the capability hasn't been requested.
5366 */
5367 if (!vcpu->kvm->arch.exception_payload_enabled &&
5368 ex->pending && ex->has_payload)
5369 kvm_deliver_exception_payload(vcpu, ex);
5370
5371 memset(events, 0, sizeof(*events));
5372
5373 /*
5374 * The API doesn't provide the instruction length for software
5375 * exceptions, so don't report them. As long as the guest RIP
5376 * isn't advanced, we should expect to encounter the exception
5377 * again.
5378 */
5379 if (!kvm_exception_is_soft(ex->vector)) {
5380 events->exception.injected = ex->injected;
5381 events->exception.pending = ex->pending;
5382 /*
5383 * For ABI compatibility, deliberately conflate
5384 * pending and injected exceptions when
5385 * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled.
5386 */
5387 if (!vcpu->kvm->arch.exception_payload_enabled)
5388 events->exception.injected |= ex->pending;
5389 }
5390 events->exception.nr = ex->vector;
5391 events->exception.has_error_code = ex->has_error_code;
5392 events->exception.error_code = ex->error_code;
5393 events->exception_has_payload = ex->has_payload;
5394 events->exception_payload = ex->payload;
5395
5396 events->interrupt.injected =
5397 vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft;
5398 events->interrupt.nr = vcpu->arch.interrupt.nr;
5399 events->interrupt.shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
5400
5401 events->nmi.injected = vcpu->arch.nmi_injected;
5402 events->nmi.pending = kvm_get_nr_pending_nmis(vcpu);
5403 events->nmi.masked = static_call(kvm_x86_get_nmi_mask)(vcpu);
5404
5405 /* events->sipi_vector is never valid when reporting to user space */
5406
5407 #ifdef CONFIG_KVM_SMM
5408 events->smi.smm = is_smm(vcpu);
5409 events->smi.pending = vcpu->arch.smi_pending;
5410 events->smi.smm_inside_nmi =
5411 !!(vcpu->arch.hflags & HF_SMM_INSIDE_NMI_MASK);
5412 #endif
5413 events->smi.latched_init = kvm_lapic_latched_init(vcpu);
5414
5415 events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
5416 | KVM_VCPUEVENT_VALID_SHADOW
5417 | KVM_VCPUEVENT_VALID_SMM);
5418 if (vcpu->kvm->arch.exception_payload_enabled)
5419 events->flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
5420 if (vcpu->kvm->arch.triple_fault_event) {
5421 events->triple_fault.pending = kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5422 events->flags |= KVM_VCPUEVENT_VALID_TRIPLE_FAULT;
5423 }
5424 }
5425
kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu * vcpu,struct kvm_vcpu_events * events)5426 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
5427 struct kvm_vcpu_events *events)
5428 {
5429 if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
5430 | KVM_VCPUEVENT_VALID_SIPI_VECTOR
5431 | KVM_VCPUEVENT_VALID_SHADOW
5432 | KVM_VCPUEVENT_VALID_SMM
5433 | KVM_VCPUEVENT_VALID_PAYLOAD
5434 | KVM_VCPUEVENT_VALID_TRIPLE_FAULT))
5435 return -EINVAL;
5436
5437 if (events->flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
5438 if (!vcpu->kvm->arch.exception_payload_enabled)
5439 return -EINVAL;
5440 if (events->exception.pending)
5441 events->exception.injected = 0;
5442 else
5443 events->exception_has_payload = 0;
5444 } else {
5445 events->exception.pending = 0;
5446 events->exception_has_payload = 0;
5447 }
5448
5449 if ((events->exception.injected || events->exception.pending) &&
5450 (events->exception.nr > 31 || events->exception.nr == NMI_VECTOR))
5451 return -EINVAL;
5452
5453 /* INITs are latched while in SMM */
5454 if (events->flags & KVM_VCPUEVENT_VALID_SMM &&
5455 (events->smi.smm || events->smi.pending) &&
5456 vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED)
5457 return -EINVAL;
5458
5459 process_nmi(vcpu);
5460
5461 /*
5462 * Flag that userspace is stuffing an exception, the next KVM_RUN will
5463 * morph the exception to a VM-Exit if appropriate. Do this only for
5464 * pending exceptions, already-injected exceptions are not subject to
5465 * intercpetion. Note, userspace that conflates pending and injected
5466 * is hosed, and will incorrectly convert an injected exception into a
5467 * pending exception, which in turn may cause a spurious VM-Exit.
5468 */
5469 vcpu->arch.exception_from_userspace = events->exception.pending;
5470
5471 vcpu->arch.exception_vmexit.pending = false;
5472
5473 vcpu->arch.exception.injected = events->exception.injected;
5474 vcpu->arch.exception.pending = events->exception.pending;
5475 vcpu->arch.exception.vector = events->exception.nr;
5476 vcpu->arch.exception.has_error_code = events->exception.has_error_code;
5477 vcpu->arch.exception.error_code = events->exception.error_code;
5478 vcpu->arch.exception.has_payload = events->exception_has_payload;
5479 vcpu->arch.exception.payload = events->exception_payload;
5480
5481 vcpu->arch.interrupt.injected = events->interrupt.injected;
5482 vcpu->arch.interrupt.nr = events->interrupt.nr;
5483 vcpu->arch.interrupt.soft = events->interrupt.soft;
5484 if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
5485 static_call(kvm_x86_set_interrupt_shadow)(vcpu,
5486 events->interrupt.shadow);
5487
5488 vcpu->arch.nmi_injected = events->nmi.injected;
5489 if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING) {
5490 vcpu->arch.nmi_pending = 0;
5491 atomic_set(&vcpu->arch.nmi_queued, events->nmi.pending);
5492 if (events->nmi.pending)
5493 kvm_make_request(KVM_REQ_NMI, vcpu);
5494 }
5495 static_call(kvm_x86_set_nmi_mask)(vcpu, events->nmi.masked);
5496
5497 if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR &&
5498 lapic_in_kernel(vcpu))
5499 vcpu->arch.apic->sipi_vector = events->sipi_vector;
5500
5501 if (events->flags & KVM_VCPUEVENT_VALID_SMM) {
5502 #ifdef CONFIG_KVM_SMM
5503 if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) {
5504 kvm_leave_nested(vcpu);
5505 kvm_smm_changed(vcpu, events->smi.smm);
5506 }
5507
5508 vcpu->arch.smi_pending = events->smi.pending;
5509
5510 if (events->smi.smm) {
5511 if (events->smi.smm_inside_nmi)
5512 vcpu->arch.hflags |= HF_SMM_INSIDE_NMI_MASK;
5513 else
5514 vcpu->arch.hflags &= ~HF_SMM_INSIDE_NMI_MASK;
5515 }
5516
5517 #else
5518 if (events->smi.smm || events->smi.pending ||
5519 events->smi.smm_inside_nmi)
5520 return -EINVAL;
5521 #endif
5522
5523 if (lapic_in_kernel(vcpu)) {
5524 if (events->smi.latched_init)
5525 set_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5526 else
5527 clear_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
5528 }
5529 }
5530
5531 if (events->flags & KVM_VCPUEVENT_VALID_TRIPLE_FAULT) {
5532 if (!vcpu->kvm->arch.triple_fault_event)
5533 return -EINVAL;
5534 if (events->triple_fault.pending)
5535 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5536 else
5537 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
5538 }
5539
5540 kvm_make_request(KVM_REQ_EVENT, vcpu);
5541
5542 return 0;
5543 }
5544
kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu * vcpu,struct kvm_debugregs * dbgregs)5545 static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
5546 struct kvm_debugregs *dbgregs)
5547 {
5548 unsigned int i;
5549
5550 if (vcpu->kvm->arch.has_protected_state &&
5551 vcpu->arch.guest_state_protected)
5552 return -EINVAL;
5553
5554 memset(dbgregs, 0, sizeof(*dbgregs));
5555
5556 BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db));
5557 for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++)
5558 dbgregs->db[i] = vcpu->arch.db[i];
5559
5560 dbgregs->dr6 = vcpu->arch.dr6;
5561 dbgregs->dr7 = vcpu->arch.dr7;
5562 return 0;
5563 }
5564
kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu * vcpu,struct kvm_debugregs * dbgregs)5565 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
5566 struct kvm_debugregs *dbgregs)
5567 {
5568 unsigned int i;
5569
5570 if (vcpu->kvm->arch.has_protected_state &&
5571 vcpu->arch.guest_state_protected)
5572 return -EINVAL;
5573
5574 if (dbgregs->flags)
5575 return -EINVAL;
5576
5577 if (!kvm_dr6_valid(dbgregs->dr6))
5578 return -EINVAL;
5579 if (!kvm_dr7_valid(dbgregs->dr7))
5580 return -EINVAL;
5581
5582 for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++)
5583 vcpu->arch.db[i] = dbgregs->db[i];
5584
5585 kvm_update_dr0123(vcpu);
5586 vcpu->arch.dr6 = dbgregs->dr6;
5587 vcpu->arch.dr7 = dbgregs->dr7;
5588 kvm_update_dr7(vcpu);
5589
5590 return 0;
5591 }
5592
5593
kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu * vcpu,u8 * state,unsigned int size)5594 static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu,
5595 u8 *state, unsigned int size)
5596 {
5597 /*
5598 * Only copy state for features that are enabled for the guest. The
5599 * state itself isn't problematic, but setting bits in the header for
5600 * features that are supported in *this* host but not exposed to the
5601 * guest can result in KVM_SET_XSAVE failing when live migrating to a
5602 * compatible host without the features that are NOT exposed to the
5603 * guest.
5604 *
5605 * FP+SSE can always be saved/restored via KVM_{G,S}ET_XSAVE, even if
5606 * XSAVE/XCRO are not exposed to the guest, and even if XSAVE isn't
5607 * supported by the host.
5608 */
5609 u64 supported_xcr0 = vcpu->arch.guest_supported_xcr0 |
5610 XFEATURE_MASK_FPSSE;
5611
5612 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5613 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
5614
5615 fpu_copy_guest_fpstate_to_uabi(&vcpu->arch.guest_fpu, state, size,
5616 supported_xcr0, vcpu->arch.pkru);
5617 return 0;
5618 }
5619
kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu * vcpu,struct kvm_xsave * guest_xsave)5620 static int kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu,
5621 struct kvm_xsave *guest_xsave)
5622 {
5623 return kvm_vcpu_ioctl_x86_get_xsave2(vcpu, (void *)guest_xsave->region,
5624 sizeof(guest_xsave->region));
5625 }
5626
kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu * vcpu,struct kvm_xsave * guest_xsave)5627 static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu,
5628 struct kvm_xsave *guest_xsave)
5629 {
5630 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
5631 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
5632
5633 return fpu_copy_uabi_to_guest_fpstate(&vcpu->arch.guest_fpu,
5634 guest_xsave->region,
5635 kvm_caps.supported_xcr0,
5636 &vcpu->arch.pkru);
5637 }
5638
kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu * vcpu,struct kvm_xcrs * guest_xcrs)5639 static int kvm_vcpu_ioctl_x86_get_xcrs(struct kvm_vcpu *vcpu,
5640 struct kvm_xcrs *guest_xcrs)
5641 {
5642 if (vcpu->kvm->arch.has_protected_state &&
5643 vcpu->arch.guest_state_protected)
5644 return -EINVAL;
5645
5646 if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
5647 guest_xcrs->nr_xcrs = 0;
5648 return 0;
5649 }
5650
5651 guest_xcrs->nr_xcrs = 1;
5652 guest_xcrs->flags = 0;
5653 guest_xcrs->xcrs[0].xcr = XCR_XFEATURE_ENABLED_MASK;
5654 guest_xcrs->xcrs[0].value = vcpu->arch.xcr0;
5655 return 0;
5656 }
5657
kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu * vcpu,struct kvm_xcrs * guest_xcrs)5658 static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
5659 struct kvm_xcrs *guest_xcrs)
5660 {
5661 int i, r = 0;
5662
5663 if (vcpu->kvm->arch.has_protected_state &&
5664 vcpu->arch.guest_state_protected)
5665 return -EINVAL;
5666
5667 if (!boot_cpu_has(X86_FEATURE_XSAVE))
5668 return -EINVAL;
5669
5670 if (guest_xcrs->nr_xcrs > KVM_MAX_XCRS || guest_xcrs->flags)
5671 return -EINVAL;
5672
5673 for (i = 0; i < guest_xcrs->nr_xcrs; i++)
5674 /* Only support XCR0 currently */
5675 if (guest_xcrs->xcrs[i].xcr == XCR_XFEATURE_ENABLED_MASK) {
5676 r = __kvm_set_xcr(vcpu, XCR_XFEATURE_ENABLED_MASK,
5677 guest_xcrs->xcrs[i].value);
5678 break;
5679 }
5680 if (r)
5681 r = -EINVAL;
5682 return r;
5683 }
5684
5685 /*
5686 * kvm_set_guest_paused() indicates to the guest kernel that it has been
5687 * stopped by the hypervisor. This function will be called from the host only.
5688 * EINVAL is returned when the host attempts to set the flag for a guest that
5689 * does not support pv clocks.
5690 */
kvm_set_guest_paused(struct kvm_vcpu * vcpu)5691 static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
5692 {
5693 if (!vcpu->arch.pv_time.active)
5694 return -EINVAL;
5695 vcpu->arch.pvclock_set_guest_stopped_request = true;
5696 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
5697 return 0;
5698 }
5699
kvm_arch_tsc_has_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)5700 static int kvm_arch_tsc_has_attr(struct kvm_vcpu *vcpu,
5701 struct kvm_device_attr *attr)
5702 {
5703 int r;
5704
5705 switch (attr->attr) {
5706 case KVM_VCPU_TSC_OFFSET:
5707 r = 0;
5708 break;
5709 default:
5710 r = -ENXIO;
5711 }
5712
5713 return r;
5714 }
5715
kvm_arch_tsc_get_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)5716 static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,
5717 struct kvm_device_attr *attr)
5718 {
5719 u64 __user *uaddr = u64_to_user_ptr(attr->addr);
5720 int r;
5721
5722 switch (attr->attr) {
5723 case KVM_VCPU_TSC_OFFSET:
5724 r = -EFAULT;
5725 if (put_user(vcpu->arch.l1_tsc_offset, uaddr))
5726 break;
5727 r = 0;
5728 break;
5729 default:
5730 r = -ENXIO;
5731 }
5732
5733 return r;
5734 }
5735
kvm_arch_tsc_set_attr(struct kvm_vcpu * vcpu,struct kvm_device_attr * attr)5736 static int kvm_arch_tsc_set_attr(struct kvm_vcpu *vcpu,
5737 struct kvm_device_attr *attr)
5738 {
5739 u64 __user *uaddr = u64_to_user_ptr(attr->addr);
5740 struct kvm *kvm = vcpu->kvm;
5741 int r;
5742
5743 switch (attr->attr) {
5744 case KVM_VCPU_TSC_OFFSET: {
5745 u64 offset, tsc, ns;
5746 unsigned long flags;
5747 bool matched;
5748
5749 r = -EFAULT;
5750 if (get_user(offset, uaddr))
5751 break;
5752
5753 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
5754
5755 matched = (vcpu->arch.virtual_tsc_khz &&
5756 kvm->arch.last_tsc_khz == vcpu->arch.virtual_tsc_khz &&
5757 kvm->arch.last_tsc_offset == offset);
5758
5759 tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio) + offset;
5760 ns = get_kvmclock_base_ns();
5761
5762 kvm->arch.user_set_tsc = true;
5763 __kvm_synchronize_tsc(vcpu, offset, tsc, ns, matched);
5764 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
5765
5766 r = 0;
5767 break;
5768 }
5769 default:
5770 r = -ENXIO;
5771 }
5772
5773 return r;
5774 }
5775
kvm_vcpu_ioctl_device_attr(struct kvm_vcpu * vcpu,unsigned int ioctl,void __user * argp)5776 static int kvm_vcpu_ioctl_device_attr(struct kvm_vcpu *vcpu,
5777 unsigned int ioctl,
5778 void __user *argp)
5779 {
5780 struct kvm_device_attr attr;
5781 int r;
5782
5783 if (copy_from_user(&attr, argp, sizeof(attr)))
5784 return -EFAULT;
5785
5786 if (attr.group != KVM_VCPU_TSC_CTRL)
5787 return -ENXIO;
5788
5789 switch (ioctl) {
5790 case KVM_HAS_DEVICE_ATTR:
5791 r = kvm_arch_tsc_has_attr(vcpu, &attr);
5792 break;
5793 case KVM_GET_DEVICE_ATTR:
5794 r = kvm_arch_tsc_get_attr(vcpu, &attr);
5795 break;
5796 case KVM_SET_DEVICE_ATTR:
5797 r = kvm_arch_tsc_set_attr(vcpu, &attr);
5798 break;
5799 }
5800
5801 return r;
5802 }
5803
kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu * vcpu,struct kvm_enable_cap * cap)5804 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5805 struct kvm_enable_cap *cap)
5806 {
5807 if (cap->flags)
5808 return -EINVAL;
5809
5810 switch (cap->cap) {
5811 #ifdef CONFIG_KVM_HYPERV
5812 case KVM_CAP_HYPERV_SYNIC2:
5813 if (cap->args[0])
5814 return -EINVAL;
5815 fallthrough;
5816
5817 case KVM_CAP_HYPERV_SYNIC:
5818 if (!irqchip_in_kernel(vcpu->kvm))
5819 return -EINVAL;
5820 return kvm_hv_activate_synic(vcpu, cap->cap ==
5821 KVM_CAP_HYPERV_SYNIC2);
5822 case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
5823 {
5824 int r;
5825 uint16_t vmcs_version;
5826 void __user *user_ptr;
5827
5828 if (!kvm_x86_ops.nested_ops->enable_evmcs)
5829 return -ENOTTY;
5830 r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version);
5831 if (!r) {
5832 user_ptr = (void __user *)(uintptr_t)cap->args[0];
5833 if (copy_to_user(user_ptr, &vmcs_version,
5834 sizeof(vmcs_version)))
5835 r = -EFAULT;
5836 }
5837 return r;
5838 }
5839 case KVM_CAP_HYPERV_DIRECT_TLBFLUSH:
5840 if (!kvm_x86_ops.enable_l2_tlb_flush)
5841 return -ENOTTY;
5842
5843 return static_call(kvm_x86_enable_l2_tlb_flush)(vcpu);
5844
5845 case KVM_CAP_HYPERV_ENFORCE_CPUID:
5846 return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]);
5847 #endif
5848
5849 case KVM_CAP_ENFORCE_PV_FEATURE_CPUID:
5850 vcpu->arch.pv_cpuid.enforce = cap->args[0];
5851 if (vcpu->arch.pv_cpuid.enforce)
5852 kvm_update_pv_runtime(vcpu);
5853
5854 return 0;
5855 default:
5856 return -EINVAL;
5857 }
5858 }
5859
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5860 long kvm_arch_vcpu_ioctl(struct file *filp,
5861 unsigned int ioctl, unsigned long arg)
5862 {
5863 struct kvm_vcpu *vcpu = filp->private_data;
5864 void __user *argp = (void __user *)arg;
5865 int r;
5866 union {
5867 struct kvm_sregs2 *sregs2;
5868 struct kvm_lapic_state *lapic;
5869 struct kvm_xsave *xsave;
5870 struct kvm_xcrs *xcrs;
5871 void *buffer;
5872 } u;
5873
5874 vcpu_load(vcpu);
5875
5876 u.buffer = NULL;
5877 switch (ioctl) {
5878 case KVM_GET_LAPIC: {
5879 r = -EINVAL;
5880 if (!lapic_in_kernel(vcpu))
5881 goto out;
5882 u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
5883 GFP_KERNEL_ACCOUNT);
5884
5885 r = -ENOMEM;
5886 if (!u.lapic)
5887 goto out;
5888 r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
5889 if (r)
5890 goto out;
5891 r = -EFAULT;
5892 if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
5893 goto out;
5894 r = 0;
5895 break;
5896 }
5897 case KVM_SET_LAPIC: {
5898 r = -EINVAL;
5899 if (!lapic_in_kernel(vcpu))
5900 goto out;
5901 u.lapic = memdup_user(argp, sizeof(*u.lapic));
5902 if (IS_ERR(u.lapic)) {
5903 r = PTR_ERR(u.lapic);
5904 goto out_nofree;
5905 }
5906
5907 r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
5908 break;
5909 }
5910 case KVM_INTERRUPT: {
5911 struct kvm_interrupt irq;
5912
5913 r = -EFAULT;
5914 if (copy_from_user(&irq, argp, sizeof(irq)))
5915 goto out;
5916 r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
5917 break;
5918 }
5919 case KVM_NMI: {
5920 r = kvm_vcpu_ioctl_nmi(vcpu);
5921 break;
5922 }
5923 case KVM_SMI: {
5924 r = kvm_inject_smi(vcpu);
5925 break;
5926 }
5927 case KVM_SET_CPUID: {
5928 struct kvm_cpuid __user *cpuid_arg = argp;
5929 struct kvm_cpuid cpuid;
5930
5931 r = -EFAULT;
5932 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5933 goto out;
5934 r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
5935 break;
5936 }
5937 case KVM_SET_CPUID2: {
5938 struct kvm_cpuid2 __user *cpuid_arg = argp;
5939 struct kvm_cpuid2 cpuid;
5940
5941 r = -EFAULT;
5942 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5943 goto out;
5944 r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
5945 cpuid_arg->entries);
5946 break;
5947 }
5948 case KVM_GET_CPUID2: {
5949 struct kvm_cpuid2 __user *cpuid_arg = argp;
5950 struct kvm_cpuid2 cpuid;
5951
5952 r = -EFAULT;
5953 if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
5954 goto out;
5955 r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
5956 cpuid_arg->entries);
5957 if (r)
5958 goto out;
5959 r = -EFAULT;
5960 if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
5961 goto out;
5962 r = 0;
5963 break;
5964 }
5965 case KVM_GET_MSRS: {
5966 int idx = srcu_read_lock(&vcpu->kvm->srcu);
5967 r = msr_io(vcpu, argp, do_get_msr, 1);
5968 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5969 break;
5970 }
5971 case KVM_SET_MSRS: {
5972 int idx = srcu_read_lock(&vcpu->kvm->srcu);
5973 r = msr_io(vcpu, argp, do_set_msr, 0);
5974 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5975 break;
5976 }
5977 case KVM_TPR_ACCESS_REPORTING: {
5978 struct kvm_tpr_access_ctl tac;
5979
5980 r = -EFAULT;
5981 if (copy_from_user(&tac, argp, sizeof(tac)))
5982 goto out;
5983 r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
5984 if (r)
5985 goto out;
5986 r = -EFAULT;
5987 if (copy_to_user(argp, &tac, sizeof(tac)))
5988 goto out;
5989 r = 0;
5990 break;
5991 };
5992 case KVM_SET_VAPIC_ADDR: {
5993 struct kvm_vapic_addr va;
5994 int idx;
5995
5996 r = -EINVAL;
5997 if (!lapic_in_kernel(vcpu))
5998 goto out;
5999 r = -EFAULT;
6000 if (copy_from_user(&va, argp, sizeof(va)))
6001 goto out;
6002 idx = srcu_read_lock(&vcpu->kvm->srcu);
6003 r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
6004 srcu_read_unlock(&vcpu->kvm->srcu, idx);
6005 break;
6006 }
6007 case KVM_X86_SETUP_MCE: {
6008 u64 mcg_cap;
6009
6010 r = -EFAULT;
6011 if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
6012 goto out;
6013 r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
6014 break;
6015 }
6016 case KVM_X86_SET_MCE: {
6017 struct kvm_x86_mce mce;
6018
6019 r = -EFAULT;
6020 if (copy_from_user(&mce, argp, sizeof(mce)))
6021 goto out;
6022 r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
6023 break;
6024 }
6025 case KVM_GET_VCPU_EVENTS: {
6026 struct kvm_vcpu_events events;
6027
6028 kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
6029
6030 r = -EFAULT;
6031 if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
6032 break;
6033 r = 0;
6034 break;
6035 }
6036 case KVM_SET_VCPU_EVENTS: {
6037 struct kvm_vcpu_events events;
6038
6039 r = -EFAULT;
6040 if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
6041 break;
6042
6043 r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
6044 break;
6045 }
6046 case KVM_GET_DEBUGREGS: {
6047 struct kvm_debugregs dbgregs;
6048
6049 r = kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
6050 if (r < 0)
6051 break;
6052
6053 r = -EFAULT;
6054 if (copy_to_user(argp, &dbgregs,
6055 sizeof(struct kvm_debugregs)))
6056 break;
6057 r = 0;
6058 break;
6059 }
6060 case KVM_SET_DEBUGREGS: {
6061 struct kvm_debugregs dbgregs;
6062
6063 r = -EFAULT;
6064 if (copy_from_user(&dbgregs, argp,
6065 sizeof(struct kvm_debugregs)))
6066 break;
6067
6068 r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
6069 break;
6070 }
6071 case KVM_GET_XSAVE: {
6072 r = -EINVAL;
6073 if (vcpu->arch.guest_fpu.uabi_size > sizeof(struct kvm_xsave))
6074 break;
6075
6076 u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
6077 r = -ENOMEM;
6078 if (!u.xsave)
6079 break;
6080
6081 r = kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);
6082 if (r < 0)
6083 break;
6084
6085 r = -EFAULT;
6086 if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
6087 break;
6088 r = 0;
6089 break;
6090 }
6091 case KVM_SET_XSAVE: {
6092 int size = vcpu->arch.guest_fpu.uabi_size;
6093
6094 u.xsave = memdup_user(argp, size);
6095 if (IS_ERR(u.xsave)) {
6096 r = PTR_ERR(u.xsave);
6097 goto out_nofree;
6098 }
6099
6100 r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
6101 break;
6102 }
6103
6104 case KVM_GET_XSAVE2: {
6105 int size = vcpu->arch.guest_fpu.uabi_size;
6106
6107 u.xsave = kzalloc(size, GFP_KERNEL_ACCOUNT);
6108 r = -ENOMEM;
6109 if (!u.xsave)
6110 break;
6111
6112 r = kvm_vcpu_ioctl_x86_get_xsave2(vcpu, u.buffer, size);
6113 if (r < 0)
6114 break;
6115
6116 r = -EFAULT;
6117 if (copy_to_user(argp, u.xsave, size))
6118 break;
6119
6120 r = 0;
6121 break;
6122 }
6123
6124 case KVM_GET_XCRS: {
6125 u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
6126 r = -ENOMEM;
6127 if (!u.xcrs)
6128 break;
6129
6130 r = kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);
6131 if (r < 0)
6132 break;
6133
6134 r = -EFAULT;
6135 if (copy_to_user(argp, u.xcrs,
6136 sizeof(struct kvm_xcrs)))
6137 break;
6138 r = 0;
6139 break;
6140 }
6141 case KVM_SET_XCRS: {
6142 u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
6143 if (IS_ERR(u.xcrs)) {
6144 r = PTR_ERR(u.xcrs);
6145 goto out_nofree;
6146 }
6147
6148 r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
6149 break;
6150 }
6151 case KVM_SET_TSC_KHZ: {
6152 u32 user_tsc_khz;
6153
6154 r = -EINVAL;
6155 user_tsc_khz = (u32)arg;
6156
6157 if (kvm_caps.has_tsc_control &&
6158 user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
6159 goto out;
6160
6161 if (user_tsc_khz == 0)
6162 user_tsc_khz = tsc_khz;
6163
6164 if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
6165 r = 0;
6166
6167 goto out;
6168 }
6169 case KVM_GET_TSC_KHZ: {
6170 r = vcpu->arch.virtual_tsc_khz;
6171 goto out;
6172 }
6173 case KVM_KVMCLOCK_CTRL: {
6174 r = kvm_set_guest_paused(vcpu);
6175 goto out;
6176 }
6177 case KVM_ENABLE_CAP: {
6178 struct kvm_enable_cap cap;
6179
6180 r = -EFAULT;
6181 if (copy_from_user(&cap, argp, sizeof(cap)))
6182 goto out;
6183 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
6184 break;
6185 }
6186 case KVM_GET_NESTED_STATE: {
6187 struct kvm_nested_state __user *user_kvm_nested_state = argp;
6188 u32 user_data_size;
6189
6190 r = -EINVAL;
6191 if (!kvm_x86_ops.nested_ops->get_state)
6192 break;
6193
6194 BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
6195 r = -EFAULT;
6196 if (get_user(user_data_size, &user_kvm_nested_state->size))
6197 break;
6198
6199 r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
6200 user_data_size);
6201 if (r < 0)
6202 break;
6203
6204 if (r > user_data_size) {
6205 if (put_user(r, &user_kvm_nested_state->size))
6206 r = -EFAULT;
6207 else
6208 r = -E2BIG;
6209 break;
6210 }
6211
6212 r = 0;
6213 break;
6214 }
6215 case KVM_SET_NESTED_STATE: {
6216 struct kvm_nested_state __user *user_kvm_nested_state = argp;
6217 struct kvm_nested_state kvm_state;
6218 int idx;
6219
6220 r = -EINVAL;
6221 if (!kvm_x86_ops.nested_ops->set_state)
6222 break;
6223
6224 r = -EFAULT;
6225 if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
6226 break;
6227
6228 r = -EINVAL;
6229 if (kvm_state.size < sizeof(kvm_state))
6230 break;
6231
6232 if (kvm_state.flags &
6233 ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
6234 | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING
6235 | KVM_STATE_NESTED_GIF_SET))
6236 break;
6237
6238 /* nested_run_pending implies guest_mode. */
6239 if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
6240 && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
6241 break;
6242
6243 idx = srcu_read_lock(&vcpu->kvm->srcu);
6244 r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
6245 srcu_read_unlock(&vcpu->kvm->srcu, idx);
6246 break;
6247 }
6248 #ifdef CONFIG_KVM_HYPERV
6249 case KVM_GET_SUPPORTED_HV_CPUID:
6250 r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);
6251 break;
6252 #endif
6253 #ifdef CONFIG_KVM_XEN
6254 case KVM_XEN_VCPU_GET_ATTR: {
6255 struct kvm_xen_vcpu_attr xva;
6256
6257 r = -EFAULT;
6258 if (copy_from_user(&xva, argp, sizeof(xva)))
6259 goto out;
6260 r = kvm_xen_vcpu_get_attr(vcpu, &xva);
6261 if (!r && copy_to_user(argp, &xva, sizeof(xva)))
6262 r = -EFAULT;
6263 break;
6264 }
6265 case KVM_XEN_VCPU_SET_ATTR: {
6266 struct kvm_xen_vcpu_attr xva;
6267
6268 r = -EFAULT;
6269 if (copy_from_user(&xva, argp, sizeof(xva)))
6270 goto out;
6271 r = kvm_xen_vcpu_set_attr(vcpu, &xva);
6272 break;
6273 }
6274 #endif
6275 case KVM_GET_SREGS2: {
6276 r = -EINVAL;
6277 if (vcpu->kvm->arch.has_protected_state &&
6278 vcpu->arch.guest_state_protected)
6279 goto out;
6280
6281 u.sregs2 = kzalloc(sizeof(struct kvm_sregs2), GFP_KERNEL);
6282 r = -ENOMEM;
6283 if (!u.sregs2)
6284 goto out;
6285 __get_sregs2(vcpu, u.sregs2);
6286 r = -EFAULT;
6287 if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2)))
6288 goto out;
6289 r = 0;
6290 break;
6291 }
6292 case KVM_SET_SREGS2: {
6293 r = -EINVAL;
6294 if (vcpu->kvm->arch.has_protected_state &&
6295 vcpu->arch.guest_state_protected)
6296 goto out;
6297
6298 u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2));
6299 if (IS_ERR(u.sregs2)) {
6300 r = PTR_ERR(u.sregs2);
6301 u.sregs2 = NULL;
6302 goto out;
6303 }
6304 r = __set_sregs2(vcpu, u.sregs2);
6305 break;
6306 }
6307 case KVM_HAS_DEVICE_ATTR:
6308 case KVM_GET_DEVICE_ATTR:
6309 case KVM_SET_DEVICE_ATTR:
6310 r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);
6311 break;
6312 default:
6313 r = -EINVAL;
6314 }
6315 out:
6316 kfree(u.buffer);
6317 out_nofree:
6318 vcpu_put(vcpu);
6319 return r;
6320 }
6321
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)6322 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
6323 {
6324 return VM_FAULT_SIGBUS;
6325 }
6326
kvm_vm_ioctl_set_tss_addr(struct kvm * kvm,unsigned long addr)6327 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
6328 {
6329 int ret;
6330
6331 if (addr > (unsigned int)(-3 * PAGE_SIZE))
6332 return -EINVAL;
6333 ret = static_call(kvm_x86_set_tss_addr)(kvm, addr);
6334 return ret;
6335 }
6336
kvm_vm_ioctl_set_identity_map_addr(struct kvm * kvm,u64 ident_addr)6337 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
6338 u64 ident_addr)
6339 {
6340 return static_call(kvm_x86_set_identity_map_addr)(kvm, ident_addr);
6341 }
6342
kvm_vm_ioctl_set_nr_mmu_pages(struct kvm * kvm,unsigned long kvm_nr_mmu_pages)6343 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
6344 unsigned long kvm_nr_mmu_pages)
6345 {
6346 if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
6347 return -EINVAL;
6348
6349 mutex_lock(&kvm->slots_lock);
6350
6351 kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
6352 kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
6353
6354 mutex_unlock(&kvm->slots_lock);
6355 return 0;
6356 }
6357
kvm_vm_ioctl_get_irqchip(struct kvm * kvm,struct kvm_irqchip * chip)6358 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
6359 {
6360 struct kvm_pic *pic = kvm->arch.vpic;
6361 int r;
6362
6363 r = 0;
6364 switch (chip->chip_id) {
6365 case KVM_IRQCHIP_PIC_MASTER:
6366 memcpy(&chip->chip.pic, &pic->pics[0],
6367 sizeof(struct kvm_pic_state));
6368 break;
6369 case KVM_IRQCHIP_PIC_SLAVE:
6370 memcpy(&chip->chip.pic, &pic->pics[1],
6371 sizeof(struct kvm_pic_state));
6372 break;
6373 case KVM_IRQCHIP_IOAPIC:
6374 kvm_get_ioapic(kvm, &chip->chip.ioapic);
6375 break;
6376 default:
6377 r = -EINVAL;
6378 break;
6379 }
6380 return r;
6381 }
6382
kvm_vm_ioctl_set_irqchip(struct kvm * kvm,struct kvm_irqchip * chip)6383 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
6384 {
6385 struct kvm_pic *pic = kvm->arch.vpic;
6386 int r;
6387
6388 r = 0;
6389 switch (chip->chip_id) {
6390 case KVM_IRQCHIP_PIC_MASTER:
6391 spin_lock(&pic->lock);
6392 memcpy(&pic->pics[0], &chip->chip.pic,
6393 sizeof(struct kvm_pic_state));
6394 spin_unlock(&pic->lock);
6395 break;
6396 case KVM_IRQCHIP_PIC_SLAVE:
6397 spin_lock(&pic->lock);
6398 memcpy(&pic->pics[1], &chip->chip.pic,
6399 sizeof(struct kvm_pic_state));
6400 spin_unlock(&pic->lock);
6401 break;
6402 case KVM_IRQCHIP_IOAPIC:
6403 kvm_set_ioapic(kvm, &chip->chip.ioapic);
6404 break;
6405 default:
6406 r = -EINVAL;
6407 break;
6408 }
6409 kvm_pic_update_irq(pic);
6410 return r;
6411 }
6412
kvm_vm_ioctl_get_pit(struct kvm * kvm,struct kvm_pit_state * ps)6413 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
6414 {
6415 struct kvm_kpit_state *kps = &kvm->arch.vpit->pit_state;
6416
6417 BUILD_BUG_ON(sizeof(*ps) != sizeof(kps->channels));
6418
6419 mutex_lock(&kps->lock);
6420 memcpy(ps, &kps->channels, sizeof(*ps));
6421 mutex_unlock(&kps->lock);
6422 return 0;
6423 }
6424
kvm_vm_ioctl_set_pit(struct kvm * kvm,struct kvm_pit_state * ps)6425 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
6426 {
6427 int i;
6428 struct kvm_pit *pit = kvm->arch.vpit;
6429
6430 mutex_lock(&pit->pit_state.lock);
6431 memcpy(&pit->pit_state.channels, ps, sizeof(*ps));
6432 for (i = 0; i < 3; i++)
6433 kvm_pit_load_count(pit, i, ps->channels[i].count, 0);
6434 mutex_unlock(&pit->pit_state.lock);
6435 return 0;
6436 }
6437
kvm_vm_ioctl_get_pit2(struct kvm * kvm,struct kvm_pit_state2 * ps)6438 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
6439 {
6440 mutex_lock(&kvm->arch.vpit->pit_state.lock);
6441 memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
6442 sizeof(ps->channels));
6443 ps->flags = kvm->arch.vpit->pit_state.flags;
6444 mutex_unlock(&kvm->arch.vpit->pit_state.lock);
6445 memset(&ps->reserved, 0, sizeof(ps->reserved));
6446 return 0;
6447 }
6448
kvm_vm_ioctl_set_pit2(struct kvm * kvm,struct kvm_pit_state2 * ps)6449 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
6450 {
6451 int start = 0;
6452 int i;
6453 u32 prev_legacy, cur_legacy;
6454 struct kvm_pit *pit = kvm->arch.vpit;
6455
6456 mutex_lock(&pit->pit_state.lock);
6457 prev_legacy = pit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
6458 cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
6459 if (!prev_legacy && cur_legacy)
6460 start = 1;
6461 memcpy(&pit->pit_state.channels, &ps->channels,
6462 sizeof(pit->pit_state.channels));
6463 pit->pit_state.flags = ps->flags;
6464 for (i = 0; i < 3; i++)
6465 kvm_pit_load_count(pit, i, pit->pit_state.channels[i].count,
6466 start && i == 0);
6467 mutex_unlock(&pit->pit_state.lock);
6468 return 0;
6469 }
6470
kvm_vm_ioctl_reinject(struct kvm * kvm,struct kvm_reinject_control * control)6471 static int kvm_vm_ioctl_reinject(struct kvm *kvm,
6472 struct kvm_reinject_control *control)
6473 {
6474 struct kvm_pit *pit = kvm->arch.vpit;
6475
6476 /* pit->pit_state.lock was overloaded to prevent userspace from getting
6477 * an inconsistent state after running multiple KVM_REINJECT_CONTROL
6478 * ioctls in parallel. Use a separate lock if that ioctl isn't rare.
6479 */
6480 mutex_lock(&pit->pit_state.lock);
6481 kvm_pit_set_reinject(pit, control->pit_reinject);
6482 mutex_unlock(&pit->pit_state.lock);
6483
6484 return 0;
6485 }
6486
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)6487 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
6488 {
6489
6490 /*
6491 * Flush all CPUs' dirty log buffers to the dirty_bitmap. Called
6492 * before reporting dirty_bitmap to userspace. KVM flushes the buffers
6493 * on all VM-Exits, thus we only need to kick running vCPUs to force a
6494 * VM-Exit.
6495 */
6496 struct kvm_vcpu *vcpu;
6497 unsigned long i;
6498
6499 if (!kvm_x86_ops.cpu_dirty_log_size)
6500 return;
6501
6502 kvm_for_each_vcpu(i, vcpu, kvm)
6503 kvm_vcpu_kick(vcpu);
6504 }
6505
kvm_vm_ioctl_irq_line(struct kvm * kvm,struct kvm_irq_level * irq_event,bool line_status)6506 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
6507 bool line_status)
6508 {
6509 if (!irqchip_in_kernel(kvm))
6510 return -ENXIO;
6511
6512 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
6513 irq_event->irq, irq_event->level,
6514 line_status);
6515 return 0;
6516 }
6517
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)6518 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
6519 struct kvm_enable_cap *cap)
6520 {
6521 int r;
6522
6523 if (cap->flags)
6524 return -EINVAL;
6525
6526 switch (cap->cap) {
6527 case KVM_CAP_DISABLE_QUIRKS2:
6528 r = -EINVAL;
6529 if (cap->args[0] & ~KVM_X86_VALID_QUIRKS)
6530 break;
6531 fallthrough;
6532 case KVM_CAP_DISABLE_QUIRKS:
6533 kvm->arch.disabled_quirks = cap->args[0];
6534 r = 0;
6535 break;
6536 case KVM_CAP_SPLIT_IRQCHIP: {
6537 mutex_lock(&kvm->lock);
6538 r = -EINVAL;
6539 if (cap->args[0] > MAX_NR_RESERVED_IOAPIC_PINS)
6540 goto split_irqchip_unlock;
6541 r = -EEXIST;
6542 if (irqchip_in_kernel(kvm))
6543 goto split_irqchip_unlock;
6544 if (kvm->created_vcpus)
6545 goto split_irqchip_unlock;
6546 r = kvm_setup_empty_irq_routing(kvm);
6547 if (r)
6548 goto split_irqchip_unlock;
6549 /* Pairs with irqchip_in_kernel. */
6550 smp_wmb();
6551 kvm->arch.irqchip_mode = KVM_IRQCHIP_SPLIT;
6552 kvm->arch.nr_reserved_ioapic_pins = cap->args[0];
6553 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT);
6554 r = 0;
6555 split_irqchip_unlock:
6556 mutex_unlock(&kvm->lock);
6557 break;
6558 }
6559 case KVM_CAP_X2APIC_API:
6560 r = -EINVAL;
6561 if (cap->args[0] & ~KVM_X2APIC_API_VALID_FLAGS)
6562 break;
6563
6564 if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_IDS)
6565 kvm->arch.x2apic_format = true;
6566 if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
6567 kvm->arch.x2apic_broadcast_quirk_disabled = true;
6568
6569 r = 0;
6570 break;
6571 case KVM_CAP_X86_DISABLE_EXITS:
6572 r = -EINVAL;
6573 if (cap->args[0] & ~KVM_X86_DISABLE_VALID_EXITS)
6574 break;
6575
6576 if (cap->args[0] & KVM_X86_DISABLE_EXITS_PAUSE)
6577 kvm->arch.pause_in_guest = true;
6578
6579 #define SMT_RSB_MSG "This processor is affected by the Cross-Thread Return Predictions vulnerability. " \
6580 "KVM_CAP_X86_DISABLE_EXITS should only be used with SMT disabled or trusted guests."
6581
6582 if (!mitigate_smt_rsb) {
6583 if (boot_cpu_has_bug(X86_BUG_SMT_RSB) && cpu_smt_possible() &&
6584 (cap->args[0] & ~KVM_X86_DISABLE_EXITS_PAUSE))
6585 pr_warn_once(SMT_RSB_MSG);
6586
6587 if ((cap->args[0] & KVM_X86_DISABLE_EXITS_MWAIT) &&
6588 kvm_can_mwait_in_guest())
6589 kvm->arch.mwait_in_guest = true;
6590 if (cap->args[0] & KVM_X86_DISABLE_EXITS_HLT)
6591 kvm->arch.hlt_in_guest = true;
6592 if (cap->args[0] & KVM_X86_DISABLE_EXITS_CSTATE)
6593 kvm->arch.cstate_in_guest = true;
6594 }
6595
6596 r = 0;
6597 break;
6598 case KVM_CAP_MSR_PLATFORM_INFO:
6599 kvm->arch.guest_can_read_msr_platform_info = cap->args[0];
6600 r = 0;
6601 break;
6602 case KVM_CAP_EXCEPTION_PAYLOAD:
6603 kvm->arch.exception_payload_enabled = cap->args[0];
6604 r = 0;
6605 break;
6606 case KVM_CAP_X86_TRIPLE_FAULT_EVENT:
6607 kvm->arch.triple_fault_event = cap->args[0];
6608 r = 0;
6609 break;
6610 case KVM_CAP_X86_USER_SPACE_MSR:
6611 r = -EINVAL;
6612 if (cap->args[0] & ~KVM_MSR_EXIT_REASON_VALID_MASK)
6613 break;
6614 kvm->arch.user_space_msr_mask = cap->args[0];
6615 r = 0;
6616 break;
6617 case KVM_CAP_X86_BUS_LOCK_EXIT:
6618 r = -EINVAL;
6619 if (cap->args[0] & ~KVM_BUS_LOCK_DETECTION_VALID_MODE)
6620 break;
6621
6622 if ((cap->args[0] & KVM_BUS_LOCK_DETECTION_OFF) &&
6623 (cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT))
6624 break;
6625
6626 if (kvm_caps.has_bus_lock_exit &&
6627 cap->args[0] & KVM_BUS_LOCK_DETECTION_EXIT)
6628 kvm->arch.bus_lock_detection_enabled = true;
6629 r = 0;
6630 break;
6631 #ifdef CONFIG_X86_SGX_KVM
6632 case KVM_CAP_SGX_ATTRIBUTE: {
6633 unsigned long allowed_attributes = 0;
6634
6635 r = sgx_set_attribute(&allowed_attributes, cap->args[0]);
6636 if (r)
6637 break;
6638
6639 /* KVM only supports the PROVISIONKEY privileged attribute. */
6640 if ((allowed_attributes & SGX_ATTR_PROVISIONKEY) &&
6641 !(allowed_attributes & ~SGX_ATTR_PROVISIONKEY))
6642 kvm->arch.sgx_provisioning_allowed = true;
6643 else
6644 r = -EINVAL;
6645 break;
6646 }
6647 #endif
6648 case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM:
6649 r = -EINVAL;
6650 if (!kvm_x86_ops.vm_copy_enc_context_from)
6651 break;
6652
6653 r = static_call(kvm_x86_vm_copy_enc_context_from)(kvm, cap->args[0]);
6654 break;
6655 case KVM_CAP_VM_MOVE_ENC_CONTEXT_FROM:
6656 r = -EINVAL;
6657 if (!kvm_x86_ops.vm_move_enc_context_from)
6658 break;
6659
6660 r = static_call(kvm_x86_vm_move_enc_context_from)(kvm, cap->args[0]);
6661 break;
6662 case KVM_CAP_EXIT_HYPERCALL:
6663 if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) {
6664 r = -EINVAL;
6665 break;
6666 }
6667 kvm->arch.hypercall_exit_enabled = cap->args[0];
6668 r = 0;
6669 break;
6670 case KVM_CAP_EXIT_ON_EMULATION_FAILURE:
6671 r = -EINVAL;
6672 if (cap->args[0] & ~1)
6673 break;
6674 kvm->arch.exit_on_emulation_error = cap->args[0];
6675 r = 0;
6676 break;
6677 case KVM_CAP_PMU_CAPABILITY:
6678 r = -EINVAL;
6679 if (!enable_pmu || (cap->args[0] & ~KVM_CAP_PMU_VALID_MASK))
6680 break;
6681
6682 mutex_lock(&kvm->lock);
6683 if (!kvm->created_vcpus) {
6684 kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE);
6685 r = 0;
6686 }
6687 mutex_unlock(&kvm->lock);
6688 break;
6689 case KVM_CAP_MAX_VCPU_ID:
6690 r = -EINVAL;
6691 if (cap->args[0] > KVM_MAX_VCPU_IDS)
6692 break;
6693
6694 mutex_lock(&kvm->lock);
6695 if (kvm->arch.max_vcpu_ids == cap->args[0]) {
6696 r = 0;
6697 } else if (!kvm->arch.max_vcpu_ids) {
6698 kvm->arch.max_vcpu_ids = cap->args[0];
6699 r = 0;
6700 }
6701 mutex_unlock(&kvm->lock);
6702 break;
6703 case KVM_CAP_X86_NOTIFY_VMEXIT:
6704 r = -EINVAL;
6705 if ((u32)cap->args[0] & ~KVM_X86_NOTIFY_VMEXIT_VALID_BITS)
6706 break;
6707 if (!kvm_caps.has_notify_vmexit)
6708 break;
6709 if (!((u32)cap->args[0] & KVM_X86_NOTIFY_VMEXIT_ENABLED))
6710 break;
6711 mutex_lock(&kvm->lock);
6712 if (!kvm->created_vcpus) {
6713 kvm->arch.notify_window = cap->args[0] >> 32;
6714 kvm->arch.notify_vmexit_flags = (u32)cap->args[0];
6715 r = 0;
6716 }
6717 mutex_unlock(&kvm->lock);
6718 break;
6719 case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES:
6720 r = -EINVAL;
6721
6722 /*
6723 * Since the risk of disabling NX hugepages is a guest crashing
6724 * the system, ensure the userspace process has permission to
6725 * reboot the system.
6726 *
6727 * Note that unlike the reboot() syscall, the process must have
6728 * this capability in the root namespace because exposing
6729 * /dev/kvm into a container does not limit the scope of the
6730 * iTLB multihit bug to that container. In other words,
6731 * this must use capable(), not ns_capable().
6732 */
6733 if (!capable(CAP_SYS_BOOT)) {
6734 r = -EPERM;
6735 break;
6736 }
6737
6738 if (cap->args[0])
6739 break;
6740
6741 mutex_lock(&kvm->lock);
6742 if (!kvm->created_vcpus) {
6743 kvm->arch.disable_nx_huge_pages = true;
6744 r = 0;
6745 }
6746 mutex_unlock(&kvm->lock);
6747 break;
6748 default:
6749 r = -EINVAL;
6750 break;
6751 }
6752 return r;
6753 }
6754
kvm_alloc_msr_filter(bool default_allow)6755 static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow)
6756 {
6757 struct kvm_x86_msr_filter *msr_filter;
6758
6759 msr_filter = kzalloc(sizeof(*msr_filter), GFP_KERNEL_ACCOUNT);
6760 if (!msr_filter)
6761 return NULL;
6762
6763 msr_filter->default_allow = default_allow;
6764 return msr_filter;
6765 }
6766
kvm_free_msr_filter(struct kvm_x86_msr_filter * msr_filter)6767 static void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter)
6768 {
6769 u32 i;
6770
6771 if (!msr_filter)
6772 return;
6773
6774 for (i = 0; i < msr_filter->count; i++)
6775 kfree(msr_filter->ranges[i].bitmap);
6776
6777 kfree(msr_filter);
6778 }
6779
kvm_add_msr_filter(struct kvm_x86_msr_filter * msr_filter,struct kvm_msr_filter_range * user_range)6780 static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter,
6781 struct kvm_msr_filter_range *user_range)
6782 {
6783 unsigned long *bitmap;
6784 size_t bitmap_size;
6785
6786 if (!user_range->nmsrs)
6787 return 0;
6788
6789 if (user_range->flags & ~KVM_MSR_FILTER_RANGE_VALID_MASK)
6790 return -EINVAL;
6791
6792 if (!user_range->flags)
6793 return -EINVAL;
6794
6795 bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long);
6796 if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE)
6797 return -EINVAL;
6798
6799 bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size);
6800 if (IS_ERR(bitmap))
6801 return PTR_ERR(bitmap);
6802
6803 msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) {
6804 .flags = user_range->flags,
6805 .base = user_range->base,
6806 .nmsrs = user_range->nmsrs,
6807 .bitmap = bitmap,
6808 };
6809
6810 msr_filter->count++;
6811 return 0;
6812 }
6813
kvm_vm_ioctl_set_msr_filter(struct kvm * kvm,struct kvm_msr_filter * filter)6814 static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm,
6815 struct kvm_msr_filter *filter)
6816 {
6817 struct kvm_x86_msr_filter *new_filter, *old_filter;
6818 bool default_allow;
6819 bool empty = true;
6820 int r;
6821 u32 i;
6822
6823 if (filter->flags & ~KVM_MSR_FILTER_VALID_MASK)
6824 return -EINVAL;
6825
6826 for (i = 0; i < ARRAY_SIZE(filter->ranges); i++)
6827 empty &= !filter->ranges[i].nmsrs;
6828
6829 default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY);
6830 if (empty && !default_allow)
6831 return -EINVAL;
6832
6833 new_filter = kvm_alloc_msr_filter(default_allow);
6834 if (!new_filter)
6835 return -ENOMEM;
6836
6837 for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) {
6838 r = kvm_add_msr_filter(new_filter, &filter->ranges[i]);
6839 if (r) {
6840 kvm_free_msr_filter(new_filter);
6841 return r;
6842 }
6843 }
6844
6845 mutex_lock(&kvm->lock);
6846 old_filter = rcu_replace_pointer(kvm->arch.msr_filter, new_filter,
6847 mutex_is_locked(&kvm->lock));
6848 mutex_unlock(&kvm->lock);
6849 synchronize_srcu(&kvm->srcu);
6850
6851 kvm_free_msr_filter(old_filter);
6852
6853 kvm_make_all_cpus_request(kvm, KVM_REQ_MSR_FILTER_CHANGED);
6854
6855 return 0;
6856 }
6857
6858 #ifdef CONFIG_KVM_COMPAT
6859 /* for KVM_X86_SET_MSR_FILTER */
6860 struct kvm_msr_filter_range_compat {
6861 __u32 flags;
6862 __u32 nmsrs;
6863 __u32 base;
6864 __u32 bitmap;
6865 };
6866
6867 struct kvm_msr_filter_compat {
6868 __u32 flags;
6869 struct kvm_msr_filter_range_compat ranges[KVM_MSR_FILTER_MAX_RANGES];
6870 };
6871
6872 #define KVM_X86_SET_MSR_FILTER_COMPAT _IOW(KVMIO, 0xc6, struct kvm_msr_filter_compat)
6873
kvm_arch_vm_compat_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)6874 long kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl,
6875 unsigned long arg)
6876 {
6877 void __user *argp = (void __user *)arg;
6878 struct kvm *kvm = filp->private_data;
6879 long r = -ENOTTY;
6880
6881 switch (ioctl) {
6882 case KVM_X86_SET_MSR_FILTER_COMPAT: {
6883 struct kvm_msr_filter __user *user_msr_filter = argp;
6884 struct kvm_msr_filter_compat filter_compat;
6885 struct kvm_msr_filter filter;
6886 int i;
6887
6888 if (copy_from_user(&filter_compat, user_msr_filter,
6889 sizeof(filter_compat)))
6890 return -EFAULT;
6891
6892 filter.flags = filter_compat.flags;
6893 for (i = 0; i < ARRAY_SIZE(filter.ranges); i++) {
6894 struct kvm_msr_filter_range_compat *cr;
6895
6896 cr = &filter_compat.ranges[i];
6897 filter.ranges[i] = (struct kvm_msr_filter_range) {
6898 .flags = cr->flags,
6899 .nmsrs = cr->nmsrs,
6900 .base = cr->base,
6901 .bitmap = (__u8 *)(ulong)cr->bitmap,
6902 };
6903 }
6904
6905 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter);
6906 break;
6907 }
6908 }
6909
6910 return r;
6911 }
6912 #endif
6913
6914 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
kvm_arch_suspend_notifier(struct kvm * kvm)6915 static int kvm_arch_suspend_notifier(struct kvm *kvm)
6916 {
6917 struct kvm_vcpu *vcpu;
6918 unsigned long i;
6919 int ret = 0;
6920
6921 mutex_lock(&kvm->lock);
6922 kvm_for_each_vcpu(i, vcpu, kvm) {
6923 if (!vcpu->arch.pv_time.active)
6924 continue;
6925
6926 ret = kvm_set_guest_paused(vcpu);
6927 if (ret) {
6928 kvm_err("Failed to pause guest VCPU%d: %d\n",
6929 vcpu->vcpu_id, ret);
6930 break;
6931 }
6932 }
6933 mutex_unlock(&kvm->lock);
6934
6935 return ret ? NOTIFY_BAD : NOTIFY_DONE;
6936 }
6937
kvm_arch_pm_notifier(struct kvm * kvm,unsigned long state)6938 int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state)
6939 {
6940 switch (state) {
6941 case PM_HIBERNATION_PREPARE:
6942 case PM_SUSPEND_PREPARE:
6943 return kvm_arch_suspend_notifier(kvm);
6944 }
6945
6946 return NOTIFY_DONE;
6947 }
6948 #endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
6949
kvm_vm_ioctl_get_clock(struct kvm * kvm,void __user * argp)6950 static int kvm_vm_ioctl_get_clock(struct kvm *kvm, void __user *argp)
6951 {
6952 struct kvm_clock_data data = { 0 };
6953
6954 get_kvmclock(kvm, &data);
6955 if (copy_to_user(argp, &data, sizeof(data)))
6956 return -EFAULT;
6957
6958 return 0;
6959 }
6960
kvm_vm_ioctl_set_clock(struct kvm * kvm,void __user * argp)6961 static int kvm_vm_ioctl_set_clock(struct kvm *kvm, void __user *argp)
6962 {
6963 struct kvm_arch *ka = &kvm->arch;
6964 struct kvm_clock_data data;
6965 u64 now_raw_ns;
6966
6967 if (copy_from_user(&data, argp, sizeof(data)))
6968 return -EFAULT;
6969
6970 /*
6971 * Only KVM_CLOCK_REALTIME is used, but allow passing the
6972 * result of KVM_GET_CLOCK back to KVM_SET_CLOCK.
6973 */
6974 if (data.flags & ~KVM_CLOCK_VALID_FLAGS)
6975 return -EINVAL;
6976
6977 kvm_hv_request_tsc_page_update(kvm);
6978 kvm_start_pvclock_update(kvm);
6979 pvclock_update_vm_gtod_copy(kvm);
6980
6981 /*
6982 * This pairs with kvm_guest_time_update(): when masterclock is
6983 * in use, we use master_kernel_ns + kvmclock_offset to set
6984 * unsigned 'system_time' so if we use get_kvmclock_ns() (which
6985 * is slightly ahead) here we risk going negative on unsigned
6986 * 'system_time' when 'data.clock' is very small.
6987 */
6988 if (data.flags & KVM_CLOCK_REALTIME) {
6989 u64 now_real_ns = ktime_get_real_ns();
6990
6991 /*
6992 * Avoid stepping the kvmclock backwards.
6993 */
6994 if (now_real_ns > data.realtime)
6995 data.clock += now_real_ns - data.realtime;
6996 }
6997
6998 if (ka->use_master_clock)
6999 now_raw_ns = ka->master_kernel_ns;
7000 else
7001 now_raw_ns = get_kvmclock_base_ns();
7002 ka->kvmclock_offset = data.clock - now_raw_ns;
7003 kvm_end_pvclock_update(kvm);
7004 return 0;
7005 }
7006
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)7007 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
7008 {
7009 struct kvm *kvm = filp->private_data;
7010 void __user *argp = (void __user *)arg;
7011 int r = -ENOTTY;
7012 /*
7013 * This union makes it completely explicit to gcc-3.x
7014 * that these two variables' stack usage should be
7015 * combined, not added together.
7016 */
7017 union {
7018 struct kvm_pit_state ps;
7019 struct kvm_pit_state2 ps2;
7020 struct kvm_pit_config pit_config;
7021 } u;
7022
7023 switch (ioctl) {
7024 case KVM_SET_TSS_ADDR:
7025 r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
7026 break;
7027 case KVM_SET_IDENTITY_MAP_ADDR: {
7028 u64 ident_addr;
7029
7030 mutex_lock(&kvm->lock);
7031 r = -EINVAL;
7032 if (kvm->created_vcpus)
7033 goto set_identity_unlock;
7034 r = -EFAULT;
7035 if (copy_from_user(&ident_addr, argp, sizeof(ident_addr)))
7036 goto set_identity_unlock;
7037 r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
7038 set_identity_unlock:
7039 mutex_unlock(&kvm->lock);
7040 break;
7041 }
7042 case KVM_SET_NR_MMU_PAGES:
7043 r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
7044 break;
7045 case KVM_CREATE_IRQCHIP: {
7046 mutex_lock(&kvm->lock);
7047
7048 r = -EEXIST;
7049 if (irqchip_in_kernel(kvm))
7050 goto create_irqchip_unlock;
7051
7052 r = -EINVAL;
7053 if (kvm->created_vcpus)
7054 goto create_irqchip_unlock;
7055
7056 r = kvm_pic_init(kvm);
7057 if (r)
7058 goto create_irqchip_unlock;
7059
7060 r = kvm_ioapic_init(kvm);
7061 if (r) {
7062 kvm_pic_destroy(kvm);
7063 goto create_irqchip_unlock;
7064 }
7065
7066 r = kvm_setup_default_irq_routing(kvm);
7067 if (r) {
7068 kvm_ioapic_destroy(kvm);
7069 kvm_pic_destroy(kvm);
7070 goto create_irqchip_unlock;
7071 }
7072 /* Write kvm->irq_routing before enabling irqchip_in_kernel. */
7073 smp_wmb();
7074 kvm->arch.irqchip_mode = KVM_IRQCHIP_KERNEL;
7075 kvm_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_ABSENT);
7076 create_irqchip_unlock:
7077 mutex_unlock(&kvm->lock);
7078 break;
7079 }
7080 case KVM_CREATE_PIT:
7081 u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
7082 goto create_pit;
7083 case KVM_CREATE_PIT2:
7084 r = -EFAULT;
7085 if (copy_from_user(&u.pit_config, argp,
7086 sizeof(struct kvm_pit_config)))
7087 goto out;
7088 create_pit:
7089 mutex_lock(&kvm->lock);
7090 r = -EEXIST;
7091 if (kvm->arch.vpit)
7092 goto create_pit_unlock;
7093 r = -ENOENT;
7094 if (!pic_in_kernel(kvm))
7095 goto create_pit_unlock;
7096 r = -ENOMEM;
7097 kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
7098 if (kvm->arch.vpit)
7099 r = 0;
7100 create_pit_unlock:
7101 mutex_unlock(&kvm->lock);
7102 break;
7103 case KVM_GET_IRQCHIP: {
7104 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
7105 struct kvm_irqchip *chip;
7106
7107 chip = memdup_user(argp, sizeof(*chip));
7108 if (IS_ERR(chip)) {
7109 r = PTR_ERR(chip);
7110 goto out;
7111 }
7112
7113 r = -ENXIO;
7114 if (!irqchip_kernel(kvm))
7115 goto get_irqchip_out;
7116 r = kvm_vm_ioctl_get_irqchip(kvm, chip);
7117 if (r)
7118 goto get_irqchip_out;
7119 r = -EFAULT;
7120 if (copy_to_user(argp, chip, sizeof(*chip)))
7121 goto get_irqchip_out;
7122 r = 0;
7123 get_irqchip_out:
7124 kfree(chip);
7125 break;
7126 }
7127 case KVM_SET_IRQCHIP: {
7128 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
7129 struct kvm_irqchip *chip;
7130
7131 chip = memdup_user(argp, sizeof(*chip));
7132 if (IS_ERR(chip)) {
7133 r = PTR_ERR(chip);
7134 goto out;
7135 }
7136
7137 r = -ENXIO;
7138 if (!irqchip_kernel(kvm))
7139 goto set_irqchip_out;
7140 r = kvm_vm_ioctl_set_irqchip(kvm, chip);
7141 set_irqchip_out:
7142 kfree(chip);
7143 break;
7144 }
7145 case KVM_GET_PIT: {
7146 r = -EFAULT;
7147 if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
7148 goto out;
7149 r = -ENXIO;
7150 if (!kvm->arch.vpit)
7151 goto out;
7152 r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
7153 if (r)
7154 goto out;
7155 r = -EFAULT;
7156 if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
7157 goto out;
7158 r = 0;
7159 break;
7160 }
7161 case KVM_SET_PIT: {
7162 r = -EFAULT;
7163 if (copy_from_user(&u.ps, argp, sizeof(u.ps)))
7164 goto out;
7165 mutex_lock(&kvm->lock);
7166 r = -ENXIO;
7167 if (!kvm->arch.vpit)
7168 goto set_pit_out;
7169 r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
7170 set_pit_out:
7171 mutex_unlock(&kvm->lock);
7172 break;
7173 }
7174 case KVM_GET_PIT2: {
7175 r = -ENXIO;
7176 if (!kvm->arch.vpit)
7177 goto out;
7178 r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
7179 if (r)
7180 goto out;
7181 r = -EFAULT;
7182 if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
7183 goto out;
7184 r = 0;
7185 break;
7186 }
7187 case KVM_SET_PIT2: {
7188 r = -EFAULT;
7189 if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
7190 goto out;
7191 mutex_lock(&kvm->lock);
7192 r = -ENXIO;
7193 if (!kvm->arch.vpit)
7194 goto set_pit2_out;
7195 r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
7196 set_pit2_out:
7197 mutex_unlock(&kvm->lock);
7198 break;
7199 }
7200 case KVM_REINJECT_CONTROL: {
7201 struct kvm_reinject_control control;
7202 r = -EFAULT;
7203 if (copy_from_user(&control, argp, sizeof(control)))
7204 goto out;
7205 r = -ENXIO;
7206 if (!kvm->arch.vpit)
7207 goto out;
7208 r = kvm_vm_ioctl_reinject(kvm, &control);
7209 break;
7210 }
7211 case KVM_SET_BOOT_CPU_ID:
7212 r = 0;
7213 mutex_lock(&kvm->lock);
7214 if (kvm->created_vcpus)
7215 r = -EBUSY;
7216 else
7217 kvm->arch.bsp_vcpu_id = arg;
7218 mutex_unlock(&kvm->lock);
7219 break;
7220 #ifdef CONFIG_KVM_XEN
7221 case KVM_XEN_HVM_CONFIG: {
7222 struct kvm_xen_hvm_config xhc;
7223 r = -EFAULT;
7224 if (copy_from_user(&xhc, argp, sizeof(xhc)))
7225 goto out;
7226 r = kvm_xen_hvm_config(kvm, &xhc);
7227 break;
7228 }
7229 case KVM_XEN_HVM_GET_ATTR: {
7230 struct kvm_xen_hvm_attr xha;
7231
7232 r = -EFAULT;
7233 if (copy_from_user(&xha, argp, sizeof(xha)))
7234 goto out;
7235 r = kvm_xen_hvm_get_attr(kvm, &xha);
7236 if (!r && copy_to_user(argp, &xha, sizeof(xha)))
7237 r = -EFAULT;
7238 break;
7239 }
7240 case KVM_XEN_HVM_SET_ATTR: {
7241 struct kvm_xen_hvm_attr xha;
7242
7243 r = -EFAULT;
7244 if (copy_from_user(&xha, argp, sizeof(xha)))
7245 goto out;
7246 r = kvm_xen_hvm_set_attr(kvm, &xha);
7247 break;
7248 }
7249 case KVM_XEN_HVM_EVTCHN_SEND: {
7250 struct kvm_irq_routing_xen_evtchn uxe;
7251
7252 r = -EFAULT;
7253 if (copy_from_user(&uxe, argp, sizeof(uxe)))
7254 goto out;
7255 r = kvm_xen_hvm_evtchn_send(kvm, &uxe);
7256 break;
7257 }
7258 #endif
7259 case KVM_SET_CLOCK:
7260 r = kvm_vm_ioctl_set_clock(kvm, argp);
7261 break;
7262 case KVM_GET_CLOCK:
7263 r = kvm_vm_ioctl_get_clock(kvm, argp);
7264 break;
7265 case KVM_SET_TSC_KHZ: {
7266 u32 user_tsc_khz;
7267
7268 r = -EINVAL;
7269 user_tsc_khz = (u32)arg;
7270
7271 if (kvm_caps.has_tsc_control &&
7272 user_tsc_khz >= kvm_caps.max_guest_tsc_khz)
7273 goto out;
7274
7275 if (user_tsc_khz == 0)
7276 user_tsc_khz = tsc_khz;
7277
7278 WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz);
7279 r = 0;
7280
7281 goto out;
7282 }
7283 case KVM_GET_TSC_KHZ: {
7284 r = READ_ONCE(kvm->arch.default_tsc_khz);
7285 goto out;
7286 }
7287 case KVM_MEMORY_ENCRYPT_OP: {
7288 r = -ENOTTY;
7289 if (!kvm_x86_ops.mem_enc_ioctl)
7290 goto out;
7291
7292 r = static_call(kvm_x86_mem_enc_ioctl)(kvm, argp);
7293 break;
7294 }
7295 case KVM_MEMORY_ENCRYPT_REG_REGION: {
7296 struct kvm_enc_region region;
7297
7298 r = -EFAULT;
7299 if (copy_from_user(®ion, argp, sizeof(region)))
7300 goto out;
7301
7302 r = -ENOTTY;
7303 if (!kvm_x86_ops.mem_enc_register_region)
7304 goto out;
7305
7306 r = static_call(kvm_x86_mem_enc_register_region)(kvm, ®ion);
7307 break;
7308 }
7309 case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
7310 struct kvm_enc_region region;
7311
7312 r = -EFAULT;
7313 if (copy_from_user(®ion, argp, sizeof(region)))
7314 goto out;
7315
7316 r = -ENOTTY;
7317 if (!kvm_x86_ops.mem_enc_unregister_region)
7318 goto out;
7319
7320 r = static_call(kvm_x86_mem_enc_unregister_region)(kvm, ®ion);
7321 break;
7322 }
7323 #ifdef CONFIG_KVM_HYPERV
7324 case KVM_HYPERV_EVENTFD: {
7325 struct kvm_hyperv_eventfd hvevfd;
7326
7327 r = -EFAULT;
7328 if (copy_from_user(&hvevfd, argp, sizeof(hvevfd)))
7329 goto out;
7330 r = kvm_vm_ioctl_hv_eventfd(kvm, &hvevfd);
7331 break;
7332 }
7333 #endif
7334 case KVM_SET_PMU_EVENT_FILTER:
7335 r = kvm_vm_ioctl_set_pmu_event_filter(kvm, argp);
7336 break;
7337 case KVM_X86_SET_MSR_FILTER: {
7338 struct kvm_msr_filter __user *user_msr_filter = argp;
7339 struct kvm_msr_filter filter;
7340
7341 if (copy_from_user(&filter, user_msr_filter, sizeof(filter)))
7342 return -EFAULT;
7343
7344 r = kvm_vm_ioctl_set_msr_filter(kvm, &filter);
7345 break;
7346 }
7347 default:
7348 r = -ENOTTY;
7349 }
7350 out:
7351 return r;
7352 }
7353
kvm_probe_feature_msr(u32 msr_index)7354 static void kvm_probe_feature_msr(u32 msr_index)
7355 {
7356 struct kvm_msr_entry msr = {
7357 .index = msr_index,
7358 };
7359
7360 if (kvm_get_msr_feature(&msr))
7361 return;
7362
7363 msr_based_features[num_msr_based_features++] = msr_index;
7364 }
7365
kvm_probe_msr_to_save(u32 msr_index)7366 static void kvm_probe_msr_to_save(u32 msr_index)
7367 {
7368 u32 dummy[2];
7369
7370 if (rdmsr_safe(msr_index, &dummy[0], &dummy[1]))
7371 return;
7372
7373 /*
7374 * Even MSRs that are valid in the host may not be exposed to guests in
7375 * some cases.
7376 */
7377 switch (msr_index) {
7378 case MSR_IA32_BNDCFGS:
7379 if (!kvm_mpx_supported())
7380 return;
7381 break;
7382 case MSR_TSC_AUX:
7383 if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) &&
7384 !kvm_cpu_cap_has(X86_FEATURE_RDPID))
7385 return;
7386 break;
7387 case MSR_IA32_UMWAIT_CONTROL:
7388 if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG))
7389 return;
7390 break;
7391 case MSR_IA32_RTIT_CTL:
7392 case MSR_IA32_RTIT_STATUS:
7393 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT))
7394 return;
7395 break;
7396 case MSR_IA32_RTIT_CR3_MATCH:
7397 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
7398 !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering))
7399 return;
7400 break;
7401 case MSR_IA32_RTIT_OUTPUT_BASE:
7402 case MSR_IA32_RTIT_OUTPUT_MASK:
7403 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
7404 (!intel_pt_validate_hw_cap(PT_CAP_topa_output) &&
7405 !intel_pt_validate_hw_cap(PT_CAP_single_range_output)))
7406 return;
7407 break;
7408 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
7409 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) ||
7410 (msr_index - MSR_IA32_RTIT_ADDR0_A >=
7411 intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2))
7412 return;
7413 break;
7414 case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR_MAX:
7415 if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >=
7416 kvm_pmu_cap.num_counters_gp)
7417 return;
7418 break;
7419 case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL_MAX:
7420 if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >=
7421 kvm_pmu_cap.num_counters_gp)
7422 return;
7423 break;
7424 case MSR_ARCH_PERFMON_FIXED_CTR0 ... MSR_ARCH_PERFMON_FIXED_CTR_MAX:
7425 if (msr_index - MSR_ARCH_PERFMON_FIXED_CTR0 >=
7426 kvm_pmu_cap.num_counters_fixed)
7427 return;
7428 break;
7429 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
7430 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
7431 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
7432 if (!kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2))
7433 return;
7434 break;
7435 case MSR_IA32_XFD:
7436 case MSR_IA32_XFD_ERR:
7437 if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
7438 return;
7439 break;
7440 case MSR_IA32_TSX_CTRL:
7441 if (!(kvm_get_arch_capabilities() & ARCH_CAP_TSX_CTRL_MSR))
7442 return;
7443 break;
7444 default:
7445 break;
7446 }
7447
7448 msrs_to_save[num_msrs_to_save++] = msr_index;
7449 }
7450
kvm_init_msr_lists(void)7451 static void kvm_init_msr_lists(void)
7452 {
7453 unsigned i;
7454
7455 BUILD_BUG_ON_MSG(KVM_PMC_MAX_FIXED != 3,
7456 "Please update the fixed PMCs in msrs_to_save_pmu[]");
7457
7458 num_msrs_to_save = 0;
7459 num_emulated_msrs = 0;
7460 num_msr_based_features = 0;
7461
7462 for (i = 0; i < ARRAY_SIZE(msrs_to_save_base); i++)
7463 kvm_probe_msr_to_save(msrs_to_save_base[i]);
7464
7465 if (enable_pmu) {
7466 for (i = 0; i < ARRAY_SIZE(msrs_to_save_pmu); i++)
7467 kvm_probe_msr_to_save(msrs_to_save_pmu[i]);
7468 }
7469
7470 for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) {
7471 if (!static_call(kvm_x86_has_emulated_msr)(NULL, emulated_msrs_all[i]))
7472 continue;
7473
7474 emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i];
7475 }
7476
7477 for (i = KVM_FIRST_EMULATED_VMX_MSR; i <= KVM_LAST_EMULATED_VMX_MSR; i++)
7478 kvm_probe_feature_msr(i);
7479
7480 for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++)
7481 kvm_probe_feature_msr(msr_based_features_all_except_vmx[i]);
7482 }
7483
vcpu_mmio_write(struct kvm_vcpu * vcpu,gpa_t addr,int len,const void * v)7484 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
7485 const void *v)
7486 {
7487 int handled = 0;
7488 int n;
7489
7490 do {
7491 n = min(len, 8);
7492 if (!(lapic_in_kernel(vcpu) &&
7493 !kvm_iodevice_write(vcpu, &vcpu->arch.apic->dev, addr, n, v))
7494 && kvm_io_bus_write(vcpu, KVM_MMIO_BUS, addr, n, v))
7495 break;
7496 handled += n;
7497 addr += n;
7498 len -= n;
7499 v += n;
7500 } while (len);
7501
7502 return handled;
7503 }
7504
vcpu_mmio_read(struct kvm_vcpu * vcpu,gpa_t addr,int len,void * v)7505 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
7506 {
7507 int handled = 0;
7508 int n;
7509
7510 do {
7511 n = min(len, 8);
7512 if (!(lapic_in_kernel(vcpu) &&
7513 !kvm_iodevice_read(vcpu, &vcpu->arch.apic->dev,
7514 addr, n, v))
7515 && kvm_io_bus_read(vcpu, KVM_MMIO_BUS, addr, n, v))
7516 break;
7517 trace_kvm_mmio(KVM_TRACE_MMIO_READ, n, addr, v);
7518 handled += n;
7519 addr += n;
7520 len -= n;
7521 v += n;
7522 } while (len);
7523
7524 return handled;
7525 }
7526
kvm_set_segment(struct kvm_vcpu * vcpu,struct kvm_segment * var,int seg)7527 void kvm_set_segment(struct kvm_vcpu *vcpu,
7528 struct kvm_segment *var, int seg)
7529 {
7530 static_call(kvm_x86_set_segment)(vcpu, var, seg);
7531 }
7532
kvm_get_segment(struct kvm_vcpu * vcpu,struct kvm_segment * var,int seg)7533 void kvm_get_segment(struct kvm_vcpu *vcpu,
7534 struct kvm_segment *var, int seg)
7535 {
7536 static_call(kvm_x86_get_segment)(vcpu, var, seg);
7537 }
7538
translate_nested_gpa(struct kvm_vcpu * vcpu,gpa_t gpa,u64 access,struct x86_exception * exception)7539 gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,
7540 struct x86_exception *exception)
7541 {
7542 struct kvm_mmu *mmu = vcpu->arch.mmu;
7543 gpa_t t_gpa;
7544
7545 BUG_ON(!mmu_is_nested(vcpu));
7546
7547 /* NPT walks are always user-walks */
7548 access |= PFERR_USER_MASK;
7549 t_gpa = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
7550
7551 return t_gpa;
7552 }
7553
kvm_mmu_gva_to_gpa_read(struct kvm_vcpu * vcpu,gva_t gva,struct x86_exception * exception)7554 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
7555 struct x86_exception *exception)
7556 {
7557 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7558
7559 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7560 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7561 }
7562 EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_read);
7563
kvm_mmu_gva_to_gpa_write(struct kvm_vcpu * vcpu,gva_t gva,struct x86_exception * exception)7564 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
7565 struct x86_exception *exception)
7566 {
7567 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7568
7569 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7570 access |= PFERR_WRITE_MASK;
7571 return mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7572 }
7573 EXPORT_SYMBOL_GPL(kvm_mmu_gva_to_gpa_write);
7574
7575 /* uses this to access any guest's mapped memory without checking CPL */
kvm_mmu_gva_to_gpa_system(struct kvm_vcpu * vcpu,gva_t gva,struct x86_exception * exception)7576 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
7577 struct x86_exception *exception)
7578 {
7579 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7580
7581 return mmu->gva_to_gpa(vcpu, mmu, gva, 0, exception);
7582 }
7583
kvm_read_guest_virt_helper(gva_t addr,void * val,unsigned int bytes,struct kvm_vcpu * vcpu,u64 access,struct x86_exception * exception)7584 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
7585 struct kvm_vcpu *vcpu, u64 access,
7586 struct x86_exception *exception)
7587 {
7588 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7589 void *data = val;
7590 int r = X86EMUL_CONTINUE;
7591
7592 while (bytes) {
7593 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
7594 unsigned offset = addr & (PAGE_SIZE-1);
7595 unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
7596 int ret;
7597
7598 if (gpa == INVALID_GPA)
7599 return X86EMUL_PROPAGATE_FAULT;
7600 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, data,
7601 offset, toread);
7602 if (ret < 0) {
7603 r = X86EMUL_IO_NEEDED;
7604 goto out;
7605 }
7606
7607 bytes -= toread;
7608 data += toread;
7609 addr += toread;
7610 }
7611 out:
7612 return r;
7613 }
7614
7615 /* used for instruction fetching */
kvm_fetch_guest_virt(struct x86_emulate_ctxt * ctxt,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception)7616 static int kvm_fetch_guest_virt(struct x86_emulate_ctxt *ctxt,
7617 gva_t addr, void *val, unsigned int bytes,
7618 struct x86_exception *exception)
7619 {
7620 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7621 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7622 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7623 unsigned offset;
7624 int ret;
7625
7626 /* Inline kvm_read_guest_virt_helper for speed. */
7627 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access|PFERR_FETCH_MASK,
7628 exception);
7629 if (unlikely(gpa == INVALID_GPA))
7630 return X86EMUL_PROPAGATE_FAULT;
7631
7632 offset = addr & (PAGE_SIZE-1);
7633 if (WARN_ON(offset + bytes > PAGE_SIZE))
7634 bytes = (unsigned)PAGE_SIZE - offset;
7635 ret = kvm_vcpu_read_guest_page(vcpu, gpa >> PAGE_SHIFT, val,
7636 offset, bytes);
7637 if (unlikely(ret < 0))
7638 return X86EMUL_IO_NEEDED;
7639
7640 return X86EMUL_CONTINUE;
7641 }
7642
kvm_read_guest_virt(struct kvm_vcpu * vcpu,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception)7643 int kvm_read_guest_virt(struct kvm_vcpu *vcpu,
7644 gva_t addr, void *val, unsigned int bytes,
7645 struct x86_exception *exception)
7646 {
7647 u64 access = (static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0;
7648
7649 /*
7650 * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED
7651 * is returned, but our callers are not ready for that and they blindly
7652 * call kvm_inject_page_fault. Ensure that they at least do not leak
7653 * uninitialized kernel stack memory into cr2 and error code.
7654 */
7655 memset(exception, 0, sizeof(*exception));
7656 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
7657 exception);
7658 }
7659 EXPORT_SYMBOL_GPL(kvm_read_guest_virt);
7660
emulator_read_std(struct x86_emulate_ctxt * ctxt,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception,bool system)7661 static int emulator_read_std(struct x86_emulate_ctxt *ctxt,
7662 gva_t addr, void *val, unsigned int bytes,
7663 struct x86_exception *exception, bool system)
7664 {
7665 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7666 u64 access = 0;
7667
7668 if (system)
7669 access |= PFERR_IMPLICIT_ACCESS;
7670 else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)
7671 access |= PFERR_USER_MASK;
7672
7673 return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access, exception);
7674 }
7675
kvm_write_guest_virt_helper(gva_t addr,void * val,unsigned int bytes,struct kvm_vcpu * vcpu,u64 access,struct x86_exception * exception)7676 static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
7677 struct kvm_vcpu *vcpu, u64 access,
7678 struct x86_exception *exception)
7679 {
7680 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7681 void *data = val;
7682 int r = X86EMUL_CONTINUE;
7683
7684 while (bytes) {
7685 gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
7686 unsigned offset = addr & (PAGE_SIZE-1);
7687 unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
7688 int ret;
7689
7690 if (gpa == INVALID_GPA)
7691 return X86EMUL_PROPAGATE_FAULT;
7692 ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
7693 if (ret < 0) {
7694 r = X86EMUL_IO_NEEDED;
7695 goto out;
7696 }
7697
7698 bytes -= towrite;
7699 data += towrite;
7700 addr += towrite;
7701 }
7702 out:
7703 return r;
7704 }
7705
emulator_write_std(struct x86_emulate_ctxt * ctxt,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception,bool system)7706 static int emulator_write_std(struct x86_emulate_ctxt *ctxt, gva_t addr, void *val,
7707 unsigned int bytes, struct x86_exception *exception,
7708 bool system)
7709 {
7710 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7711 u64 access = PFERR_WRITE_MASK;
7712
7713 if (system)
7714 access |= PFERR_IMPLICIT_ACCESS;
7715 else if (static_call(kvm_x86_get_cpl)(vcpu) == 3)
7716 access |= PFERR_USER_MASK;
7717
7718 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
7719 access, exception);
7720 }
7721
kvm_write_guest_virt_system(struct kvm_vcpu * vcpu,gva_t addr,void * val,unsigned int bytes,struct x86_exception * exception)7722 int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
7723 unsigned int bytes, struct x86_exception *exception)
7724 {
7725 /* kvm_write_guest_virt_system can pull in tons of pages. */
7726 vcpu->arch.l1tf_flush_l1d = true;
7727
7728 return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
7729 PFERR_WRITE_MASK, exception);
7730 }
7731 EXPORT_SYMBOL_GPL(kvm_write_guest_virt_system);
7732
kvm_check_emulate_insn(struct kvm_vcpu * vcpu,int emul_type,void * insn,int insn_len)7733 static int kvm_check_emulate_insn(struct kvm_vcpu *vcpu, int emul_type,
7734 void *insn, int insn_len)
7735 {
7736 return static_call(kvm_x86_check_emulate_instruction)(vcpu, emul_type,
7737 insn, insn_len);
7738 }
7739
handle_ud(struct kvm_vcpu * vcpu)7740 int handle_ud(struct kvm_vcpu *vcpu)
7741 {
7742 static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX };
7743 int fep_flags = READ_ONCE(force_emulation_prefix);
7744 int emul_type = EMULTYPE_TRAP_UD;
7745 char sig[5]; /* ud2; .ascii "kvm" */
7746 struct x86_exception e;
7747 int r;
7748
7749 r = kvm_check_emulate_insn(vcpu, emul_type, NULL, 0);
7750 if (r != X86EMUL_CONTINUE)
7751 return 1;
7752
7753 if (fep_flags &&
7754 kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu),
7755 sig, sizeof(sig), &e) == 0 &&
7756 memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) {
7757 if (fep_flags & KVM_FEP_CLEAR_RFLAGS_RF)
7758 kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF);
7759 kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig));
7760 emul_type = EMULTYPE_TRAP_UD_FORCED;
7761 }
7762
7763 return kvm_emulate_instruction(vcpu, emul_type);
7764 }
7765 EXPORT_SYMBOL_GPL(handle_ud);
7766
vcpu_is_mmio_gpa(struct kvm_vcpu * vcpu,unsigned long gva,gpa_t gpa,bool write)7767 static int vcpu_is_mmio_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
7768 gpa_t gpa, bool write)
7769 {
7770 /* For APIC access vmexit */
7771 if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
7772 return 1;
7773
7774 if (vcpu_match_mmio_gpa(vcpu, gpa)) {
7775 trace_vcpu_match_mmio(gva, gpa, write, true);
7776 return 1;
7777 }
7778
7779 return 0;
7780 }
7781
vcpu_mmio_gva_to_gpa(struct kvm_vcpu * vcpu,unsigned long gva,gpa_t * gpa,struct x86_exception * exception,bool write)7782 static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
7783 gpa_t *gpa, struct x86_exception *exception,
7784 bool write)
7785 {
7786 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
7787 u64 access = ((static_call(kvm_x86_get_cpl)(vcpu) == 3) ? PFERR_USER_MASK : 0)
7788 | (write ? PFERR_WRITE_MASK : 0);
7789
7790 /*
7791 * currently PKRU is only applied to ept enabled guest so
7792 * there is no pkey in EPT page table for L1 guest or EPT
7793 * shadow page table for L2 guest.
7794 */
7795 if (vcpu_match_mmio_gva(vcpu, gva) && (!is_paging(vcpu) ||
7796 !permission_fault(vcpu, vcpu->arch.walk_mmu,
7797 vcpu->arch.mmio_access, 0, access))) {
7798 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
7799 (gva & (PAGE_SIZE - 1));
7800 trace_vcpu_match_mmio(gva, *gpa, write, false);
7801 return 1;
7802 }
7803
7804 *gpa = mmu->gva_to_gpa(vcpu, mmu, gva, access, exception);
7805
7806 if (*gpa == INVALID_GPA)
7807 return -1;
7808
7809 return vcpu_is_mmio_gpa(vcpu, gva, *gpa, write);
7810 }
7811
emulator_write_phys(struct kvm_vcpu * vcpu,gpa_t gpa,const void * val,int bytes)7812 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
7813 const void *val, int bytes)
7814 {
7815 int ret;
7816
7817 ret = kvm_vcpu_write_guest(vcpu, gpa, val, bytes);
7818 if (ret < 0)
7819 return 0;
7820 kvm_page_track_write(vcpu, gpa, val, bytes);
7821 return 1;
7822 }
7823
7824 struct read_write_emulator_ops {
7825 int (*read_write_prepare)(struct kvm_vcpu *vcpu, void *val,
7826 int bytes);
7827 int (*read_write_emulate)(struct kvm_vcpu *vcpu, gpa_t gpa,
7828 void *val, int bytes);
7829 int (*read_write_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7830 int bytes, void *val);
7831 int (*read_write_exit_mmio)(struct kvm_vcpu *vcpu, gpa_t gpa,
7832 void *val, int bytes);
7833 bool write;
7834 };
7835
read_prepare(struct kvm_vcpu * vcpu,void * val,int bytes)7836 static int read_prepare(struct kvm_vcpu *vcpu, void *val, int bytes)
7837 {
7838 if (vcpu->mmio_read_completed) {
7839 trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
7840 vcpu->mmio_fragments[0].gpa, val);
7841 vcpu->mmio_read_completed = 0;
7842 return 1;
7843 }
7844
7845 return 0;
7846 }
7847
read_emulate(struct kvm_vcpu * vcpu,gpa_t gpa,void * val,int bytes)7848 static int read_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
7849 void *val, int bytes)
7850 {
7851 return !kvm_vcpu_read_guest(vcpu, gpa, val, bytes);
7852 }
7853
write_emulate(struct kvm_vcpu * vcpu,gpa_t gpa,void * val,int bytes)7854 static int write_emulate(struct kvm_vcpu *vcpu, gpa_t gpa,
7855 void *val, int bytes)
7856 {
7857 return emulator_write_phys(vcpu, gpa, val, bytes);
7858 }
7859
write_mmio(struct kvm_vcpu * vcpu,gpa_t gpa,int bytes,void * val)7860 static int write_mmio(struct kvm_vcpu *vcpu, gpa_t gpa, int bytes, void *val)
7861 {
7862 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, val);
7863 return vcpu_mmio_write(vcpu, gpa, bytes, val);
7864 }
7865
read_exit_mmio(struct kvm_vcpu * vcpu,gpa_t gpa,void * val,int bytes)7866 static int read_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7867 void *val, int bytes)
7868 {
7869 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, NULL);
7870 return X86EMUL_IO_NEEDED;
7871 }
7872
write_exit_mmio(struct kvm_vcpu * vcpu,gpa_t gpa,void * val,int bytes)7873 static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
7874 void *val, int bytes)
7875 {
7876 struct kvm_mmio_fragment *frag = &vcpu->mmio_fragments[0];
7877
7878 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
7879 return X86EMUL_CONTINUE;
7880 }
7881
7882 static const struct read_write_emulator_ops read_emultor = {
7883 .read_write_prepare = read_prepare,
7884 .read_write_emulate = read_emulate,
7885 .read_write_mmio = vcpu_mmio_read,
7886 .read_write_exit_mmio = read_exit_mmio,
7887 };
7888
7889 static const struct read_write_emulator_ops write_emultor = {
7890 .read_write_emulate = write_emulate,
7891 .read_write_mmio = write_mmio,
7892 .read_write_exit_mmio = write_exit_mmio,
7893 .write = true,
7894 };
7895
emulator_read_write_onepage(unsigned long addr,void * val,unsigned int bytes,struct x86_exception * exception,struct kvm_vcpu * vcpu,const struct read_write_emulator_ops * ops)7896 static int emulator_read_write_onepage(unsigned long addr, void *val,
7897 unsigned int bytes,
7898 struct x86_exception *exception,
7899 struct kvm_vcpu *vcpu,
7900 const struct read_write_emulator_ops *ops)
7901 {
7902 gpa_t gpa;
7903 int handled, ret;
7904 bool write = ops->write;
7905 struct kvm_mmio_fragment *frag;
7906 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
7907
7908 /*
7909 * If the exit was due to a NPF we may already have a GPA.
7910 * If the GPA is present, use it to avoid the GVA to GPA table walk.
7911 * Note, this cannot be used on string operations since string
7912 * operation using rep will only have the initial GPA from the NPF
7913 * occurred.
7914 */
7915 if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) &&
7916 (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) {
7917 gpa = ctxt->gpa_val;
7918 ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write);
7919 } else {
7920 ret = vcpu_mmio_gva_to_gpa(vcpu, addr, &gpa, exception, write);
7921 if (ret < 0)
7922 return X86EMUL_PROPAGATE_FAULT;
7923 }
7924
7925 if (!ret && ops->read_write_emulate(vcpu, gpa, val, bytes))
7926 return X86EMUL_CONTINUE;
7927
7928 /*
7929 * Is this MMIO handled locally?
7930 */
7931 handled = ops->read_write_mmio(vcpu, gpa, bytes, val);
7932 if (handled == bytes)
7933 return X86EMUL_CONTINUE;
7934
7935 gpa += handled;
7936 bytes -= handled;
7937 val += handled;
7938
7939 WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS);
7940 frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++];
7941 frag->gpa = gpa;
7942 frag->data = val;
7943 frag->len = bytes;
7944 return X86EMUL_CONTINUE;
7945 }
7946
emulator_read_write(struct x86_emulate_ctxt * ctxt,unsigned long addr,void * val,unsigned int bytes,struct x86_exception * exception,const struct read_write_emulator_ops * ops)7947 static int emulator_read_write(struct x86_emulate_ctxt *ctxt,
7948 unsigned long addr,
7949 void *val, unsigned int bytes,
7950 struct x86_exception *exception,
7951 const struct read_write_emulator_ops *ops)
7952 {
7953 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
7954 gpa_t gpa;
7955 int rc;
7956
7957 if (ops->read_write_prepare &&
7958 ops->read_write_prepare(vcpu, val, bytes))
7959 return X86EMUL_CONTINUE;
7960
7961 vcpu->mmio_nr_fragments = 0;
7962
7963 /* Crossing a page boundary? */
7964 if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
7965 int now;
7966
7967 now = -addr & ~PAGE_MASK;
7968 rc = emulator_read_write_onepage(addr, val, now, exception,
7969 vcpu, ops);
7970
7971 if (rc != X86EMUL_CONTINUE)
7972 return rc;
7973 addr += now;
7974 if (ctxt->mode != X86EMUL_MODE_PROT64)
7975 addr = (u32)addr;
7976 val += now;
7977 bytes -= now;
7978 }
7979
7980 rc = emulator_read_write_onepage(addr, val, bytes, exception,
7981 vcpu, ops);
7982 if (rc != X86EMUL_CONTINUE)
7983 return rc;
7984
7985 if (!vcpu->mmio_nr_fragments)
7986 return rc;
7987
7988 gpa = vcpu->mmio_fragments[0].gpa;
7989
7990 vcpu->mmio_needed = 1;
7991 vcpu->mmio_cur_fragment = 0;
7992
7993 vcpu->run->mmio.len = min(8u, vcpu->mmio_fragments[0].len);
7994 vcpu->run->mmio.is_write = vcpu->mmio_is_write = ops->write;
7995 vcpu->run->exit_reason = KVM_EXIT_MMIO;
7996 vcpu->run->mmio.phys_addr = gpa;
7997
7998 return ops->read_write_exit_mmio(vcpu, gpa, val, bytes);
7999 }
8000
emulator_read_emulated(struct x86_emulate_ctxt * ctxt,unsigned long addr,void * val,unsigned int bytes,struct x86_exception * exception)8001 static int emulator_read_emulated(struct x86_emulate_ctxt *ctxt,
8002 unsigned long addr,
8003 void *val,
8004 unsigned int bytes,
8005 struct x86_exception *exception)
8006 {
8007 return emulator_read_write(ctxt, addr, val, bytes,
8008 exception, &read_emultor);
8009 }
8010
emulator_write_emulated(struct x86_emulate_ctxt * ctxt,unsigned long addr,const void * val,unsigned int bytes,struct x86_exception * exception)8011 static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt,
8012 unsigned long addr,
8013 const void *val,
8014 unsigned int bytes,
8015 struct x86_exception *exception)
8016 {
8017 return emulator_read_write(ctxt, addr, (void *)val, bytes,
8018 exception, &write_emultor);
8019 }
8020
8021 #define emulator_try_cmpxchg_user(t, ptr, old, new) \
8022 (__try_cmpxchg_user((t __user *)(ptr), (t *)(old), *(t *)(new), efault ## t))
8023
emulator_cmpxchg_emulated(struct x86_emulate_ctxt * ctxt,unsigned long addr,const void * old,const void * new,unsigned int bytes,struct x86_exception * exception)8024 static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
8025 unsigned long addr,
8026 const void *old,
8027 const void *new,
8028 unsigned int bytes,
8029 struct x86_exception *exception)
8030 {
8031 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8032 u64 page_line_mask;
8033 unsigned long hva;
8034 gpa_t gpa;
8035 int r;
8036
8037 /* guests cmpxchg8b have to be emulated atomically */
8038 if (bytes > 8 || (bytes & (bytes - 1)))
8039 goto emul_write;
8040
8041 gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
8042
8043 if (gpa == INVALID_GPA ||
8044 (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
8045 goto emul_write;
8046
8047 /*
8048 * Emulate the atomic as a straight write to avoid #AC if SLD is
8049 * enabled in the host and the access splits a cache line.
8050 */
8051 if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
8052 page_line_mask = ~(cache_line_size() - 1);
8053 else
8054 page_line_mask = PAGE_MASK;
8055
8056 if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask))
8057 goto emul_write;
8058
8059 hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa));
8060 if (kvm_is_error_hva(hva))
8061 goto emul_write;
8062
8063 hva += offset_in_page(gpa);
8064
8065 switch (bytes) {
8066 case 1:
8067 r = emulator_try_cmpxchg_user(u8, hva, old, new);
8068 break;
8069 case 2:
8070 r = emulator_try_cmpxchg_user(u16, hva, old, new);
8071 break;
8072 case 4:
8073 r = emulator_try_cmpxchg_user(u32, hva, old, new);
8074 break;
8075 case 8:
8076 r = emulator_try_cmpxchg_user(u64, hva, old, new);
8077 break;
8078 default:
8079 BUG();
8080 }
8081
8082 if (r < 0)
8083 return X86EMUL_UNHANDLEABLE;
8084
8085 /*
8086 * Mark the page dirty _before_ checking whether or not the CMPXCHG was
8087 * successful, as the old value is written back on failure. Note, for
8088 * live migration, this is unnecessarily conservative as CMPXCHG writes
8089 * back the original value and the access is atomic, but KVM's ABI is
8090 * that all writes are dirty logged, regardless of the value written.
8091 */
8092 kvm_vcpu_mark_page_dirty(vcpu, gpa_to_gfn(gpa));
8093
8094 if (r)
8095 return X86EMUL_CMPXCHG_FAILED;
8096
8097 kvm_page_track_write(vcpu, gpa, new, bytes);
8098
8099 return X86EMUL_CONTINUE;
8100
8101 emul_write:
8102 pr_warn_once("emulating exchange as write\n");
8103
8104 return emulator_write_emulated(ctxt, addr, new, bytes, exception);
8105 }
8106
emulator_pio_in_out(struct kvm_vcpu * vcpu,int size,unsigned short port,void * data,unsigned int count,bool in)8107 static int emulator_pio_in_out(struct kvm_vcpu *vcpu, int size,
8108 unsigned short port, void *data,
8109 unsigned int count, bool in)
8110 {
8111 unsigned i;
8112 int r;
8113
8114 WARN_ON_ONCE(vcpu->arch.pio.count);
8115 for (i = 0; i < count; i++) {
8116 if (in)
8117 r = kvm_io_bus_read(vcpu, KVM_PIO_BUS, port, size, data);
8118 else
8119 r = kvm_io_bus_write(vcpu, KVM_PIO_BUS, port, size, data);
8120
8121 if (r) {
8122 if (i == 0)
8123 goto userspace_io;
8124
8125 /*
8126 * Userspace must have unregistered the device while PIO
8127 * was running. Drop writes / read as 0.
8128 */
8129 if (in)
8130 memset(data, 0, size * (count - i));
8131 break;
8132 }
8133
8134 data += size;
8135 }
8136 return 1;
8137
8138 userspace_io:
8139 vcpu->arch.pio.port = port;
8140 vcpu->arch.pio.in = in;
8141 vcpu->arch.pio.count = count;
8142 vcpu->arch.pio.size = size;
8143
8144 if (in)
8145 memset(vcpu->arch.pio_data, 0, size * count);
8146 else
8147 memcpy(vcpu->arch.pio_data, data, size * count);
8148
8149 vcpu->run->exit_reason = KVM_EXIT_IO;
8150 vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT;
8151 vcpu->run->io.size = size;
8152 vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
8153 vcpu->run->io.count = count;
8154 vcpu->run->io.port = port;
8155 return 0;
8156 }
8157
emulator_pio_in(struct kvm_vcpu * vcpu,int size,unsigned short port,void * val,unsigned int count)8158 static int emulator_pio_in(struct kvm_vcpu *vcpu, int size,
8159 unsigned short port, void *val, unsigned int count)
8160 {
8161 int r = emulator_pio_in_out(vcpu, size, port, val, count, true);
8162 if (r)
8163 trace_kvm_pio(KVM_PIO_IN, port, size, count, val);
8164
8165 return r;
8166 }
8167
complete_emulator_pio_in(struct kvm_vcpu * vcpu,void * val)8168 static void complete_emulator_pio_in(struct kvm_vcpu *vcpu, void *val)
8169 {
8170 int size = vcpu->arch.pio.size;
8171 unsigned int count = vcpu->arch.pio.count;
8172 memcpy(val, vcpu->arch.pio_data, size * count);
8173 trace_kvm_pio(KVM_PIO_IN, vcpu->arch.pio.port, size, count, vcpu->arch.pio_data);
8174 vcpu->arch.pio.count = 0;
8175 }
8176
emulator_pio_in_emulated(struct x86_emulate_ctxt * ctxt,int size,unsigned short port,void * val,unsigned int count)8177 static int emulator_pio_in_emulated(struct x86_emulate_ctxt *ctxt,
8178 int size, unsigned short port, void *val,
8179 unsigned int count)
8180 {
8181 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8182 if (vcpu->arch.pio.count) {
8183 /*
8184 * Complete a previous iteration that required userspace I/O.
8185 * Note, @count isn't guaranteed to match pio.count as userspace
8186 * can modify ECX before rerunning the vCPU. Ignore any such
8187 * shenanigans as KVM doesn't support modifying the rep count,
8188 * and the emulator ensures @count doesn't overflow the buffer.
8189 */
8190 complete_emulator_pio_in(vcpu, val);
8191 return 1;
8192 }
8193
8194 return emulator_pio_in(vcpu, size, port, val, count);
8195 }
8196
emulator_pio_out(struct kvm_vcpu * vcpu,int size,unsigned short port,const void * val,unsigned int count)8197 static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
8198 unsigned short port, const void *val,
8199 unsigned int count)
8200 {
8201 trace_kvm_pio(KVM_PIO_OUT, port, size, count, val);
8202 return emulator_pio_in_out(vcpu, size, port, (void *)val, count, false);
8203 }
8204
emulator_pio_out_emulated(struct x86_emulate_ctxt * ctxt,int size,unsigned short port,const void * val,unsigned int count)8205 static int emulator_pio_out_emulated(struct x86_emulate_ctxt *ctxt,
8206 int size, unsigned short port,
8207 const void *val, unsigned int count)
8208 {
8209 return emulator_pio_out(emul_to_vcpu(ctxt), size, port, val, count);
8210 }
8211
get_segment_base(struct kvm_vcpu * vcpu,int seg)8212 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
8213 {
8214 return static_call(kvm_x86_get_segment_base)(vcpu, seg);
8215 }
8216
emulator_invlpg(struct x86_emulate_ctxt * ctxt,ulong address)8217 static void emulator_invlpg(struct x86_emulate_ctxt *ctxt, ulong address)
8218 {
8219 kvm_mmu_invlpg(emul_to_vcpu(ctxt), address);
8220 }
8221
kvm_emulate_wbinvd_noskip(struct kvm_vcpu * vcpu)8222 static int kvm_emulate_wbinvd_noskip(struct kvm_vcpu *vcpu)
8223 {
8224 if (!need_emulate_wbinvd(vcpu))
8225 return X86EMUL_CONTINUE;
8226
8227 if (static_call(kvm_x86_has_wbinvd_exit)()) {
8228 int cpu = get_cpu();
8229
8230 cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
8231 on_each_cpu_mask(vcpu->arch.wbinvd_dirty_mask,
8232 wbinvd_ipi, NULL, 1);
8233 put_cpu();
8234 cpumask_clear(vcpu->arch.wbinvd_dirty_mask);
8235 } else
8236 wbinvd();
8237 return X86EMUL_CONTINUE;
8238 }
8239
kvm_emulate_wbinvd(struct kvm_vcpu * vcpu)8240 int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu)
8241 {
8242 kvm_emulate_wbinvd_noskip(vcpu);
8243 return kvm_skip_emulated_instruction(vcpu);
8244 }
8245 EXPORT_SYMBOL_GPL(kvm_emulate_wbinvd);
8246
8247
8248
emulator_wbinvd(struct x86_emulate_ctxt * ctxt)8249 static void emulator_wbinvd(struct x86_emulate_ctxt *ctxt)
8250 {
8251 kvm_emulate_wbinvd_noskip(emul_to_vcpu(ctxt));
8252 }
8253
emulator_get_dr(struct x86_emulate_ctxt * ctxt,int dr)8254 static unsigned long emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr)
8255 {
8256 return kvm_get_dr(emul_to_vcpu(ctxt), dr);
8257 }
8258
emulator_set_dr(struct x86_emulate_ctxt * ctxt,int dr,unsigned long value)8259 static int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
8260 unsigned long value)
8261 {
8262
8263 return kvm_set_dr(emul_to_vcpu(ctxt), dr, value);
8264 }
8265
mk_cr_64(u64 curr_cr,u32 new_val)8266 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
8267 {
8268 return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
8269 }
8270
emulator_get_cr(struct x86_emulate_ctxt * ctxt,int cr)8271 static unsigned long emulator_get_cr(struct x86_emulate_ctxt *ctxt, int cr)
8272 {
8273 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8274 unsigned long value;
8275
8276 switch (cr) {
8277 case 0:
8278 value = kvm_read_cr0(vcpu);
8279 break;
8280 case 2:
8281 value = vcpu->arch.cr2;
8282 break;
8283 case 3:
8284 value = kvm_read_cr3(vcpu);
8285 break;
8286 case 4:
8287 value = kvm_read_cr4(vcpu);
8288 break;
8289 case 8:
8290 value = kvm_get_cr8(vcpu);
8291 break;
8292 default:
8293 kvm_err("%s: unexpected cr %u\n", __func__, cr);
8294 return 0;
8295 }
8296
8297 return value;
8298 }
8299
emulator_set_cr(struct x86_emulate_ctxt * ctxt,int cr,ulong val)8300 static int emulator_set_cr(struct x86_emulate_ctxt *ctxt, int cr, ulong val)
8301 {
8302 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8303 int res = 0;
8304
8305 switch (cr) {
8306 case 0:
8307 res = kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
8308 break;
8309 case 2:
8310 vcpu->arch.cr2 = val;
8311 break;
8312 case 3:
8313 res = kvm_set_cr3(vcpu, val);
8314 break;
8315 case 4:
8316 res = kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
8317 break;
8318 case 8:
8319 res = kvm_set_cr8(vcpu, val);
8320 break;
8321 default:
8322 kvm_err("%s: unexpected cr %u\n", __func__, cr);
8323 res = -1;
8324 }
8325
8326 return res;
8327 }
8328
emulator_get_cpl(struct x86_emulate_ctxt * ctxt)8329 static int emulator_get_cpl(struct x86_emulate_ctxt *ctxt)
8330 {
8331 return static_call(kvm_x86_get_cpl)(emul_to_vcpu(ctxt));
8332 }
8333
emulator_get_gdt(struct x86_emulate_ctxt * ctxt,struct desc_ptr * dt)8334 static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
8335 {
8336 static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt);
8337 }
8338
emulator_get_idt(struct x86_emulate_ctxt * ctxt,struct desc_ptr * dt)8339 static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
8340 {
8341 static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt);
8342 }
8343
emulator_set_gdt(struct x86_emulate_ctxt * ctxt,struct desc_ptr * dt)8344 static void emulator_set_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
8345 {
8346 static_call(kvm_x86_set_gdt)(emul_to_vcpu(ctxt), dt);
8347 }
8348
emulator_set_idt(struct x86_emulate_ctxt * ctxt,struct desc_ptr * dt)8349 static void emulator_set_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
8350 {
8351 static_call(kvm_x86_set_idt)(emul_to_vcpu(ctxt), dt);
8352 }
8353
emulator_get_cached_segment_base(struct x86_emulate_ctxt * ctxt,int seg)8354 static unsigned long emulator_get_cached_segment_base(
8355 struct x86_emulate_ctxt *ctxt, int seg)
8356 {
8357 return get_segment_base(emul_to_vcpu(ctxt), seg);
8358 }
8359
emulator_get_segment(struct x86_emulate_ctxt * ctxt,u16 * selector,struct desc_struct * desc,u32 * base3,int seg)8360 static bool emulator_get_segment(struct x86_emulate_ctxt *ctxt, u16 *selector,
8361 struct desc_struct *desc, u32 *base3,
8362 int seg)
8363 {
8364 struct kvm_segment var;
8365
8366 kvm_get_segment(emul_to_vcpu(ctxt), &var, seg);
8367 *selector = var.selector;
8368
8369 if (var.unusable) {
8370 memset(desc, 0, sizeof(*desc));
8371 if (base3)
8372 *base3 = 0;
8373 return false;
8374 }
8375
8376 if (var.g)
8377 var.limit >>= 12;
8378 set_desc_limit(desc, var.limit);
8379 set_desc_base(desc, (unsigned long)var.base);
8380 #ifdef CONFIG_X86_64
8381 if (base3)
8382 *base3 = var.base >> 32;
8383 #endif
8384 desc->type = var.type;
8385 desc->s = var.s;
8386 desc->dpl = var.dpl;
8387 desc->p = var.present;
8388 desc->avl = var.avl;
8389 desc->l = var.l;
8390 desc->d = var.db;
8391 desc->g = var.g;
8392
8393 return true;
8394 }
8395
emulator_set_segment(struct x86_emulate_ctxt * ctxt,u16 selector,struct desc_struct * desc,u32 base3,int seg)8396 static void emulator_set_segment(struct x86_emulate_ctxt *ctxt, u16 selector,
8397 struct desc_struct *desc, u32 base3,
8398 int seg)
8399 {
8400 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8401 struct kvm_segment var;
8402
8403 var.selector = selector;
8404 var.base = get_desc_base(desc);
8405 #ifdef CONFIG_X86_64
8406 var.base |= ((u64)base3) << 32;
8407 #endif
8408 var.limit = get_desc_limit(desc);
8409 if (desc->g)
8410 var.limit = (var.limit << 12) | 0xfff;
8411 var.type = desc->type;
8412 var.dpl = desc->dpl;
8413 var.db = desc->d;
8414 var.s = desc->s;
8415 var.l = desc->l;
8416 var.g = desc->g;
8417 var.avl = desc->avl;
8418 var.present = desc->p;
8419 var.unusable = !var.present;
8420 var.padding = 0;
8421
8422 kvm_set_segment(vcpu, &var, seg);
8423 return;
8424 }
8425
emulator_get_msr_with_filter(struct x86_emulate_ctxt * ctxt,u32 msr_index,u64 * pdata)8426 static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt,
8427 u32 msr_index, u64 *pdata)
8428 {
8429 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8430 int r;
8431
8432 r = kvm_get_msr_with_filter(vcpu, msr_index, pdata);
8433 if (r < 0)
8434 return X86EMUL_UNHANDLEABLE;
8435
8436 if (r) {
8437 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0,
8438 complete_emulated_rdmsr, r))
8439 return X86EMUL_IO_NEEDED;
8440
8441 trace_kvm_msr_read_ex(msr_index);
8442 return X86EMUL_PROPAGATE_FAULT;
8443 }
8444
8445 trace_kvm_msr_read(msr_index, *pdata);
8446 return X86EMUL_CONTINUE;
8447 }
8448
emulator_set_msr_with_filter(struct x86_emulate_ctxt * ctxt,u32 msr_index,u64 data)8449 static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt,
8450 u32 msr_index, u64 data)
8451 {
8452 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8453 int r;
8454
8455 r = kvm_set_msr_with_filter(vcpu, msr_index, data);
8456 if (r < 0)
8457 return X86EMUL_UNHANDLEABLE;
8458
8459 if (r) {
8460 if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,
8461 complete_emulated_msr_access, r))
8462 return X86EMUL_IO_NEEDED;
8463
8464 trace_kvm_msr_write_ex(msr_index, data);
8465 return X86EMUL_PROPAGATE_FAULT;
8466 }
8467
8468 trace_kvm_msr_write(msr_index, data);
8469 return X86EMUL_CONTINUE;
8470 }
8471
emulator_get_msr(struct x86_emulate_ctxt * ctxt,u32 msr_index,u64 * pdata)8472 static int emulator_get_msr(struct x86_emulate_ctxt *ctxt,
8473 u32 msr_index, u64 *pdata)
8474 {
8475 return kvm_get_msr(emul_to_vcpu(ctxt), msr_index, pdata);
8476 }
8477
emulator_check_rdpmc_early(struct x86_emulate_ctxt * ctxt,u32 pmc)8478 static int emulator_check_rdpmc_early(struct x86_emulate_ctxt *ctxt, u32 pmc)
8479 {
8480 return kvm_pmu_check_rdpmc_early(emul_to_vcpu(ctxt), pmc);
8481 }
8482
emulator_read_pmc(struct x86_emulate_ctxt * ctxt,u32 pmc,u64 * pdata)8483 static int emulator_read_pmc(struct x86_emulate_ctxt *ctxt,
8484 u32 pmc, u64 *pdata)
8485 {
8486 return kvm_pmu_rdpmc(emul_to_vcpu(ctxt), pmc, pdata);
8487 }
8488
emulator_halt(struct x86_emulate_ctxt * ctxt)8489 static void emulator_halt(struct x86_emulate_ctxt *ctxt)
8490 {
8491 emul_to_vcpu(ctxt)->arch.halt_request = 1;
8492 }
8493
emulator_intercept(struct x86_emulate_ctxt * ctxt,struct x86_instruction_info * info,enum x86_intercept_stage stage)8494 static int emulator_intercept(struct x86_emulate_ctxt *ctxt,
8495 struct x86_instruction_info *info,
8496 enum x86_intercept_stage stage)
8497 {
8498 return static_call(kvm_x86_check_intercept)(emul_to_vcpu(ctxt), info, stage,
8499 &ctxt->exception);
8500 }
8501
emulator_get_cpuid(struct x86_emulate_ctxt * ctxt,u32 * eax,u32 * ebx,u32 * ecx,u32 * edx,bool exact_only)8502 static bool emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
8503 u32 *eax, u32 *ebx, u32 *ecx, u32 *edx,
8504 bool exact_only)
8505 {
8506 return kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx, exact_only);
8507 }
8508
emulator_guest_has_movbe(struct x86_emulate_ctxt * ctxt)8509 static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
8510 {
8511 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
8512 }
8513
emulator_guest_has_fxsr(struct x86_emulate_ctxt * ctxt)8514 static bool emulator_guest_has_fxsr(struct x86_emulate_ctxt *ctxt)
8515 {
8516 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_FXSR);
8517 }
8518
emulator_guest_has_rdpid(struct x86_emulate_ctxt * ctxt)8519 static bool emulator_guest_has_rdpid(struct x86_emulate_ctxt *ctxt)
8520 {
8521 return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_RDPID);
8522 }
8523
emulator_read_gpr(struct x86_emulate_ctxt * ctxt,unsigned reg)8524 static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
8525 {
8526 return kvm_register_read_raw(emul_to_vcpu(ctxt), reg);
8527 }
8528
emulator_write_gpr(struct x86_emulate_ctxt * ctxt,unsigned reg,ulong val)8529 static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
8530 {
8531 kvm_register_write_raw(emul_to_vcpu(ctxt), reg, val);
8532 }
8533
emulator_set_nmi_mask(struct x86_emulate_ctxt * ctxt,bool masked)8534 static void emulator_set_nmi_mask(struct x86_emulate_ctxt *ctxt, bool masked)
8535 {
8536 static_call(kvm_x86_set_nmi_mask)(emul_to_vcpu(ctxt), masked);
8537 }
8538
emulator_is_smm(struct x86_emulate_ctxt * ctxt)8539 static bool emulator_is_smm(struct x86_emulate_ctxt *ctxt)
8540 {
8541 return is_smm(emul_to_vcpu(ctxt));
8542 }
8543
emulator_is_guest_mode(struct x86_emulate_ctxt * ctxt)8544 static bool emulator_is_guest_mode(struct x86_emulate_ctxt *ctxt)
8545 {
8546 return is_guest_mode(emul_to_vcpu(ctxt));
8547 }
8548
8549 #ifndef CONFIG_KVM_SMM
emulator_leave_smm(struct x86_emulate_ctxt * ctxt)8550 static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt)
8551 {
8552 WARN_ON_ONCE(1);
8553 return X86EMUL_UNHANDLEABLE;
8554 }
8555 #endif
8556
emulator_triple_fault(struct x86_emulate_ctxt * ctxt)8557 static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt)
8558 {
8559 kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt));
8560 }
8561
emulator_set_xcr(struct x86_emulate_ctxt * ctxt,u32 index,u64 xcr)8562 static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr)
8563 {
8564 return __kvm_set_xcr(emul_to_vcpu(ctxt), index, xcr);
8565 }
8566
emulator_vm_bugged(struct x86_emulate_ctxt * ctxt)8567 static void emulator_vm_bugged(struct x86_emulate_ctxt *ctxt)
8568 {
8569 struct kvm *kvm = emul_to_vcpu(ctxt)->kvm;
8570
8571 if (!kvm->vm_bugged)
8572 kvm_vm_bugged(kvm);
8573 }
8574
emulator_get_untagged_addr(struct x86_emulate_ctxt * ctxt,gva_t addr,unsigned int flags)8575 static gva_t emulator_get_untagged_addr(struct x86_emulate_ctxt *ctxt,
8576 gva_t addr, unsigned int flags)
8577 {
8578 if (!kvm_x86_ops.get_untagged_addr)
8579 return addr;
8580
8581 return static_call(kvm_x86_get_untagged_addr)(emul_to_vcpu(ctxt), addr, flags);
8582 }
8583
8584 static const struct x86_emulate_ops emulate_ops = {
8585 .vm_bugged = emulator_vm_bugged,
8586 .read_gpr = emulator_read_gpr,
8587 .write_gpr = emulator_write_gpr,
8588 .read_std = emulator_read_std,
8589 .write_std = emulator_write_std,
8590 .fetch = kvm_fetch_guest_virt,
8591 .read_emulated = emulator_read_emulated,
8592 .write_emulated = emulator_write_emulated,
8593 .cmpxchg_emulated = emulator_cmpxchg_emulated,
8594 .invlpg = emulator_invlpg,
8595 .pio_in_emulated = emulator_pio_in_emulated,
8596 .pio_out_emulated = emulator_pio_out_emulated,
8597 .get_segment = emulator_get_segment,
8598 .set_segment = emulator_set_segment,
8599 .get_cached_segment_base = emulator_get_cached_segment_base,
8600 .get_gdt = emulator_get_gdt,
8601 .get_idt = emulator_get_idt,
8602 .set_gdt = emulator_set_gdt,
8603 .set_idt = emulator_set_idt,
8604 .get_cr = emulator_get_cr,
8605 .set_cr = emulator_set_cr,
8606 .cpl = emulator_get_cpl,
8607 .get_dr = emulator_get_dr,
8608 .set_dr = emulator_set_dr,
8609 .set_msr_with_filter = emulator_set_msr_with_filter,
8610 .get_msr_with_filter = emulator_get_msr_with_filter,
8611 .get_msr = emulator_get_msr,
8612 .check_rdpmc_early = emulator_check_rdpmc_early,
8613 .read_pmc = emulator_read_pmc,
8614 .halt = emulator_halt,
8615 .wbinvd = emulator_wbinvd,
8616 .fix_hypercall = emulator_fix_hypercall,
8617 .intercept = emulator_intercept,
8618 .get_cpuid = emulator_get_cpuid,
8619 .guest_has_movbe = emulator_guest_has_movbe,
8620 .guest_has_fxsr = emulator_guest_has_fxsr,
8621 .guest_has_rdpid = emulator_guest_has_rdpid,
8622 .set_nmi_mask = emulator_set_nmi_mask,
8623 .is_smm = emulator_is_smm,
8624 .is_guest_mode = emulator_is_guest_mode,
8625 .leave_smm = emulator_leave_smm,
8626 .triple_fault = emulator_triple_fault,
8627 .set_xcr = emulator_set_xcr,
8628 .get_untagged_addr = emulator_get_untagged_addr,
8629 };
8630
toggle_interruptibility(struct kvm_vcpu * vcpu,u32 mask)8631 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
8632 {
8633 u32 int_shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
8634 /*
8635 * an sti; sti; sequence only disable interrupts for the first
8636 * instruction. So, if the last instruction, be it emulated or
8637 * not, left the system with the INT_STI flag enabled, it
8638 * means that the last instruction is an sti. We should not
8639 * leave the flag on in this case. The same goes for mov ss
8640 */
8641 if (int_shadow & mask)
8642 mask = 0;
8643 if (unlikely(int_shadow || mask)) {
8644 static_call(kvm_x86_set_interrupt_shadow)(vcpu, mask);
8645 if (!mask)
8646 kvm_make_request(KVM_REQ_EVENT, vcpu);
8647 }
8648 }
8649
inject_emulated_exception(struct kvm_vcpu * vcpu)8650 static void inject_emulated_exception(struct kvm_vcpu *vcpu)
8651 {
8652 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8653
8654 if (ctxt->exception.vector == PF_VECTOR)
8655 kvm_inject_emulated_page_fault(vcpu, &ctxt->exception);
8656 else if (ctxt->exception.error_code_valid)
8657 kvm_queue_exception_e(vcpu, ctxt->exception.vector,
8658 ctxt->exception.error_code);
8659 else
8660 kvm_queue_exception(vcpu, ctxt->exception.vector);
8661 }
8662
alloc_emulate_ctxt(struct kvm_vcpu * vcpu)8663 static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu)
8664 {
8665 struct x86_emulate_ctxt *ctxt;
8666
8667 ctxt = kmem_cache_zalloc(x86_emulator_cache, GFP_KERNEL_ACCOUNT);
8668 if (!ctxt) {
8669 pr_err("failed to allocate vcpu's emulator\n");
8670 return NULL;
8671 }
8672
8673 ctxt->vcpu = vcpu;
8674 ctxt->ops = &emulate_ops;
8675 vcpu->arch.emulate_ctxt = ctxt;
8676
8677 return ctxt;
8678 }
8679
init_emulate_ctxt(struct kvm_vcpu * vcpu)8680 static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
8681 {
8682 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8683 int cs_db, cs_l;
8684
8685 static_call(kvm_x86_get_cs_db_l_bits)(vcpu, &cs_db, &cs_l);
8686
8687 ctxt->gpa_available = false;
8688 ctxt->eflags = kvm_get_rflags(vcpu);
8689 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
8690
8691 ctxt->eip = kvm_rip_read(vcpu);
8692 ctxt->mode = (!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
8693 (ctxt->eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_VM86 :
8694 (cs_l && is_long_mode(vcpu)) ? X86EMUL_MODE_PROT64 :
8695 cs_db ? X86EMUL_MODE_PROT32 :
8696 X86EMUL_MODE_PROT16;
8697 ctxt->interruptibility = 0;
8698 ctxt->have_exception = false;
8699 ctxt->exception.vector = -1;
8700 ctxt->perm_ok = false;
8701
8702 init_decode_cache(ctxt);
8703 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
8704 }
8705
kvm_inject_realmode_interrupt(struct kvm_vcpu * vcpu,int irq,int inc_eip)8706 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
8707 {
8708 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8709 int ret;
8710
8711 init_emulate_ctxt(vcpu);
8712
8713 ctxt->op_bytes = 2;
8714 ctxt->ad_bytes = 2;
8715 ctxt->_eip = ctxt->eip + inc_eip;
8716 ret = emulate_int_real(ctxt, irq);
8717
8718 if (ret != X86EMUL_CONTINUE) {
8719 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
8720 } else {
8721 ctxt->eip = ctxt->_eip;
8722 kvm_rip_write(vcpu, ctxt->eip);
8723 kvm_set_rflags(vcpu, ctxt->eflags);
8724 }
8725 }
8726 EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt);
8727
prepare_emulation_failure_exit(struct kvm_vcpu * vcpu,u64 * data,u8 ndata,u8 * insn_bytes,u8 insn_size)8728 static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
8729 u8 ndata, u8 *insn_bytes, u8 insn_size)
8730 {
8731 struct kvm_run *run = vcpu->run;
8732 u64 info[5];
8733 u8 info_start;
8734
8735 /*
8736 * Zero the whole array used to retrieve the exit info, as casting to
8737 * u32 for select entries will leave some chunks uninitialized.
8738 */
8739 memset(&info, 0, sizeof(info));
8740
8741 static_call(kvm_x86_get_exit_info)(vcpu, (u32 *)&info[0], &info[1],
8742 &info[2], (u32 *)&info[3],
8743 (u32 *)&info[4]);
8744
8745 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
8746 run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION;
8747
8748 /*
8749 * There's currently space for 13 entries, but 5 are used for the exit
8750 * reason and info. Restrict to 4 to reduce the maintenance burden
8751 * when expanding kvm_run.emulation_failure in the future.
8752 */
8753 if (WARN_ON_ONCE(ndata > 4))
8754 ndata = 4;
8755
8756 /* Always include the flags as a 'data' entry. */
8757 info_start = 1;
8758 run->emulation_failure.flags = 0;
8759
8760 if (insn_size) {
8761 BUILD_BUG_ON((sizeof(run->emulation_failure.insn_size) +
8762 sizeof(run->emulation_failure.insn_bytes) != 16));
8763 info_start += 2;
8764 run->emulation_failure.flags |=
8765 KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES;
8766 run->emulation_failure.insn_size = insn_size;
8767 memset(run->emulation_failure.insn_bytes, 0x90,
8768 sizeof(run->emulation_failure.insn_bytes));
8769 memcpy(run->emulation_failure.insn_bytes, insn_bytes, insn_size);
8770 }
8771
8772 memcpy(&run->internal.data[info_start], info, sizeof(info));
8773 memcpy(&run->internal.data[info_start + ARRAY_SIZE(info)], data,
8774 ndata * sizeof(data[0]));
8775
8776 run->emulation_failure.ndata = info_start + ARRAY_SIZE(info) + ndata;
8777 }
8778
prepare_emulation_ctxt_failure_exit(struct kvm_vcpu * vcpu)8779 static void prepare_emulation_ctxt_failure_exit(struct kvm_vcpu *vcpu)
8780 {
8781 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
8782
8783 prepare_emulation_failure_exit(vcpu, NULL, 0, ctxt->fetch.data,
8784 ctxt->fetch.end - ctxt->fetch.data);
8785 }
8786
__kvm_prepare_emulation_failure_exit(struct kvm_vcpu * vcpu,u64 * data,u8 ndata)8787 void __kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu, u64 *data,
8788 u8 ndata)
8789 {
8790 prepare_emulation_failure_exit(vcpu, data, ndata, NULL, 0);
8791 }
8792 EXPORT_SYMBOL_GPL(__kvm_prepare_emulation_failure_exit);
8793
kvm_prepare_emulation_failure_exit(struct kvm_vcpu * vcpu)8794 void kvm_prepare_emulation_failure_exit(struct kvm_vcpu *vcpu)
8795 {
8796 __kvm_prepare_emulation_failure_exit(vcpu, NULL, 0);
8797 }
8798 EXPORT_SYMBOL_GPL(kvm_prepare_emulation_failure_exit);
8799
handle_emulation_failure(struct kvm_vcpu * vcpu,int emulation_type)8800 static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type)
8801 {
8802 struct kvm *kvm = vcpu->kvm;
8803
8804 ++vcpu->stat.insn_emulation_fail;
8805 trace_kvm_emulate_insn_failed(vcpu);
8806
8807 if (emulation_type & EMULTYPE_VMWARE_GP) {
8808 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
8809 return 1;
8810 }
8811
8812 if (kvm->arch.exit_on_emulation_error ||
8813 (emulation_type & EMULTYPE_SKIP)) {
8814 prepare_emulation_ctxt_failure_exit(vcpu);
8815 return 0;
8816 }
8817
8818 kvm_queue_exception(vcpu, UD_VECTOR);
8819
8820 if (!is_guest_mode(vcpu) && static_call(kvm_x86_get_cpl)(vcpu) == 0) {
8821 prepare_emulation_ctxt_failure_exit(vcpu);
8822 return 0;
8823 }
8824
8825 return 1;
8826 }
8827
reexecute_instruction(struct kvm_vcpu * vcpu,gpa_t cr2_or_gpa,int emulation_type)8828 static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
8829 int emulation_type)
8830 {
8831 gpa_t gpa = cr2_or_gpa;
8832 kvm_pfn_t pfn;
8833
8834 if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
8835 return false;
8836
8837 if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8838 WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
8839 return false;
8840
8841 if (!vcpu->arch.mmu->root_role.direct) {
8842 /*
8843 * Write permission should be allowed since only
8844 * write access need to be emulated.
8845 */
8846 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
8847
8848 /*
8849 * If the mapping is invalid in guest, let cpu retry
8850 * it to generate fault.
8851 */
8852 if (gpa == INVALID_GPA)
8853 return true;
8854 }
8855
8856 /*
8857 * Do not retry the unhandleable instruction if it faults on the
8858 * readonly host memory, otherwise it will goto a infinite loop:
8859 * retry instruction -> write #PF -> emulation fail -> retry
8860 * instruction -> ...
8861 */
8862 pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
8863
8864 /*
8865 * If the instruction failed on the error pfn, it can not be fixed,
8866 * report the error to userspace.
8867 */
8868 if (is_error_noslot_pfn(pfn))
8869 return false;
8870
8871 kvm_release_pfn_clean(pfn);
8872
8873 /*
8874 * If emulation may have been triggered by a write to a shadowed page
8875 * table, unprotect the gfn (zap any relevant SPTEs) and re-enter the
8876 * guest to let the CPU re-execute the instruction in the hope that the
8877 * CPU can cleanly execute the instruction that KVM failed to emulate.
8878 */
8879 if (vcpu->kvm->arch.indirect_shadow_pages)
8880 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8881
8882 /*
8883 * If the failed instruction faulted on an access to page tables that
8884 * are used to translate any part of the instruction, KVM can't resolve
8885 * the issue by unprotecting the gfn, as zapping the shadow page will
8886 * result in the instruction taking a !PRESENT page fault and thus put
8887 * the vCPU into an infinite loop of page faults. E.g. KVM will create
8888 * a SPTE and write-protect the gfn to resolve the !PRESENT fault, and
8889 * then zap the SPTE to unprotect the gfn, and then do it all over
8890 * again. Report the error to userspace.
8891 */
8892 return !(emulation_type & EMULTYPE_WRITE_PF_TO_SP);
8893 }
8894
retry_instruction(struct x86_emulate_ctxt * ctxt,gpa_t cr2_or_gpa,int emulation_type)8895 static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
8896 gpa_t cr2_or_gpa, int emulation_type)
8897 {
8898 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
8899 unsigned long last_retry_eip, last_retry_addr, gpa = cr2_or_gpa;
8900
8901 last_retry_eip = vcpu->arch.last_retry_eip;
8902 last_retry_addr = vcpu->arch.last_retry_addr;
8903
8904 /*
8905 * If the emulation is caused by #PF and it is non-page_table
8906 * writing instruction, it means the VM-EXIT is caused by shadow
8907 * page protected, we can zap the shadow page and retry this
8908 * instruction directly.
8909 *
8910 * Note: if the guest uses a non-page-table modifying instruction
8911 * on the PDE that points to the instruction, then we will unmap
8912 * the instruction and go to an infinite loop. So, we cache the
8913 * last retried eip and the last fault address, if we meet the eip
8914 * and the address again, we can break out of the potential infinite
8915 * loop.
8916 */
8917 vcpu->arch.last_retry_eip = vcpu->arch.last_retry_addr = 0;
8918
8919 if (!(emulation_type & EMULTYPE_ALLOW_RETRY_PF))
8920 return false;
8921
8922 if (WARN_ON_ONCE(is_guest_mode(vcpu)) ||
8923 WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF)))
8924 return false;
8925
8926 if (x86_page_table_writing_insn(ctxt))
8927 return false;
8928
8929 if (ctxt->eip == last_retry_eip && last_retry_addr == cr2_or_gpa)
8930 return false;
8931
8932 vcpu->arch.last_retry_eip = ctxt->eip;
8933 vcpu->arch.last_retry_addr = cr2_or_gpa;
8934
8935 if (!vcpu->arch.mmu->root_role.direct)
8936 gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL);
8937
8938 kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa));
8939
8940 return true;
8941 }
8942
8943 static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
8944 static int complete_emulated_pio(struct kvm_vcpu *vcpu);
8945
kvm_vcpu_check_hw_bp(unsigned long addr,u32 type,u32 dr7,unsigned long * db)8946 static int kvm_vcpu_check_hw_bp(unsigned long addr, u32 type, u32 dr7,
8947 unsigned long *db)
8948 {
8949 u32 dr6 = 0;
8950 int i;
8951 u32 enable, rwlen;
8952
8953 enable = dr7;
8954 rwlen = dr7 >> 16;
8955 for (i = 0; i < 4; i++, enable >>= 2, rwlen >>= 4)
8956 if ((enable & 3) && (rwlen & 15) == type && db[i] == addr)
8957 dr6 |= (1 << i);
8958 return dr6;
8959 }
8960
kvm_vcpu_do_singlestep(struct kvm_vcpu * vcpu)8961 static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu)
8962 {
8963 struct kvm_run *kvm_run = vcpu->run;
8964
8965 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
8966 kvm_run->debug.arch.dr6 = DR6_BS | DR6_ACTIVE_LOW;
8967 kvm_run->debug.arch.pc = kvm_get_linear_rip(vcpu);
8968 kvm_run->debug.arch.exception = DB_VECTOR;
8969 kvm_run->exit_reason = KVM_EXIT_DEBUG;
8970 return 0;
8971 }
8972 kvm_queue_exception_p(vcpu, DB_VECTOR, DR6_BS);
8973 return 1;
8974 }
8975
kvm_skip_emulated_instruction(struct kvm_vcpu * vcpu)8976 int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
8977 {
8978 unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
8979 int r;
8980
8981 r = static_call(kvm_x86_skip_emulated_instruction)(vcpu);
8982 if (unlikely(!r))
8983 return 0;
8984
8985 kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED);
8986
8987 /*
8988 * rflags is the old, "raw" value of the flags. The new value has
8989 * not been saved yet.
8990 *
8991 * This is correct even for TF set by the guest, because "the
8992 * processor will not generate this exception after the instruction
8993 * that sets the TF flag".
8994 */
8995 if (unlikely(rflags & X86_EFLAGS_TF))
8996 r = kvm_vcpu_do_singlestep(vcpu);
8997 return r;
8998 }
8999 EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction);
9000
kvm_is_code_breakpoint_inhibited(struct kvm_vcpu * vcpu)9001 static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu)
9002 {
9003 u32 shadow;
9004
9005 if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF)
9006 return true;
9007
9008 /*
9009 * Intel CPUs inhibit code #DBs when MOV/POP SS blocking is active,
9010 * but AMD CPUs do not. MOV/POP SS blocking is rare, check that first
9011 * to avoid the relatively expensive CPUID lookup.
9012 */
9013 shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu);
9014 return (shadow & KVM_X86_SHADOW_INT_MOV_SS) &&
9015 guest_cpuid_is_intel(vcpu);
9016 }
9017
kvm_vcpu_check_code_breakpoint(struct kvm_vcpu * vcpu,int emulation_type,int * r)9018 static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu,
9019 int emulation_type, int *r)
9020 {
9021 WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE);
9022
9023 /*
9024 * Do not check for code breakpoints if hardware has already done the
9025 * checks, as inferred from the emulation type. On NO_DECODE and SKIP,
9026 * the instruction has passed all exception checks, and all intercepted
9027 * exceptions that trigger emulation have lower priority than code
9028 * breakpoints, i.e. the fact that the intercepted exception occurred
9029 * means any code breakpoints have already been serviced.
9030 *
9031 * Note, KVM needs to check for code #DBs on EMULTYPE_TRAP_UD_FORCED as
9032 * hardware has checked the RIP of the magic prefix, but not the RIP of
9033 * the instruction being emulated. The intent of forced emulation is
9034 * to behave as if KVM intercepted the instruction without an exception
9035 * and without a prefix.
9036 */
9037 if (emulation_type & (EMULTYPE_NO_DECODE | EMULTYPE_SKIP |
9038 EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF))
9039 return false;
9040
9041 if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) &&
9042 (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) {
9043 struct kvm_run *kvm_run = vcpu->run;
9044 unsigned long eip = kvm_get_linear_rip(vcpu);
9045 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
9046 vcpu->arch.guest_debug_dr7,
9047 vcpu->arch.eff_db);
9048
9049 if (dr6 != 0) {
9050 kvm_run->debug.arch.dr6 = dr6 | DR6_ACTIVE_LOW;
9051 kvm_run->debug.arch.pc = eip;
9052 kvm_run->debug.arch.exception = DB_VECTOR;
9053 kvm_run->exit_reason = KVM_EXIT_DEBUG;
9054 *r = 0;
9055 return true;
9056 }
9057 }
9058
9059 if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) &&
9060 !kvm_is_code_breakpoint_inhibited(vcpu)) {
9061 unsigned long eip = kvm_get_linear_rip(vcpu);
9062 u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0,
9063 vcpu->arch.dr7,
9064 vcpu->arch.db);
9065
9066 if (dr6 != 0) {
9067 kvm_queue_exception_p(vcpu, DB_VECTOR, dr6);
9068 *r = 1;
9069 return true;
9070 }
9071 }
9072
9073 return false;
9074 }
9075
is_vmware_backdoor_opcode(struct x86_emulate_ctxt * ctxt)9076 static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
9077 {
9078 switch (ctxt->opcode_len) {
9079 case 1:
9080 switch (ctxt->b) {
9081 case 0xe4: /* IN */
9082 case 0xe5:
9083 case 0xec:
9084 case 0xed:
9085 case 0xe6: /* OUT */
9086 case 0xe7:
9087 case 0xee:
9088 case 0xef:
9089 case 0x6c: /* INS */
9090 case 0x6d:
9091 case 0x6e: /* OUTS */
9092 case 0x6f:
9093 return true;
9094 }
9095 break;
9096 case 2:
9097 switch (ctxt->b) {
9098 case 0x33: /* RDPMC */
9099 return true;
9100 }
9101 break;
9102 }
9103
9104 return false;
9105 }
9106
9107 /*
9108 * Decode an instruction for emulation. The caller is responsible for handling
9109 * code breakpoints. Note, manually detecting code breakpoints is unnecessary
9110 * (and wrong) when emulating on an intercepted fault-like exception[*], as
9111 * code breakpoints have higher priority and thus have already been done by
9112 * hardware.
9113 *
9114 * [*] Except #MC, which is higher priority, but KVM should never emulate in
9115 * response to a machine check.
9116 */
x86_decode_emulated_instruction(struct kvm_vcpu * vcpu,int emulation_type,void * insn,int insn_len)9117 int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type,
9118 void *insn, int insn_len)
9119 {
9120 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
9121 int r;
9122
9123 init_emulate_ctxt(vcpu);
9124
9125 r = x86_decode_insn(ctxt, insn, insn_len, emulation_type);
9126
9127 trace_kvm_emulate_insn_start(vcpu);
9128 ++vcpu->stat.insn_emulation;
9129
9130 return r;
9131 }
9132 EXPORT_SYMBOL_GPL(x86_decode_emulated_instruction);
9133
x86_emulate_instruction(struct kvm_vcpu * vcpu,gpa_t cr2_or_gpa,int emulation_type,void * insn,int insn_len)9134 int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
9135 int emulation_type, void *insn, int insn_len)
9136 {
9137 int r;
9138 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
9139 bool writeback = true;
9140
9141 r = kvm_check_emulate_insn(vcpu, emulation_type, insn, insn_len);
9142 if (r != X86EMUL_CONTINUE) {
9143 if (r == X86EMUL_RETRY_INSTR || r == X86EMUL_PROPAGATE_FAULT)
9144 return 1;
9145
9146 WARN_ON_ONCE(r != X86EMUL_UNHANDLEABLE);
9147 return handle_emulation_failure(vcpu, emulation_type);
9148 }
9149
9150 vcpu->arch.l1tf_flush_l1d = true;
9151
9152 if (!(emulation_type & EMULTYPE_NO_DECODE)) {
9153 kvm_clear_exception_queue(vcpu);
9154
9155 /*
9156 * Return immediately if RIP hits a code breakpoint, such #DBs
9157 * are fault-like and are higher priority than any faults on
9158 * the code fetch itself.
9159 */
9160 if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r))
9161 return r;
9162
9163 r = x86_decode_emulated_instruction(vcpu, emulation_type,
9164 insn, insn_len);
9165 if (r != EMULATION_OK) {
9166 if ((emulation_type & EMULTYPE_TRAP_UD) ||
9167 (emulation_type & EMULTYPE_TRAP_UD_FORCED)) {
9168 kvm_queue_exception(vcpu, UD_VECTOR);
9169 return 1;
9170 }
9171 if (reexecute_instruction(vcpu, cr2_or_gpa,
9172 emulation_type))
9173 return 1;
9174
9175 if (ctxt->have_exception &&
9176 !(emulation_type & EMULTYPE_SKIP)) {
9177 /*
9178 * #UD should result in just EMULATION_FAILED, and trap-like
9179 * exception should not be encountered during decode.
9180 */
9181 WARN_ON_ONCE(ctxt->exception.vector == UD_VECTOR ||
9182 exception_type(ctxt->exception.vector) == EXCPT_TRAP);
9183 inject_emulated_exception(vcpu);
9184 return 1;
9185 }
9186 return handle_emulation_failure(vcpu, emulation_type);
9187 }
9188 }
9189
9190 if ((emulation_type & EMULTYPE_VMWARE_GP) &&
9191 !is_vmware_backdoor_opcode(ctxt)) {
9192 kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
9193 return 1;
9194 }
9195
9196 /*
9197 * EMULTYPE_SKIP without EMULTYPE_COMPLETE_USER_EXIT is intended for
9198 * use *only* by vendor callbacks for kvm_skip_emulated_instruction().
9199 * The caller is responsible for updating interruptibility state and
9200 * injecting single-step #DBs.
9201 */
9202 if (emulation_type & EMULTYPE_SKIP) {
9203 if (ctxt->mode != X86EMUL_MODE_PROT64)
9204 ctxt->eip = (u32)ctxt->_eip;
9205 else
9206 ctxt->eip = ctxt->_eip;
9207
9208 if (emulation_type & EMULTYPE_COMPLETE_USER_EXIT) {
9209 r = 1;
9210 goto writeback;
9211 }
9212
9213 kvm_rip_write(vcpu, ctxt->eip);
9214 if (ctxt->eflags & X86_EFLAGS_RF)
9215 kvm_set_rflags(vcpu, ctxt->eflags & ~X86_EFLAGS_RF);
9216 return 1;
9217 }
9218
9219 if (retry_instruction(ctxt, cr2_or_gpa, emulation_type))
9220 return 1;
9221
9222 /* this is needed for vmware backdoor interface to work since it
9223 changes registers values during IO operation */
9224 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
9225 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
9226 emulator_invalidate_register_cache(ctxt);
9227 }
9228
9229 restart:
9230 if (emulation_type & EMULTYPE_PF) {
9231 /* Save the faulting GPA (cr2) in the address field */
9232 ctxt->exception.address = cr2_or_gpa;
9233
9234 /* With shadow page tables, cr2 contains a GVA or nGPA. */
9235 if (vcpu->arch.mmu->root_role.direct) {
9236 ctxt->gpa_available = true;
9237 ctxt->gpa_val = cr2_or_gpa;
9238 }
9239 } else {
9240 /* Sanitize the address out of an abundance of paranoia. */
9241 ctxt->exception.address = 0;
9242 }
9243
9244 r = x86_emulate_insn(ctxt);
9245
9246 if (r == EMULATION_INTERCEPTED)
9247 return 1;
9248
9249 if (r == EMULATION_FAILED) {
9250 if (reexecute_instruction(vcpu, cr2_or_gpa, emulation_type))
9251 return 1;
9252
9253 return handle_emulation_failure(vcpu, emulation_type);
9254 }
9255
9256 if (ctxt->have_exception) {
9257 WARN_ON_ONCE(vcpu->mmio_needed && !vcpu->mmio_is_write);
9258 vcpu->mmio_needed = false;
9259 r = 1;
9260 inject_emulated_exception(vcpu);
9261 } else if (vcpu->arch.pio.count) {
9262 if (!vcpu->arch.pio.in) {
9263 /* FIXME: return into emulator if single-stepping. */
9264 vcpu->arch.pio.count = 0;
9265 } else {
9266 writeback = false;
9267 vcpu->arch.complete_userspace_io = complete_emulated_pio;
9268 }
9269 r = 0;
9270 } else if (vcpu->mmio_needed) {
9271 ++vcpu->stat.mmio_exits;
9272
9273 if (!vcpu->mmio_is_write)
9274 writeback = false;
9275 r = 0;
9276 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
9277 } else if (vcpu->arch.complete_userspace_io) {
9278 writeback = false;
9279 r = 0;
9280 } else if (r == EMULATION_RESTART)
9281 goto restart;
9282 else
9283 r = 1;
9284
9285 writeback:
9286 if (writeback) {
9287 unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu);
9288 toggle_interruptibility(vcpu, ctxt->interruptibility);
9289 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
9290
9291 /*
9292 * Note, EXCPT_DB is assumed to be fault-like as the emulator
9293 * only supports code breakpoints and general detect #DB, both
9294 * of which are fault-like.
9295 */
9296 if (!ctxt->have_exception ||
9297 exception_type(ctxt->exception.vector) == EXCPT_TRAP) {
9298 kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.INSTRUCTIONS_RETIRED);
9299 if (ctxt->is_branch)
9300 kvm_pmu_trigger_event(vcpu, kvm_pmu_eventsel.BRANCH_INSTRUCTIONS_RETIRED);
9301 kvm_rip_write(vcpu, ctxt->eip);
9302 if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)))
9303 r = kvm_vcpu_do_singlestep(vcpu);
9304 static_call_cond(kvm_x86_update_emulated_instruction)(vcpu);
9305 __kvm_set_rflags(vcpu, ctxt->eflags);
9306 }
9307
9308 /*
9309 * For STI, interrupts are shadowed; so KVM_REQ_EVENT will
9310 * do nothing, and it will be requested again as soon as
9311 * the shadow expires. But we still need to check here,
9312 * because POPF has no interrupt shadow.
9313 */
9314 if (unlikely((ctxt->eflags & ~rflags) & X86_EFLAGS_IF))
9315 kvm_make_request(KVM_REQ_EVENT, vcpu);
9316 } else
9317 vcpu->arch.emulate_regs_need_sync_to_vcpu = true;
9318
9319 return r;
9320 }
9321
kvm_emulate_instruction(struct kvm_vcpu * vcpu,int emulation_type)9322 int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type)
9323 {
9324 return x86_emulate_instruction(vcpu, 0, emulation_type, NULL, 0);
9325 }
9326 EXPORT_SYMBOL_GPL(kvm_emulate_instruction);
9327
kvm_emulate_instruction_from_buffer(struct kvm_vcpu * vcpu,void * insn,int insn_len)9328 int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
9329 void *insn, int insn_len)
9330 {
9331 return x86_emulate_instruction(vcpu, 0, 0, insn, insn_len);
9332 }
9333 EXPORT_SYMBOL_GPL(kvm_emulate_instruction_from_buffer);
9334
complete_fast_pio_out_port_0x7e(struct kvm_vcpu * vcpu)9335 static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
9336 {
9337 vcpu->arch.pio.count = 0;
9338 return 1;
9339 }
9340
complete_fast_pio_out(struct kvm_vcpu * vcpu)9341 static int complete_fast_pio_out(struct kvm_vcpu *vcpu)
9342 {
9343 vcpu->arch.pio.count = 0;
9344
9345 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip)))
9346 return 1;
9347
9348 return kvm_skip_emulated_instruction(vcpu);
9349 }
9350
kvm_fast_pio_out(struct kvm_vcpu * vcpu,int size,unsigned short port)9351 static int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size,
9352 unsigned short port)
9353 {
9354 unsigned long val = kvm_rax_read(vcpu);
9355 int ret = emulator_pio_out(vcpu, size, port, &val, 1);
9356
9357 if (ret)
9358 return ret;
9359
9360 /*
9361 * Workaround userspace that relies on old KVM behavior of %rip being
9362 * incremented prior to exiting to userspace to handle "OUT 0x7e".
9363 */
9364 if (port == 0x7e &&
9365 kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_OUT_7E_INC_RIP)) {
9366 vcpu->arch.complete_userspace_io =
9367 complete_fast_pio_out_port_0x7e;
9368 kvm_skip_emulated_instruction(vcpu);
9369 } else {
9370 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
9371 vcpu->arch.complete_userspace_io = complete_fast_pio_out;
9372 }
9373 return 0;
9374 }
9375
complete_fast_pio_in(struct kvm_vcpu * vcpu)9376 static int complete_fast_pio_in(struct kvm_vcpu *vcpu)
9377 {
9378 unsigned long val;
9379
9380 /* We should only ever be called with arch.pio.count equal to 1 */
9381 BUG_ON(vcpu->arch.pio.count != 1);
9382
9383 if (unlikely(!kvm_is_linear_rip(vcpu, vcpu->arch.pio.linear_rip))) {
9384 vcpu->arch.pio.count = 0;
9385 return 1;
9386 }
9387
9388 /* For size less than 4 we merge, else we zero extend */
9389 val = (vcpu->arch.pio.size < 4) ? kvm_rax_read(vcpu) : 0;
9390
9391 complete_emulator_pio_in(vcpu, &val);
9392 kvm_rax_write(vcpu, val);
9393
9394 return kvm_skip_emulated_instruction(vcpu);
9395 }
9396
kvm_fast_pio_in(struct kvm_vcpu * vcpu,int size,unsigned short port)9397 static int kvm_fast_pio_in(struct kvm_vcpu *vcpu, int size,
9398 unsigned short port)
9399 {
9400 unsigned long val;
9401 int ret;
9402
9403 /* For size less than 4 we merge, else we zero extend */
9404 val = (size < 4) ? kvm_rax_read(vcpu) : 0;
9405
9406 ret = emulator_pio_in(vcpu, size, port, &val, 1);
9407 if (ret) {
9408 kvm_rax_write(vcpu, val);
9409 return ret;
9410 }
9411
9412 vcpu->arch.pio.linear_rip = kvm_get_linear_rip(vcpu);
9413 vcpu->arch.complete_userspace_io = complete_fast_pio_in;
9414
9415 return 0;
9416 }
9417
kvm_fast_pio(struct kvm_vcpu * vcpu,int size,unsigned short port,int in)9418 int kvm_fast_pio(struct kvm_vcpu *vcpu, int size, unsigned short port, int in)
9419 {
9420 int ret;
9421
9422 if (in)
9423 ret = kvm_fast_pio_in(vcpu, size, port);
9424 else
9425 ret = kvm_fast_pio_out(vcpu, size, port);
9426 return ret && kvm_skip_emulated_instruction(vcpu);
9427 }
9428 EXPORT_SYMBOL_GPL(kvm_fast_pio);
9429
kvmclock_cpu_down_prep(unsigned int cpu)9430 static int kvmclock_cpu_down_prep(unsigned int cpu)
9431 {
9432 __this_cpu_write(cpu_tsc_khz, 0);
9433 return 0;
9434 }
9435
tsc_khz_changed(void * data)9436 static void tsc_khz_changed(void *data)
9437 {
9438 struct cpufreq_freqs *freq = data;
9439 unsigned long khz;
9440
9441 WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_CONSTANT_TSC));
9442
9443 if (data)
9444 khz = freq->new;
9445 else
9446 khz = cpufreq_quick_get(raw_smp_processor_id());
9447 if (!khz)
9448 khz = tsc_khz;
9449 __this_cpu_write(cpu_tsc_khz, khz);
9450 }
9451
9452 #ifdef CONFIG_X86_64
kvm_hyperv_tsc_notifier(void)9453 static void kvm_hyperv_tsc_notifier(void)
9454 {
9455 struct kvm *kvm;
9456 int cpu;
9457
9458 mutex_lock(&kvm_lock);
9459 list_for_each_entry(kvm, &vm_list, vm_list)
9460 kvm_make_mclock_inprogress_request(kvm);
9461
9462 /* no guest entries from this point */
9463 hyperv_stop_tsc_emulation();
9464
9465 /* TSC frequency always matches when on Hyper-V */
9466 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
9467 for_each_present_cpu(cpu)
9468 per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
9469 }
9470 kvm_caps.max_guest_tsc_khz = tsc_khz;
9471
9472 list_for_each_entry(kvm, &vm_list, vm_list) {
9473 __kvm_start_pvclock_update(kvm);
9474 pvclock_update_vm_gtod_copy(kvm);
9475 kvm_end_pvclock_update(kvm);
9476 }
9477
9478 mutex_unlock(&kvm_lock);
9479 }
9480 #endif
9481
__kvmclock_cpufreq_notifier(struct cpufreq_freqs * freq,int cpu)9482 static void __kvmclock_cpufreq_notifier(struct cpufreq_freqs *freq, int cpu)
9483 {
9484 struct kvm *kvm;
9485 struct kvm_vcpu *vcpu;
9486 int send_ipi = 0;
9487 unsigned long i;
9488
9489 /*
9490 * We allow guests to temporarily run on slowing clocks,
9491 * provided we notify them after, or to run on accelerating
9492 * clocks, provided we notify them before. Thus time never
9493 * goes backwards.
9494 *
9495 * However, we have a problem. We can't atomically update
9496 * the frequency of a given CPU from this function; it is
9497 * merely a notifier, which can be called from any CPU.
9498 * Changing the TSC frequency at arbitrary points in time
9499 * requires a recomputation of local variables related to
9500 * the TSC for each VCPU. We must flag these local variables
9501 * to be updated and be sure the update takes place with the
9502 * new frequency before any guests proceed.
9503 *
9504 * Unfortunately, the combination of hotplug CPU and frequency
9505 * change creates an intractable locking scenario; the order
9506 * of when these callouts happen is undefined with respect to
9507 * CPU hotplug, and they can race with each other. As such,
9508 * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
9509 * undefined; you can actually have a CPU frequency change take
9510 * place in between the computation of X and the setting of the
9511 * variable. To protect against this problem, all updates of
9512 * the per_cpu tsc_khz variable are done in an interrupt
9513 * protected IPI, and all callers wishing to update the value
9514 * must wait for a synchronous IPI to complete (which is trivial
9515 * if the caller is on the CPU already). This establishes the
9516 * necessary total order on variable updates.
9517 *
9518 * Note that because a guest time update may take place
9519 * anytime after the setting of the VCPU's request bit, the
9520 * correct TSC value must be set before the request. However,
9521 * to ensure the update actually makes it to any guest which
9522 * starts running in hardware virtualization between the set
9523 * and the acquisition of the spinlock, we must also ping the
9524 * CPU after setting the request bit.
9525 *
9526 */
9527
9528 smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
9529
9530 mutex_lock(&kvm_lock);
9531 list_for_each_entry(kvm, &vm_list, vm_list) {
9532 kvm_for_each_vcpu(i, vcpu, kvm) {
9533 if (vcpu->cpu != cpu)
9534 continue;
9535 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
9536 if (vcpu->cpu != raw_smp_processor_id())
9537 send_ipi = 1;
9538 }
9539 }
9540 mutex_unlock(&kvm_lock);
9541
9542 if (freq->old < freq->new && send_ipi) {
9543 /*
9544 * We upscale the frequency. Must make the guest
9545 * doesn't see old kvmclock values while running with
9546 * the new frequency, otherwise we risk the guest sees
9547 * time go backwards.
9548 *
9549 * In case we update the frequency for another cpu
9550 * (which might be in guest context) send an interrupt
9551 * to kick the cpu out of guest context. Next time
9552 * guest context is entered kvmclock will be updated,
9553 * so the guest will not see stale values.
9554 */
9555 smp_call_function_single(cpu, tsc_khz_changed, freq, 1);
9556 }
9557 }
9558
kvmclock_cpufreq_notifier(struct notifier_block * nb,unsigned long val,void * data)9559 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
9560 void *data)
9561 {
9562 struct cpufreq_freqs *freq = data;
9563 int cpu;
9564
9565 if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
9566 return 0;
9567 if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
9568 return 0;
9569
9570 for_each_cpu(cpu, freq->policy->cpus)
9571 __kvmclock_cpufreq_notifier(freq, cpu);
9572
9573 return 0;
9574 }
9575
9576 static struct notifier_block kvmclock_cpufreq_notifier_block = {
9577 .notifier_call = kvmclock_cpufreq_notifier
9578 };
9579
kvmclock_cpu_online(unsigned int cpu)9580 static int kvmclock_cpu_online(unsigned int cpu)
9581 {
9582 tsc_khz_changed(NULL);
9583 return 0;
9584 }
9585
kvm_timer_init(void)9586 static void kvm_timer_init(void)
9587 {
9588 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
9589 max_tsc_khz = tsc_khz;
9590
9591 if (IS_ENABLED(CONFIG_CPU_FREQ)) {
9592 struct cpufreq_policy *policy;
9593 int cpu;
9594
9595 cpu = get_cpu();
9596 policy = cpufreq_cpu_get(cpu);
9597 if (policy) {
9598 if (policy->cpuinfo.max_freq)
9599 max_tsc_khz = policy->cpuinfo.max_freq;
9600 cpufreq_cpu_put(policy);
9601 }
9602 put_cpu();
9603 }
9604 cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
9605 CPUFREQ_TRANSITION_NOTIFIER);
9606
9607 cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "x86/kvm/clk:online",
9608 kvmclock_cpu_online, kvmclock_cpu_down_prep);
9609 }
9610 }
9611
9612 #ifdef CONFIG_X86_64
pvclock_gtod_update_fn(struct work_struct * work)9613 static void pvclock_gtod_update_fn(struct work_struct *work)
9614 {
9615 struct kvm *kvm;
9616 struct kvm_vcpu *vcpu;
9617 unsigned long i;
9618
9619 mutex_lock(&kvm_lock);
9620 list_for_each_entry(kvm, &vm_list, vm_list)
9621 kvm_for_each_vcpu(i, vcpu, kvm)
9622 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
9623 atomic_set(&kvm_guest_has_master_clock, 0);
9624 mutex_unlock(&kvm_lock);
9625 }
9626
9627 static DECLARE_WORK(pvclock_gtod_work, pvclock_gtod_update_fn);
9628
9629 /*
9630 * Indirection to move queue_work() out of the tk_core.seq write held
9631 * region to prevent possible deadlocks against time accessors which
9632 * are invoked with work related locks held.
9633 */
pvclock_irq_work_fn(struct irq_work * w)9634 static void pvclock_irq_work_fn(struct irq_work *w)
9635 {
9636 queue_work(system_long_wq, &pvclock_gtod_work);
9637 }
9638
9639 static DEFINE_IRQ_WORK(pvclock_irq_work, pvclock_irq_work_fn);
9640
9641 /*
9642 * Notification about pvclock gtod data update.
9643 */
pvclock_gtod_notify(struct notifier_block * nb,unsigned long unused,void * priv)9644 static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
9645 void *priv)
9646 {
9647 struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
9648 struct timekeeper *tk = priv;
9649
9650 update_pvclock_gtod(tk);
9651
9652 /*
9653 * Disable master clock if host does not trust, or does not use,
9654 * TSC based clocksource. Delegate queue_work() to irq_work as
9655 * this is invoked with tk_core.seq write held.
9656 */
9657 if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
9658 atomic_read(&kvm_guest_has_master_clock) != 0)
9659 irq_work_queue(&pvclock_irq_work);
9660 return 0;
9661 }
9662
9663 static struct notifier_block pvclock_gtod_notifier = {
9664 .notifier_call = pvclock_gtod_notify,
9665 };
9666 #endif
9667
kvm_ops_update(struct kvm_x86_init_ops * ops)9668 static inline void kvm_ops_update(struct kvm_x86_init_ops *ops)
9669 {
9670 memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops));
9671
9672 #define __KVM_X86_OP(func) \
9673 static_call_update(kvm_x86_##func, kvm_x86_ops.func);
9674 #define KVM_X86_OP(func) \
9675 WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func)
9676 #define KVM_X86_OP_OPTIONAL __KVM_X86_OP
9677 #define KVM_X86_OP_OPTIONAL_RET0(func) \
9678 static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \
9679 (void *)__static_call_return0);
9680 #include <asm/kvm-x86-ops.h>
9681 #undef __KVM_X86_OP
9682
9683 kvm_pmu_ops_update(ops->pmu_ops);
9684 }
9685
kvm_x86_check_processor_compatibility(void)9686 static int kvm_x86_check_processor_compatibility(void)
9687 {
9688 int cpu = smp_processor_id();
9689 struct cpuinfo_x86 *c = &cpu_data(cpu);
9690
9691 /*
9692 * Compatibility checks are done when loading KVM and when enabling
9693 * hardware, e.g. during CPU hotplug, to ensure all online CPUs are
9694 * compatible, i.e. KVM should never perform a compatibility check on
9695 * an offline CPU.
9696 */
9697 WARN_ON(!cpu_online(cpu));
9698
9699 if (__cr4_reserved_bits(cpu_has, c) !=
9700 __cr4_reserved_bits(cpu_has, &boot_cpu_data))
9701 return -EIO;
9702
9703 return static_call(kvm_x86_check_processor_compatibility)();
9704 }
9705
kvm_x86_check_cpu_compat(void * ret)9706 static void kvm_x86_check_cpu_compat(void *ret)
9707 {
9708 *(int *)ret = kvm_x86_check_processor_compatibility();
9709 }
9710
kvm_x86_vendor_init(struct kvm_x86_init_ops * ops)9711 int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops)
9712 {
9713 u64 host_pat;
9714 int r, cpu;
9715
9716 guard(mutex)(&vendor_module_lock);
9717
9718 if (kvm_x86_ops.hardware_enable) {
9719 pr_err("already loaded vendor module '%s'\n", kvm_x86_ops.name);
9720 return -EEXIST;
9721 }
9722
9723 /*
9724 * KVM explicitly assumes that the guest has an FPU and
9725 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
9726 * vCPU's FPU state as a fxregs_state struct.
9727 */
9728 if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
9729 pr_err("inadequate fpu\n");
9730 return -EOPNOTSUPP;
9731 }
9732
9733 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
9734 pr_err("RT requires X86_FEATURE_CONSTANT_TSC\n");
9735 return -EOPNOTSUPP;
9736 }
9737
9738 /*
9739 * KVM assumes that PAT entry '0' encodes WB memtype and simply zeroes
9740 * the PAT bits in SPTEs. Bail if PAT[0] is programmed to something
9741 * other than WB. Note, EPT doesn't utilize the PAT, but don't bother
9742 * with an exception. PAT[0] is set to WB on RESET and also by the
9743 * kernel, i.e. failure indicates a kernel bug or broken firmware.
9744 */
9745 if (rdmsrl_safe(MSR_IA32_CR_PAT, &host_pat) ||
9746 (host_pat & GENMASK(2, 0)) != 6) {
9747 pr_err("host PAT[0] is not WB\n");
9748 return -EIO;
9749 }
9750
9751 memset(&kvm_caps, 0, sizeof(kvm_caps));
9752
9753 x86_emulator_cache = kvm_alloc_emulator_cache();
9754 if (!x86_emulator_cache) {
9755 pr_err("failed to allocate cache for x86 emulator\n");
9756 return -ENOMEM;
9757 }
9758
9759 user_return_msrs = alloc_percpu(struct kvm_user_return_msrs);
9760 if (!user_return_msrs) {
9761 pr_err("failed to allocate percpu kvm_user_return_msrs\n");
9762 r = -ENOMEM;
9763 goto out_free_x86_emulator_cache;
9764 }
9765 kvm_nr_uret_msrs = 0;
9766
9767 r = kvm_mmu_vendor_module_init();
9768 if (r)
9769 goto out_free_percpu;
9770
9771 kvm_caps.supported_vm_types = BIT(KVM_X86_DEFAULT_VM);
9772 kvm_caps.supported_mce_cap = MCG_CTL_P | MCG_SER_P;
9773
9774 if (boot_cpu_has(X86_FEATURE_XSAVE)) {
9775 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
9776 kvm_caps.supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
9777 }
9778
9779 rdmsrl_safe(MSR_EFER, &host_efer);
9780
9781 if (boot_cpu_has(X86_FEATURE_XSAVES))
9782 rdmsrl(MSR_IA32_XSS, host_xss);
9783
9784 kvm_init_pmu_capability(ops->pmu_ops);
9785
9786 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
9787 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, host_arch_capabilities);
9788
9789 r = ops->hardware_setup();
9790 if (r != 0)
9791 goto out_mmu_exit;
9792
9793 kvm_ops_update(ops);
9794
9795 for_each_online_cpu(cpu) {
9796 smp_call_function_single(cpu, kvm_x86_check_cpu_compat, &r, 1);
9797 if (r < 0)
9798 goto out_unwind_ops;
9799 }
9800
9801 /*
9802 * Point of no return! DO NOT add error paths below this point unless
9803 * absolutely necessary, as most operations from this point forward
9804 * require unwinding.
9805 */
9806 kvm_timer_init();
9807
9808 if (pi_inject_timer == -1)
9809 pi_inject_timer = housekeeping_enabled(HK_TYPE_TIMER);
9810 #ifdef CONFIG_X86_64
9811 pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
9812
9813 if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
9814 set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
9815 #endif
9816
9817 kvm_register_perf_callbacks(ops->handle_intel_pt_intr);
9818
9819 if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && tdp_mmu_enabled)
9820 kvm_caps.supported_vm_types |= BIT(KVM_X86_SW_PROTECTED_VM);
9821
9822 if (!kvm_cpu_cap_has(X86_FEATURE_XSAVES))
9823 kvm_caps.supported_xss = 0;
9824
9825 #define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
9826 cr4_reserved_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_);
9827 #undef __kvm_cpu_cap_has
9828
9829 if (kvm_caps.has_tsc_control) {
9830 /*
9831 * Make sure the user can only configure tsc_khz values that
9832 * fit into a signed integer.
9833 * A min value is not calculated because it will always
9834 * be 1 on all machines.
9835 */
9836 u64 max = min(0x7fffffffULL,
9837 __scale_tsc(kvm_caps.max_tsc_scaling_ratio, tsc_khz));
9838 kvm_caps.max_guest_tsc_khz = max;
9839 }
9840 kvm_caps.default_tsc_scaling_ratio = 1ULL << kvm_caps.tsc_scaling_ratio_frac_bits;
9841 kvm_init_msr_lists();
9842 return 0;
9843
9844 out_unwind_ops:
9845 kvm_x86_ops.hardware_enable = NULL;
9846 static_call(kvm_x86_hardware_unsetup)();
9847 out_mmu_exit:
9848 kvm_mmu_vendor_module_exit();
9849 out_free_percpu:
9850 free_percpu(user_return_msrs);
9851 out_free_x86_emulator_cache:
9852 kmem_cache_destroy(x86_emulator_cache);
9853 return r;
9854 }
9855 EXPORT_SYMBOL_GPL(kvm_x86_vendor_init);
9856
kvm_x86_vendor_exit(void)9857 void kvm_x86_vendor_exit(void)
9858 {
9859 kvm_unregister_perf_callbacks();
9860
9861 #ifdef CONFIG_X86_64
9862 if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
9863 clear_hv_tscchange_cb();
9864 #endif
9865 kvm_lapic_exit();
9866
9867 if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
9868 cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
9869 CPUFREQ_TRANSITION_NOTIFIER);
9870 cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
9871 }
9872 #ifdef CONFIG_X86_64
9873 pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
9874 irq_work_sync(&pvclock_irq_work);
9875 cancel_work_sync(&pvclock_gtod_work);
9876 #endif
9877 static_call(kvm_x86_hardware_unsetup)();
9878 kvm_mmu_vendor_module_exit();
9879 free_percpu(user_return_msrs);
9880 kmem_cache_destroy(x86_emulator_cache);
9881 #ifdef CONFIG_KVM_XEN
9882 static_key_deferred_flush(&kvm_xen_enabled);
9883 WARN_ON(static_branch_unlikely(&kvm_xen_enabled.key));
9884 #endif
9885 mutex_lock(&vendor_module_lock);
9886 kvm_x86_ops.hardware_enable = NULL;
9887 mutex_unlock(&vendor_module_lock);
9888 }
9889 EXPORT_SYMBOL_GPL(kvm_x86_vendor_exit);
9890
__kvm_emulate_halt(struct kvm_vcpu * vcpu,int state,int reason)9891 static int __kvm_emulate_halt(struct kvm_vcpu *vcpu, int state, int reason)
9892 {
9893 /*
9894 * The vCPU has halted, e.g. executed HLT. Update the run state if the
9895 * local APIC is in-kernel, the run loop will detect the non-runnable
9896 * state and halt the vCPU. Exit to userspace if the local APIC is
9897 * managed by userspace, in which case userspace is responsible for
9898 * handling wake events.
9899 */
9900 ++vcpu->stat.halt_exits;
9901 if (lapic_in_kernel(vcpu)) {
9902 vcpu->arch.mp_state = state;
9903 return 1;
9904 } else {
9905 vcpu->run->exit_reason = reason;
9906 return 0;
9907 }
9908 }
9909
kvm_emulate_halt_noskip(struct kvm_vcpu * vcpu)9910 int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)
9911 {
9912 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);
9913 }
9914 EXPORT_SYMBOL_GPL(kvm_emulate_halt_noskip);
9915
kvm_emulate_halt(struct kvm_vcpu * vcpu)9916 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
9917 {
9918 int ret = kvm_skip_emulated_instruction(vcpu);
9919 /*
9920 * TODO: we might be squashing a GUESTDBG_SINGLESTEP-triggered
9921 * KVM_EXIT_DEBUG here.
9922 */
9923 return kvm_emulate_halt_noskip(vcpu) && ret;
9924 }
9925 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
9926
kvm_emulate_ap_reset_hold(struct kvm_vcpu * vcpu)9927 int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)
9928 {
9929 int ret = kvm_skip_emulated_instruction(vcpu);
9930
9931 return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,
9932 KVM_EXIT_AP_RESET_HOLD) && ret;
9933 }
9934 EXPORT_SYMBOL_GPL(kvm_emulate_ap_reset_hold);
9935
9936 #ifdef CONFIG_X86_64
kvm_pv_clock_pairing(struct kvm_vcpu * vcpu,gpa_t paddr,unsigned long clock_type)9937 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
9938 unsigned long clock_type)
9939 {
9940 struct kvm_clock_pairing clock_pairing;
9941 struct timespec64 ts;
9942 u64 cycle;
9943 int ret;
9944
9945 if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
9946 return -KVM_EOPNOTSUPP;
9947
9948 /*
9949 * When tsc is in permanent catchup mode guests won't be able to use
9950 * pvclock_read_retry loop to get consistent view of pvclock
9951 */
9952 if (vcpu->arch.tsc_always_catchup)
9953 return -KVM_EOPNOTSUPP;
9954
9955 if (!kvm_get_walltime_and_clockread(&ts, &cycle))
9956 return -KVM_EOPNOTSUPP;
9957
9958 clock_pairing.sec = ts.tv_sec;
9959 clock_pairing.nsec = ts.tv_nsec;
9960 clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
9961 clock_pairing.flags = 0;
9962 memset(&clock_pairing.pad, 0, sizeof(clock_pairing.pad));
9963
9964 ret = 0;
9965 if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
9966 sizeof(struct kvm_clock_pairing)))
9967 ret = -KVM_EFAULT;
9968
9969 return ret;
9970 }
9971 #endif
9972
9973 /*
9974 * kvm_pv_kick_cpu_op: Kick a vcpu.
9975 *
9976 * @apicid - apicid of vcpu to be kicked.
9977 */
kvm_pv_kick_cpu_op(struct kvm * kvm,int apicid)9978 static void kvm_pv_kick_cpu_op(struct kvm *kvm, int apicid)
9979 {
9980 /*
9981 * All other fields are unused for APIC_DM_REMRD, but may be consumed by
9982 * common code, e.g. for tracing. Defer initialization to the compiler.
9983 */
9984 struct kvm_lapic_irq lapic_irq = {
9985 .delivery_mode = APIC_DM_REMRD,
9986 .dest_mode = APIC_DEST_PHYSICAL,
9987 .shorthand = APIC_DEST_NOSHORT,
9988 .dest_id = apicid,
9989 };
9990
9991 kvm_irq_delivery_to_apic(kvm, NULL, &lapic_irq, NULL);
9992 }
9993
kvm_apicv_activated(struct kvm * kvm)9994 bool kvm_apicv_activated(struct kvm *kvm)
9995 {
9996 return (READ_ONCE(kvm->arch.apicv_inhibit_reasons) == 0);
9997 }
9998 EXPORT_SYMBOL_GPL(kvm_apicv_activated);
9999
kvm_vcpu_apicv_activated(struct kvm_vcpu * vcpu)10000 bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu)
10001 {
10002 ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons);
10003 ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu);
10004
10005 return (vm_reasons | vcpu_reasons) == 0;
10006 }
10007 EXPORT_SYMBOL_GPL(kvm_vcpu_apicv_activated);
10008
set_or_clear_apicv_inhibit(unsigned long * inhibits,enum kvm_apicv_inhibit reason,bool set)10009 static void set_or_clear_apicv_inhibit(unsigned long *inhibits,
10010 enum kvm_apicv_inhibit reason, bool set)
10011 {
10012 if (set)
10013 __set_bit(reason, inhibits);
10014 else
10015 __clear_bit(reason, inhibits);
10016
10017 trace_kvm_apicv_inhibit_changed(reason, set, *inhibits);
10018 }
10019
kvm_apicv_init(struct kvm * kvm)10020 static void kvm_apicv_init(struct kvm *kvm)
10021 {
10022 enum kvm_apicv_inhibit reason = enable_apicv ? APICV_INHIBIT_REASON_ABSENT :
10023 APICV_INHIBIT_REASON_DISABLE;
10024
10025 set_or_clear_apicv_inhibit(&kvm->arch.apicv_inhibit_reasons, reason, true);
10026
10027 init_rwsem(&kvm->arch.apicv_update_lock);
10028 }
10029
kvm_sched_yield(struct kvm_vcpu * vcpu,unsigned long dest_id)10030 static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id)
10031 {
10032 struct kvm_vcpu *target = NULL;
10033 struct kvm_apic_map *map;
10034
10035 vcpu->stat.directed_yield_attempted++;
10036
10037 if (single_task_running())
10038 goto no_yield;
10039
10040 rcu_read_lock();
10041 map = rcu_dereference(vcpu->kvm->arch.apic_map);
10042
10043 if (likely(map) && dest_id <= map->max_apic_id && map->phys_map[dest_id])
10044 target = map->phys_map[dest_id]->vcpu;
10045
10046 rcu_read_unlock();
10047
10048 if (!target || !READ_ONCE(target->ready))
10049 goto no_yield;
10050
10051 /* Ignore requests to yield to self */
10052 if (vcpu == target)
10053 goto no_yield;
10054
10055 if (kvm_vcpu_yield_to(target) <= 0)
10056 goto no_yield;
10057
10058 vcpu->stat.directed_yield_successful++;
10059
10060 no_yield:
10061 return;
10062 }
10063
complete_hypercall_exit(struct kvm_vcpu * vcpu)10064 static int complete_hypercall_exit(struct kvm_vcpu *vcpu)
10065 {
10066 u64 ret = vcpu->run->hypercall.ret;
10067
10068 if (!is_64_bit_mode(vcpu))
10069 ret = (u32)ret;
10070 kvm_rax_write(vcpu, ret);
10071 ++vcpu->stat.hypercalls;
10072 return kvm_skip_emulated_instruction(vcpu);
10073 }
10074
__kvm_emulate_hypercall(struct kvm_vcpu * vcpu,unsigned long nr,unsigned long a0,unsigned long a1,unsigned long a2,unsigned long a3,int op_64_bit,int cpl)10075 unsigned long __kvm_emulate_hypercall(struct kvm_vcpu *vcpu, unsigned long nr,
10076 unsigned long a0, unsigned long a1,
10077 unsigned long a2, unsigned long a3,
10078 int op_64_bit, int cpl)
10079 {
10080 unsigned long ret;
10081
10082 trace_kvm_hypercall(nr, a0, a1, a2, a3);
10083
10084 if (!op_64_bit) {
10085 nr &= 0xFFFFFFFF;
10086 a0 &= 0xFFFFFFFF;
10087 a1 &= 0xFFFFFFFF;
10088 a2 &= 0xFFFFFFFF;
10089 a3 &= 0xFFFFFFFF;
10090 }
10091
10092 if (cpl) {
10093 ret = -KVM_EPERM;
10094 goto out;
10095 }
10096
10097 ret = -KVM_ENOSYS;
10098
10099 switch (nr) {
10100 case KVM_HC_VAPIC_POLL_IRQ:
10101 ret = 0;
10102 break;
10103 case KVM_HC_KICK_CPU:
10104 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_UNHALT))
10105 break;
10106
10107 kvm_pv_kick_cpu_op(vcpu->kvm, a1);
10108 kvm_sched_yield(vcpu, a1);
10109 ret = 0;
10110 break;
10111 #ifdef CONFIG_X86_64
10112 case KVM_HC_CLOCK_PAIRING:
10113 ret = kvm_pv_clock_pairing(vcpu, a0, a1);
10114 break;
10115 #endif
10116 case KVM_HC_SEND_IPI:
10117 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SEND_IPI))
10118 break;
10119
10120 ret = kvm_pv_send_ipi(vcpu->kvm, a0, a1, a2, a3, op_64_bit);
10121 break;
10122 case KVM_HC_SCHED_YIELD:
10123 if (!guest_pv_has(vcpu, KVM_FEATURE_PV_SCHED_YIELD))
10124 break;
10125
10126 kvm_sched_yield(vcpu, a0);
10127 ret = 0;
10128 break;
10129 case KVM_HC_MAP_GPA_RANGE: {
10130 u64 gpa = a0, npages = a1, attrs = a2;
10131
10132 ret = -KVM_ENOSYS;
10133 if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE)))
10134 break;
10135
10136 if (!PAGE_ALIGNED(gpa) || !npages ||
10137 gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) {
10138 ret = -KVM_EINVAL;
10139 break;
10140 }
10141
10142 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL;
10143 vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE;
10144 vcpu->run->hypercall.args[0] = gpa;
10145 vcpu->run->hypercall.args[1] = npages;
10146 vcpu->run->hypercall.args[2] = attrs;
10147 vcpu->run->hypercall.flags = 0;
10148 if (op_64_bit)
10149 vcpu->run->hypercall.flags |= KVM_EXIT_HYPERCALL_LONG_MODE;
10150
10151 WARN_ON_ONCE(vcpu->run->hypercall.flags & KVM_EXIT_HYPERCALL_MBZ);
10152 vcpu->arch.complete_userspace_io = complete_hypercall_exit;
10153 /* stat is incremented on completion. */
10154 return 0;
10155 }
10156 default:
10157 ret = -KVM_ENOSYS;
10158 break;
10159 }
10160
10161 out:
10162 ++vcpu->stat.hypercalls;
10163 return ret;
10164 }
10165 EXPORT_SYMBOL_GPL(__kvm_emulate_hypercall);
10166
kvm_emulate_hypercall(struct kvm_vcpu * vcpu)10167 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
10168 {
10169 unsigned long nr, a0, a1, a2, a3, ret;
10170 int op_64_bit;
10171 int cpl;
10172
10173 if (kvm_xen_hypercall_enabled(vcpu->kvm))
10174 return kvm_xen_hypercall(vcpu);
10175
10176 if (kvm_hv_hypercall_enabled(vcpu))
10177 return kvm_hv_hypercall(vcpu);
10178
10179 nr = kvm_rax_read(vcpu);
10180 a0 = kvm_rbx_read(vcpu);
10181 a1 = kvm_rcx_read(vcpu);
10182 a2 = kvm_rdx_read(vcpu);
10183 a3 = kvm_rsi_read(vcpu);
10184 op_64_bit = is_64_bit_hypercall(vcpu);
10185 cpl = static_call(kvm_x86_get_cpl)(vcpu);
10186
10187 ret = __kvm_emulate_hypercall(vcpu, nr, a0, a1, a2, a3, op_64_bit, cpl);
10188 if (nr == KVM_HC_MAP_GPA_RANGE && !ret)
10189 /* MAP_GPA tosses the request to the user space. */
10190 return 0;
10191
10192 if (!op_64_bit)
10193 ret = (u32)ret;
10194 kvm_rax_write(vcpu, ret);
10195
10196 return kvm_skip_emulated_instruction(vcpu);
10197 }
10198 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
10199
emulator_fix_hypercall(struct x86_emulate_ctxt * ctxt)10200 static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt)
10201 {
10202 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
10203 char instruction[3];
10204 unsigned long rip = kvm_rip_read(vcpu);
10205
10206 /*
10207 * If the quirk is disabled, synthesize a #UD and let the guest pick up
10208 * the pieces.
10209 */
10210 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) {
10211 ctxt->exception.error_code_valid = false;
10212 ctxt->exception.vector = UD_VECTOR;
10213 ctxt->have_exception = true;
10214 return X86EMUL_PROPAGATE_FAULT;
10215 }
10216
10217 static_call(kvm_x86_patch_hypercall)(vcpu, instruction);
10218
10219 return emulator_write_emulated(ctxt, rip, instruction, 3,
10220 &ctxt->exception);
10221 }
10222
dm_request_for_irq_injection(struct kvm_vcpu * vcpu)10223 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
10224 {
10225 return vcpu->run->request_interrupt_window &&
10226 likely(!pic_in_kernel(vcpu->kvm));
10227 }
10228
10229 /* Called within kvm->srcu read side. */
post_kvm_run_save(struct kvm_vcpu * vcpu)10230 static void post_kvm_run_save(struct kvm_vcpu *vcpu)
10231 {
10232 struct kvm_run *kvm_run = vcpu->run;
10233
10234 kvm_run->if_flag = static_call(kvm_x86_get_if_flag)(vcpu);
10235 kvm_run->cr8 = kvm_get_cr8(vcpu);
10236 kvm_run->apic_base = kvm_get_apic_base(vcpu);
10237
10238 kvm_run->ready_for_interrupt_injection =
10239 pic_in_kernel(vcpu->kvm) ||
10240 kvm_vcpu_ready_for_interrupt_injection(vcpu);
10241
10242 if (is_smm(vcpu))
10243 kvm_run->flags |= KVM_RUN_X86_SMM;
10244 }
10245
update_cr8_intercept(struct kvm_vcpu * vcpu)10246 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
10247 {
10248 int max_irr, tpr;
10249
10250 if (!kvm_x86_ops.update_cr8_intercept)
10251 return;
10252
10253 if (!lapic_in_kernel(vcpu))
10254 return;
10255
10256 if (vcpu->arch.apic->apicv_active)
10257 return;
10258
10259 if (!vcpu->arch.apic->vapic_addr)
10260 max_irr = kvm_lapic_find_highest_irr(vcpu);
10261 else
10262 max_irr = -1;
10263
10264 if (max_irr != -1)
10265 max_irr >>= 4;
10266
10267 tpr = kvm_lapic_get_cr8(vcpu);
10268
10269 static_call(kvm_x86_update_cr8_intercept)(vcpu, tpr, max_irr);
10270 }
10271
10272
kvm_check_nested_events(struct kvm_vcpu * vcpu)10273 int kvm_check_nested_events(struct kvm_vcpu *vcpu)
10274 {
10275 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
10276 kvm_x86_ops.nested_ops->triple_fault(vcpu);
10277 return 1;
10278 }
10279
10280 return kvm_x86_ops.nested_ops->check_events(vcpu);
10281 }
10282
kvm_inject_exception(struct kvm_vcpu * vcpu)10283 static void kvm_inject_exception(struct kvm_vcpu *vcpu)
10284 {
10285 /*
10286 * Suppress the error code if the vCPU is in Real Mode, as Real Mode
10287 * exceptions don't report error codes. The presence of an error code
10288 * is carried with the exception and only stripped when the exception
10289 * is injected as intercepted #PF VM-Exits for AMD's Paged Real Mode do
10290 * report an error code despite the CPU being in Real Mode.
10291 */
10292 vcpu->arch.exception.has_error_code &= is_protmode(vcpu);
10293
10294 trace_kvm_inj_exception(vcpu->arch.exception.vector,
10295 vcpu->arch.exception.has_error_code,
10296 vcpu->arch.exception.error_code,
10297 vcpu->arch.exception.injected);
10298
10299 static_call(kvm_x86_inject_exception)(vcpu);
10300 }
10301
10302 /*
10303 * Check for any event (interrupt or exception) that is ready to be injected,
10304 * and if there is at least one event, inject the event with the highest
10305 * priority. This handles both "pending" events, i.e. events that have never
10306 * been injected into the guest, and "injected" events, i.e. events that were
10307 * injected as part of a previous VM-Enter, but weren't successfully delivered
10308 * and need to be re-injected.
10309 *
10310 * Note, this is not guaranteed to be invoked on a guest instruction boundary,
10311 * i.e. doesn't guarantee that there's an event window in the guest. KVM must
10312 * be able to inject exceptions in the "middle" of an instruction, and so must
10313 * also be able to re-inject NMIs and IRQs in the middle of an instruction.
10314 * I.e. for exceptions and re-injected events, NOT invoking this on instruction
10315 * boundaries is necessary and correct.
10316 *
10317 * For simplicity, KVM uses a single path to inject all events (except events
10318 * that are injected directly from L1 to L2) and doesn't explicitly track
10319 * instruction boundaries for asynchronous events. However, because VM-Exits
10320 * that can occur during instruction execution typically result in KVM skipping
10321 * the instruction or injecting an exception, e.g. instruction and exception
10322 * intercepts, and because pending exceptions have higher priority than pending
10323 * interrupts, KVM still honors instruction boundaries in most scenarios.
10324 *
10325 * But, if a VM-Exit occurs during instruction execution, and KVM does NOT skip
10326 * the instruction or inject an exception, then KVM can incorrecty inject a new
10327 * asynchronous event if the event became pending after the CPU fetched the
10328 * instruction (in the guest). E.g. if a page fault (#PF, #NPF, EPT violation)
10329 * occurs and is resolved by KVM, a coincident NMI, SMI, IRQ, etc... can be
10330 * injected on the restarted instruction instead of being deferred until the
10331 * instruction completes.
10332 *
10333 * In practice, this virtualization hole is unlikely to be observed by the
10334 * guest, and even less likely to cause functional problems. To detect the
10335 * hole, the guest would have to trigger an event on a side effect of an early
10336 * phase of instruction execution, e.g. on the instruction fetch from memory.
10337 * And for it to be a functional problem, the guest would need to depend on the
10338 * ordering between that side effect, the instruction completing, _and_ the
10339 * delivery of the asynchronous event.
10340 */
kvm_check_and_inject_events(struct kvm_vcpu * vcpu,bool * req_immediate_exit)10341 static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu,
10342 bool *req_immediate_exit)
10343 {
10344 bool can_inject;
10345 int r;
10346
10347 /*
10348 * Process nested events first, as nested VM-Exit supersedes event
10349 * re-injection. If there's an event queued for re-injection, it will
10350 * be saved into the appropriate vmc{b,s}12 fields on nested VM-Exit.
10351 */
10352 if (is_guest_mode(vcpu))
10353 r = kvm_check_nested_events(vcpu);
10354 else
10355 r = 0;
10356
10357 /*
10358 * Re-inject exceptions and events *especially* if immediate entry+exit
10359 * to/from L2 is needed, as any event that has already been injected
10360 * into L2 needs to complete its lifecycle before injecting a new event.
10361 *
10362 * Don't re-inject an NMI or interrupt if there is a pending exception.
10363 * This collision arises if an exception occurred while vectoring the
10364 * injected event, KVM intercepted said exception, and KVM ultimately
10365 * determined the fault belongs to the guest and queues the exception
10366 * for injection back into the guest.
10367 *
10368 * "Injected" interrupts can also collide with pending exceptions if
10369 * userspace ignores the "ready for injection" flag and blindly queues
10370 * an interrupt. In that case, prioritizing the exception is correct,
10371 * as the exception "occurred" before the exit to userspace. Trap-like
10372 * exceptions, e.g. most #DBs, have higher priority than interrupts.
10373 * And while fault-like exceptions, e.g. #GP and #PF, are the lowest
10374 * priority, they're only generated (pended) during instruction
10375 * execution, and interrupts are recognized at instruction boundaries.
10376 * Thus a pending fault-like exception means the fault occurred on the
10377 * *previous* instruction and must be serviced prior to recognizing any
10378 * new events in order to fully complete the previous instruction.
10379 */
10380 if (vcpu->arch.exception.injected)
10381 kvm_inject_exception(vcpu);
10382 else if (kvm_is_exception_pending(vcpu))
10383 ; /* see above */
10384 else if (vcpu->arch.nmi_injected)
10385 static_call(kvm_x86_inject_nmi)(vcpu);
10386 else if (vcpu->arch.interrupt.injected)
10387 static_call(kvm_x86_inject_irq)(vcpu, true);
10388
10389 /*
10390 * Exceptions that morph to VM-Exits are handled above, and pending
10391 * exceptions on top of injected exceptions that do not VM-Exit should
10392 * either morph to #DF or, sadly, override the injected exception.
10393 */
10394 WARN_ON_ONCE(vcpu->arch.exception.injected &&
10395 vcpu->arch.exception.pending);
10396
10397 /*
10398 * Bail if immediate entry+exit to/from the guest is needed to complete
10399 * nested VM-Enter or event re-injection so that a different pending
10400 * event can be serviced (or if KVM needs to exit to userspace).
10401 *
10402 * Otherwise, continue processing events even if VM-Exit occurred. The
10403 * VM-Exit will have cleared exceptions that were meant for L2, but
10404 * there may now be events that can be injected into L1.
10405 */
10406 if (r < 0)
10407 goto out;
10408
10409 /*
10410 * A pending exception VM-Exit should either result in nested VM-Exit
10411 * or force an immediate re-entry and exit to/from L2, and exception
10412 * VM-Exits cannot be injected (flag should _never_ be set).
10413 */
10414 WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected ||
10415 vcpu->arch.exception_vmexit.pending);
10416
10417 /*
10418 * New events, other than exceptions, cannot be injected if KVM needs
10419 * to re-inject a previous event. See above comments on re-injecting
10420 * for why pending exceptions get priority.
10421 */
10422 can_inject = !kvm_event_needs_reinjection(vcpu);
10423
10424 if (vcpu->arch.exception.pending) {
10425 /*
10426 * Fault-class exceptions, except #DBs, set RF=1 in the RFLAGS
10427 * value pushed on the stack. Trap-like exception and all #DBs
10428 * leave RF as-is (KVM follows Intel's behavior in this regard;
10429 * AMD states that code breakpoint #DBs excplitly clear RF=0).
10430 *
10431 * Note, most versions of Intel's SDM and AMD's APM incorrectly
10432 * describe the behavior of General Detect #DBs, which are
10433 * fault-like. They do _not_ set RF, a la code breakpoints.
10434 */
10435 if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT)
10436 __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) |
10437 X86_EFLAGS_RF);
10438
10439 if (vcpu->arch.exception.vector == DB_VECTOR) {
10440 kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception);
10441 if (vcpu->arch.dr7 & DR7_GD) {
10442 vcpu->arch.dr7 &= ~DR7_GD;
10443 kvm_update_dr7(vcpu);
10444 }
10445 }
10446
10447 kvm_inject_exception(vcpu);
10448
10449 vcpu->arch.exception.pending = false;
10450 vcpu->arch.exception.injected = true;
10451
10452 can_inject = false;
10453 }
10454
10455 /* Don't inject interrupts if the user asked to avoid doing so */
10456 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ)
10457 return 0;
10458
10459 /*
10460 * Finally, inject interrupt events. If an event cannot be injected
10461 * due to architectural conditions (e.g. IF=0) a window-open exit
10462 * will re-request KVM_REQ_EVENT. Sometimes however an event is pending
10463 * and can architecturally be injected, but we cannot do it right now:
10464 * an interrupt could have arrived just now and we have to inject it
10465 * as a vmexit, or there could already an event in the queue, which is
10466 * indicated by can_inject. In that case we request an immediate exit
10467 * in order to make progress and get back here for another iteration.
10468 * The kvm_x86_ops hooks communicate this by returning -EBUSY.
10469 */
10470 #ifdef CONFIG_KVM_SMM
10471 if (vcpu->arch.smi_pending) {
10472 r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY;
10473 if (r < 0)
10474 goto out;
10475 if (r) {
10476 vcpu->arch.smi_pending = false;
10477 ++vcpu->arch.smi_count;
10478 enter_smm(vcpu);
10479 can_inject = false;
10480 } else
10481 static_call(kvm_x86_enable_smi_window)(vcpu);
10482 }
10483 #endif
10484
10485 if (vcpu->arch.nmi_pending) {
10486 r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY;
10487 if (r < 0)
10488 goto out;
10489 if (r) {
10490 --vcpu->arch.nmi_pending;
10491 vcpu->arch.nmi_injected = true;
10492 static_call(kvm_x86_inject_nmi)(vcpu);
10493 can_inject = false;
10494 WARN_ON(static_call(kvm_x86_nmi_allowed)(vcpu, true) < 0);
10495 }
10496 if (vcpu->arch.nmi_pending)
10497 static_call(kvm_x86_enable_nmi_window)(vcpu);
10498 }
10499
10500 if (kvm_cpu_has_injectable_intr(vcpu)) {
10501 r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY;
10502 if (r < 0)
10503 goto out;
10504 if (r) {
10505 int irq = kvm_cpu_get_interrupt(vcpu);
10506
10507 if (!WARN_ON_ONCE(irq == -1)) {
10508 kvm_queue_interrupt(vcpu, irq, false);
10509 static_call(kvm_x86_inject_irq)(vcpu, false);
10510 WARN_ON(static_call(kvm_x86_interrupt_allowed)(vcpu, true) < 0);
10511 }
10512 }
10513 if (kvm_cpu_has_injectable_intr(vcpu))
10514 static_call(kvm_x86_enable_irq_window)(vcpu);
10515 }
10516
10517 if (is_guest_mode(vcpu) &&
10518 kvm_x86_ops.nested_ops->has_events &&
10519 kvm_x86_ops.nested_ops->has_events(vcpu))
10520 *req_immediate_exit = true;
10521
10522 /*
10523 * KVM must never queue a new exception while injecting an event; KVM
10524 * is done emulating and should only propagate the to-be-injected event
10525 * to the VMCS/VMCB. Queueing a new exception can put the vCPU into an
10526 * infinite loop as KVM will bail from VM-Enter to inject the pending
10527 * exception and start the cycle all over.
10528 *
10529 * Exempt triple faults as they have special handling and won't put the
10530 * vCPU into an infinite loop. Triple fault can be queued when running
10531 * VMX without unrestricted guest, as that requires KVM to emulate Real
10532 * Mode events (see kvm_inject_realmode_interrupt()).
10533 */
10534 WARN_ON_ONCE(vcpu->arch.exception.pending ||
10535 vcpu->arch.exception_vmexit.pending);
10536 return 0;
10537
10538 out:
10539 if (r == -EBUSY) {
10540 *req_immediate_exit = true;
10541 r = 0;
10542 }
10543 return r;
10544 }
10545
process_nmi(struct kvm_vcpu * vcpu)10546 static void process_nmi(struct kvm_vcpu *vcpu)
10547 {
10548 unsigned int limit;
10549
10550 /*
10551 * x86 is limited to one NMI pending, but because KVM can't react to
10552 * incoming NMIs as quickly as bare metal, e.g. if the vCPU is
10553 * scheduled out, KVM needs to play nice with two queued NMIs showing
10554 * up at the same time. To handle this scenario, allow two NMIs to be
10555 * (temporarily) pending so long as NMIs are not blocked and KVM is not
10556 * waiting for a previous NMI injection to complete (which effectively
10557 * blocks NMIs). KVM will immediately inject one of the two NMIs, and
10558 * will request an NMI window to handle the second NMI.
10559 */
10560 if (static_call(kvm_x86_get_nmi_mask)(vcpu) || vcpu->arch.nmi_injected)
10561 limit = 1;
10562 else
10563 limit = 2;
10564
10565 /*
10566 * Adjust the limit to account for pending virtual NMIs, which aren't
10567 * tracked in vcpu->arch.nmi_pending.
10568 */
10569 if (static_call(kvm_x86_is_vnmi_pending)(vcpu))
10570 limit--;
10571
10572 vcpu->arch.nmi_pending += atomic_xchg(&vcpu->arch.nmi_queued, 0);
10573 vcpu->arch.nmi_pending = min(vcpu->arch.nmi_pending, limit);
10574
10575 if (vcpu->arch.nmi_pending &&
10576 (static_call(kvm_x86_set_vnmi_pending)(vcpu)))
10577 vcpu->arch.nmi_pending--;
10578
10579 if (vcpu->arch.nmi_pending)
10580 kvm_make_request(KVM_REQ_EVENT, vcpu);
10581 }
10582
10583 /* Return total number of NMIs pending injection to the VM */
kvm_get_nr_pending_nmis(struct kvm_vcpu * vcpu)10584 int kvm_get_nr_pending_nmis(struct kvm_vcpu *vcpu)
10585 {
10586 return vcpu->arch.nmi_pending +
10587 static_call(kvm_x86_is_vnmi_pending)(vcpu);
10588 }
10589
kvm_make_scan_ioapic_request_mask(struct kvm * kvm,unsigned long * vcpu_bitmap)10590 void kvm_make_scan_ioapic_request_mask(struct kvm *kvm,
10591 unsigned long *vcpu_bitmap)
10592 {
10593 kvm_make_vcpus_request_mask(kvm, KVM_REQ_SCAN_IOAPIC, vcpu_bitmap);
10594 }
10595
kvm_make_scan_ioapic_request(struct kvm * kvm)10596 void kvm_make_scan_ioapic_request(struct kvm *kvm)
10597 {
10598 kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
10599 }
10600
__kvm_vcpu_update_apicv(struct kvm_vcpu * vcpu)10601 void __kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
10602 {
10603 struct kvm_lapic *apic = vcpu->arch.apic;
10604 bool activate;
10605
10606 if (!lapic_in_kernel(vcpu))
10607 return;
10608
10609 down_read(&vcpu->kvm->arch.apicv_update_lock);
10610 preempt_disable();
10611
10612 /* Do not activate APICV when APIC is disabled */
10613 activate = kvm_vcpu_apicv_activated(vcpu) &&
10614 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED);
10615
10616 if (apic->apicv_active == activate)
10617 goto out;
10618
10619 apic->apicv_active = activate;
10620 kvm_apic_update_apicv(vcpu);
10621 static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu);
10622
10623 /*
10624 * When APICv gets disabled, we may still have injected interrupts
10625 * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was
10626 * still active when the interrupt got accepted. Make sure
10627 * kvm_check_and_inject_events() is called to check for that.
10628 */
10629 if (!apic->apicv_active)
10630 kvm_make_request(KVM_REQ_EVENT, vcpu);
10631
10632 out:
10633 preempt_enable();
10634 up_read(&vcpu->kvm->arch.apicv_update_lock);
10635 }
10636 EXPORT_SYMBOL_GPL(__kvm_vcpu_update_apicv);
10637
kvm_vcpu_update_apicv(struct kvm_vcpu * vcpu)10638 static void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu)
10639 {
10640 if (!lapic_in_kernel(vcpu))
10641 return;
10642
10643 /*
10644 * Due to sharing page tables across vCPUs, the xAPIC memslot must be
10645 * deleted if any vCPU has xAPIC virtualization and x2APIC enabled, but
10646 * and hardware doesn't support x2APIC virtualization. E.g. some AMD
10647 * CPUs support AVIC but not x2APIC. KVM still allows enabling AVIC in
10648 * this case so that KVM can the AVIC doorbell to inject interrupts to
10649 * running vCPUs, but KVM must not create SPTEs for the APIC base as
10650 * the vCPU would incorrectly be able to access the vAPIC page via MMIO
10651 * despite being in x2APIC mode. For simplicity, inhibiting the APIC
10652 * access page is sticky.
10653 */
10654 if (apic_x2apic_mode(vcpu->arch.apic) &&
10655 kvm_x86_ops.allow_apicv_in_x2apic_without_x2apic_virtualization)
10656 kvm_inhibit_apic_access_page(vcpu);
10657
10658 __kvm_vcpu_update_apicv(vcpu);
10659 }
10660
__kvm_set_or_clear_apicv_inhibit(struct kvm * kvm,enum kvm_apicv_inhibit reason,bool set)10661 void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
10662 enum kvm_apicv_inhibit reason, bool set)
10663 {
10664 unsigned long old, new;
10665
10666 lockdep_assert_held_write(&kvm->arch.apicv_update_lock);
10667
10668 if (!(kvm_x86_ops.required_apicv_inhibits & BIT(reason)))
10669 return;
10670
10671 old = new = kvm->arch.apicv_inhibit_reasons;
10672
10673 set_or_clear_apicv_inhibit(&new, reason, set);
10674
10675 if (!!old != !!new) {
10676 /*
10677 * Kick all vCPUs before setting apicv_inhibit_reasons to avoid
10678 * false positives in the sanity check WARN in svm_vcpu_run().
10679 * This task will wait for all vCPUs to ack the kick IRQ before
10680 * updating apicv_inhibit_reasons, and all other vCPUs will
10681 * block on acquiring apicv_update_lock so that vCPUs can't
10682 * redo svm_vcpu_run() without seeing the new inhibit state.
10683 *
10684 * Note, holding apicv_update_lock and taking it in the read
10685 * side (handling the request) also prevents other vCPUs from
10686 * servicing the request with a stale apicv_inhibit_reasons.
10687 */
10688 kvm_make_all_cpus_request(kvm, KVM_REQ_APICV_UPDATE);
10689 kvm->arch.apicv_inhibit_reasons = new;
10690 if (new) {
10691 unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE);
10692 int idx = srcu_read_lock(&kvm->srcu);
10693
10694 kvm_zap_gfn_range(kvm, gfn, gfn+1);
10695 srcu_read_unlock(&kvm->srcu, idx);
10696 }
10697 } else {
10698 kvm->arch.apicv_inhibit_reasons = new;
10699 }
10700 }
10701
kvm_set_or_clear_apicv_inhibit(struct kvm * kvm,enum kvm_apicv_inhibit reason,bool set)10702 void kvm_set_or_clear_apicv_inhibit(struct kvm *kvm,
10703 enum kvm_apicv_inhibit reason, bool set)
10704 {
10705 if (!enable_apicv)
10706 return;
10707
10708 down_write(&kvm->arch.apicv_update_lock);
10709 __kvm_set_or_clear_apicv_inhibit(kvm, reason, set);
10710 up_write(&kvm->arch.apicv_update_lock);
10711 }
10712 EXPORT_SYMBOL_GPL(kvm_set_or_clear_apicv_inhibit);
10713
vcpu_scan_ioapic(struct kvm_vcpu * vcpu)10714 static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
10715 {
10716 if (!kvm_apic_present(vcpu))
10717 return;
10718
10719 bitmap_zero(vcpu->arch.ioapic_handled_vectors, 256);
10720
10721 if (irqchip_split(vcpu->kvm))
10722 kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
10723 else {
10724 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
10725 if (ioapic_in_kernel(vcpu->kvm))
10726 kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
10727 }
10728
10729 if (is_guest_mode(vcpu))
10730 vcpu->arch.load_eoi_exitmap_pending = true;
10731 else
10732 kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
10733 }
10734
vcpu_load_eoi_exitmap(struct kvm_vcpu * vcpu)10735 static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu)
10736 {
10737 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
10738 return;
10739
10740 #ifdef CONFIG_KVM_HYPERV
10741 if (to_hv_vcpu(vcpu)) {
10742 u64 eoi_exit_bitmap[4];
10743
10744 bitmap_or((ulong *)eoi_exit_bitmap,
10745 vcpu->arch.ioapic_handled_vectors,
10746 to_hv_synic(vcpu)->vec_bitmap, 256);
10747 static_call_cond(kvm_x86_load_eoi_exitmap)(vcpu, eoi_exit_bitmap);
10748 return;
10749 }
10750 #endif
10751 static_call_cond(kvm_x86_load_eoi_exitmap)(
10752 vcpu, (u64 *)vcpu->arch.ioapic_handled_vectors);
10753 }
10754
kvm_arch_guest_memory_reclaimed(struct kvm * kvm)10755 void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)
10756 {
10757 static_call_cond(kvm_x86_guest_memory_reclaimed)(kvm);
10758 }
10759
kvm_vcpu_reload_apic_access_page(struct kvm_vcpu * vcpu)10760 static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu)
10761 {
10762 if (!lapic_in_kernel(vcpu))
10763 return;
10764
10765 static_call_cond(kvm_x86_set_apic_access_page_addr)(vcpu);
10766 }
10767
10768 /*
10769 * Called within kvm->srcu read side.
10770 * Returns 1 to let vcpu_run() continue the guest execution loop without
10771 * exiting to the userspace. Otherwise, the value will be returned to the
10772 * userspace.
10773 */
vcpu_enter_guest(struct kvm_vcpu * vcpu)10774 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
10775 {
10776 int r;
10777 bool req_int_win =
10778 dm_request_for_irq_injection(vcpu) &&
10779 kvm_cpu_accept_dm_intr(vcpu);
10780 fastpath_t exit_fastpath;
10781
10782 bool req_immediate_exit = false;
10783
10784 if (kvm_request_pending(vcpu)) {
10785 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) {
10786 r = -EIO;
10787 goto out;
10788 }
10789
10790 if (kvm_dirty_ring_check_request(vcpu)) {
10791 r = 0;
10792 goto out;
10793 }
10794
10795 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) {
10796 if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) {
10797 r = 0;
10798 goto out;
10799 }
10800 }
10801 if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
10802 kvm_mmu_free_obsolete_roots(vcpu);
10803 if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
10804 __kvm_migrate_timers(vcpu);
10805 if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))
10806 kvm_update_masterclock(vcpu->kvm);
10807 if (kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu))
10808 kvm_gen_kvmclock_update(vcpu);
10809 if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
10810 r = kvm_guest_time_update(vcpu);
10811 if (unlikely(r))
10812 goto out;
10813 }
10814 if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
10815 kvm_mmu_sync_roots(vcpu);
10816 if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu))
10817 kvm_mmu_load_pgd(vcpu);
10818
10819 /*
10820 * Note, the order matters here, as flushing "all" TLB entries
10821 * also flushes the "current" TLB entries, i.e. servicing the
10822 * flush "all" will clear any request to flush "current".
10823 */
10824 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
10825 kvm_vcpu_flush_tlb_all(vcpu);
10826
10827 kvm_service_local_tlb_flush_requests(vcpu);
10828
10829 /*
10830 * Fall back to a "full" guest flush if Hyper-V's precise
10831 * flushing fails. Note, Hyper-V's flushing is per-vCPU, but
10832 * the flushes are considered "remote" and not "local" because
10833 * the requests can be initiated from other vCPUs.
10834 */
10835 #ifdef CONFIG_KVM_HYPERV
10836 if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu) &&
10837 kvm_hv_vcpu_flush_tlb(vcpu))
10838 kvm_vcpu_flush_tlb_guest(vcpu);
10839 #endif
10840
10841 if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
10842 vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
10843 r = 0;
10844 goto out;
10845 }
10846 if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
10847 if (is_guest_mode(vcpu))
10848 kvm_x86_ops.nested_ops->triple_fault(vcpu);
10849
10850 if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) {
10851 vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
10852 vcpu->mmio_needed = 0;
10853 r = 0;
10854 goto out;
10855 }
10856 }
10857 if (kvm_check_request(KVM_REQ_APF_HALT, vcpu)) {
10858 /* Page is swapped out. Do synthetic halt */
10859 vcpu->arch.apf.halted = true;
10860 r = 1;
10861 goto out;
10862 }
10863 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
10864 record_steal_time(vcpu);
10865 if (kvm_check_request(KVM_REQ_PMU, vcpu))
10866 kvm_pmu_handle_event(vcpu);
10867 if (kvm_check_request(KVM_REQ_PMI, vcpu))
10868 kvm_pmu_deliver_pmi(vcpu);
10869 #ifdef CONFIG_KVM_SMM
10870 if (kvm_check_request(KVM_REQ_SMI, vcpu))
10871 process_smi(vcpu);
10872 #endif
10873 if (kvm_check_request(KVM_REQ_NMI, vcpu))
10874 process_nmi(vcpu);
10875 if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) {
10876 BUG_ON(vcpu->arch.pending_ioapic_eoi > 255);
10877 if (test_bit(vcpu->arch.pending_ioapic_eoi,
10878 vcpu->arch.ioapic_handled_vectors)) {
10879 vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI;
10880 vcpu->run->eoi.vector =
10881 vcpu->arch.pending_ioapic_eoi;
10882 r = 0;
10883 goto out;
10884 }
10885 }
10886 if (kvm_check_request(KVM_REQ_SCAN_IOAPIC, vcpu))
10887 vcpu_scan_ioapic(vcpu);
10888 if (kvm_check_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu))
10889 vcpu_load_eoi_exitmap(vcpu);
10890 if (kvm_check_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu))
10891 kvm_vcpu_reload_apic_access_page(vcpu);
10892 #ifdef CONFIG_KVM_HYPERV
10893 if (kvm_check_request(KVM_REQ_HV_CRASH, vcpu)) {
10894 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
10895 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH;
10896 vcpu->run->system_event.ndata = 0;
10897 r = 0;
10898 goto out;
10899 }
10900 if (kvm_check_request(KVM_REQ_HV_RESET, vcpu)) {
10901 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
10902 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_RESET;
10903 vcpu->run->system_event.ndata = 0;
10904 r = 0;
10905 goto out;
10906 }
10907 if (kvm_check_request(KVM_REQ_HV_EXIT, vcpu)) {
10908 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
10909
10910 vcpu->run->exit_reason = KVM_EXIT_HYPERV;
10911 vcpu->run->hyperv = hv_vcpu->exit;
10912 r = 0;
10913 goto out;
10914 }
10915
10916 /*
10917 * KVM_REQ_HV_STIMER has to be processed after
10918 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
10919 * depend on the guest clock being up-to-date
10920 */
10921 if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
10922 kvm_hv_process_stimers(vcpu);
10923 #endif
10924 if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu))
10925 kvm_vcpu_update_apicv(vcpu);
10926 if (kvm_check_request(KVM_REQ_APF_READY, vcpu))
10927 kvm_check_async_pf_completion(vcpu);
10928 if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
10929 static_call(kvm_x86_msr_filter_changed)(vcpu);
10930
10931 if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))
10932 static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);
10933 }
10934
10935 if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
10936 kvm_xen_has_interrupt(vcpu)) {
10937 ++vcpu->stat.req_event;
10938 r = kvm_apic_accept_events(vcpu);
10939 if (r < 0) {
10940 r = 0;
10941 goto out;
10942 }
10943 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
10944 r = 1;
10945 goto out;
10946 }
10947
10948 r = kvm_check_and_inject_events(vcpu, &req_immediate_exit);
10949 if (r < 0) {
10950 r = 0;
10951 goto out;
10952 }
10953 if (req_int_win)
10954 static_call(kvm_x86_enable_irq_window)(vcpu);
10955
10956 if (kvm_lapic_enabled(vcpu)) {
10957 update_cr8_intercept(vcpu);
10958 kvm_lapic_sync_to_vapic(vcpu);
10959 }
10960 }
10961
10962 r = kvm_mmu_reload(vcpu);
10963 if (unlikely(r)) {
10964 goto cancel_injection;
10965 }
10966
10967 preempt_disable();
10968
10969 static_call(kvm_x86_prepare_switch_to_guest)(vcpu);
10970
10971 /*
10972 * Disable IRQs before setting IN_GUEST_MODE. Posted interrupt
10973 * IPI are then delayed after guest entry, which ensures that they
10974 * result in virtual interrupt delivery.
10975 */
10976 local_irq_disable();
10977
10978 /* Store vcpu->apicv_active before vcpu->mode. */
10979 smp_store_release(&vcpu->mode, IN_GUEST_MODE);
10980
10981 kvm_vcpu_srcu_read_unlock(vcpu);
10982
10983 /*
10984 * 1) We should set ->mode before checking ->requests. Please see
10985 * the comment in kvm_vcpu_exiting_guest_mode().
10986 *
10987 * 2) For APICv, we should set ->mode before checking PID.ON. This
10988 * pairs with the memory barrier implicit in pi_test_and_set_on
10989 * (see vmx_deliver_posted_interrupt).
10990 *
10991 * 3) This also orders the write to mode from any reads to the page
10992 * tables done while the VCPU is running. Please see the comment
10993 * in kvm_flush_remote_tlbs.
10994 */
10995 smp_mb__after_srcu_read_unlock();
10996
10997 /*
10998 * Process pending posted interrupts to handle the case where the
10999 * notification IRQ arrived in the host, or was never sent (because the
11000 * target vCPU wasn't running). Do this regardless of the vCPU's APICv
11001 * status, KVM doesn't update assigned devices when APICv is inhibited,
11002 * i.e. they can post interrupts even if APICv is temporarily disabled.
11003 */
11004 if (kvm_lapic_enabled(vcpu))
11005 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
11006
11007 if (kvm_vcpu_exit_request(vcpu)) {
11008 vcpu->mode = OUTSIDE_GUEST_MODE;
11009 smp_wmb();
11010 local_irq_enable();
11011 preempt_enable();
11012 kvm_vcpu_srcu_read_lock(vcpu);
11013 r = 1;
11014 goto cancel_injection;
11015 }
11016
11017 if (req_immediate_exit)
11018 kvm_make_request(KVM_REQ_EVENT, vcpu);
11019
11020 fpregs_assert_state_consistent();
11021 if (test_thread_flag(TIF_NEED_FPU_LOAD))
11022 switch_fpu_return();
11023
11024 if (vcpu->arch.guest_fpu.xfd_err)
11025 wrmsrl(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err);
11026
11027 if (unlikely(vcpu->arch.switch_db_regs)) {
11028 set_debugreg(0, 7);
11029 set_debugreg(vcpu->arch.eff_db[0], 0);
11030 set_debugreg(vcpu->arch.eff_db[1], 1);
11031 set_debugreg(vcpu->arch.eff_db[2], 2);
11032 set_debugreg(vcpu->arch.eff_db[3], 3);
11033 } else if (unlikely(hw_breakpoint_active())) {
11034 set_debugreg(0, 7);
11035 }
11036
11037 guest_timing_enter_irqoff();
11038
11039 for (;;) {
11040 /*
11041 * Assert that vCPU vs. VM APICv state is consistent. An APICv
11042 * update must kick and wait for all vCPUs before toggling the
11043 * per-VM state, and responding vCPUs must wait for the update
11044 * to complete before servicing KVM_REQ_APICV_UPDATE.
11045 */
11046 WARN_ON_ONCE((kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)) &&
11047 (kvm_get_apic_mode(vcpu) != LAPIC_MODE_DISABLED));
11048
11049 exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu, req_immediate_exit);
11050 if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST))
11051 break;
11052
11053 if (kvm_lapic_enabled(vcpu))
11054 static_call_cond(kvm_x86_sync_pir_to_irr)(vcpu);
11055
11056 if (unlikely(kvm_vcpu_exit_request(vcpu))) {
11057 exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED;
11058 break;
11059 }
11060
11061 /* Note, VM-Exits that go down the "slow" path are accounted below. */
11062 ++vcpu->stat.exits;
11063 }
11064
11065 /*
11066 * Do this here before restoring debug registers on the host. And
11067 * since we do this before handling the vmexit, a DR access vmexit
11068 * can (a) read the correct value of the debug registers, (b) set
11069 * KVM_DEBUGREG_WONT_EXIT again.
11070 */
11071 if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
11072 WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
11073 static_call(kvm_x86_sync_dirty_debug_regs)(vcpu);
11074 kvm_update_dr0123(vcpu);
11075 kvm_update_dr7(vcpu);
11076 }
11077
11078 /*
11079 * If the guest has used debug registers, at least dr7
11080 * will be disabled while returning to the host.
11081 * If we don't have active breakpoints in the host, we don't
11082 * care about the messed up debug address registers. But if
11083 * we have some of them active, restore the old state.
11084 */
11085 if (hw_breakpoint_active())
11086 hw_breakpoint_restore();
11087
11088 vcpu->arch.last_vmentry_cpu = vcpu->cpu;
11089 vcpu->arch.last_guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
11090
11091 vcpu->mode = OUTSIDE_GUEST_MODE;
11092 smp_wmb();
11093
11094 /*
11095 * Sync xfd before calling handle_exit_irqoff() which may
11096 * rely on the fact that guest_fpu::xfd is up-to-date (e.g.
11097 * in #NM irqoff handler).
11098 */
11099 if (vcpu->arch.xfd_no_write_intercept)
11100 fpu_sync_guest_vmexit_xfd_state();
11101
11102 static_call(kvm_x86_handle_exit_irqoff)(vcpu);
11103
11104 if (vcpu->arch.guest_fpu.xfd_err)
11105 wrmsrl(MSR_IA32_XFD_ERR, 0);
11106
11107 /*
11108 * Consume any pending interrupts, including the possible source of
11109 * VM-Exit on SVM and any ticks that occur between VM-Exit and now.
11110 * An instruction is required after local_irq_enable() to fully unblock
11111 * interrupts on processors that implement an interrupt shadow, the
11112 * stat.exits increment will do nicely.
11113 */
11114 kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
11115 local_irq_enable();
11116 ++vcpu->stat.exits;
11117 local_irq_disable();
11118 kvm_after_interrupt(vcpu);
11119
11120 /*
11121 * Wait until after servicing IRQs to account guest time so that any
11122 * ticks that occurred while running the guest are properly accounted
11123 * to the guest. Waiting until IRQs are enabled degrades the accuracy
11124 * of accounting via context tracking, but the loss of accuracy is
11125 * acceptable for all known use cases.
11126 */
11127 guest_timing_exit_irqoff();
11128
11129 local_irq_enable();
11130 preempt_enable();
11131
11132 kvm_vcpu_srcu_read_lock(vcpu);
11133
11134 /*
11135 * Profile KVM exit RIPs:
11136 */
11137 if (unlikely(prof_on == KVM_PROFILING)) {
11138 unsigned long rip = kvm_rip_read(vcpu);
11139 profile_hit(KVM_PROFILING, (void *)rip);
11140 }
11141
11142 if (unlikely(vcpu->arch.tsc_always_catchup))
11143 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
11144
11145 if (vcpu->arch.apic_attention)
11146 kvm_lapic_sync_from_vapic(vcpu);
11147
11148 r = static_call(kvm_x86_handle_exit)(vcpu, exit_fastpath);
11149 return r;
11150
11151 cancel_injection:
11152 if (req_immediate_exit)
11153 kvm_make_request(KVM_REQ_EVENT, vcpu);
11154 static_call(kvm_x86_cancel_injection)(vcpu);
11155 if (unlikely(vcpu->arch.apic_attention))
11156 kvm_lapic_sync_from_vapic(vcpu);
11157 out:
11158 return r;
11159 }
11160
11161 /* Called within kvm->srcu read side. */
vcpu_block(struct kvm_vcpu * vcpu)11162 static inline int vcpu_block(struct kvm_vcpu *vcpu)
11163 {
11164 bool hv_timer;
11165
11166 if (!kvm_arch_vcpu_runnable(vcpu)) {
11167 /*
11168 * Switch to the software timer before halt-polling/blocking as
11169 * the guest's timer may be a break event for the vCPU, and the
11170 * hypervisor timer runs only when the CPU is in guest mode.
11171 * Switch before halt-polling so that KVM recognizes an expired
11172 * timer before blocking.
11173 */
11174 hv_timer = kvm_lapic_hv_timer_in_use(vcpu);
11175 if (hv_timer)
11176 kvm_lapic_switch_to_sw_timer(vcpu);
11177
11178 kvm_vcpu_srcu_read_unlock(vcpu);
11179 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
11180 kvm_vcpu_halt(vcpu);
11181 else
11182 kvm_vcpu_block(vcpu);
11183 kvm_vcpu_srcu_read_lock(vcpu);
11184
11185 if (hv_timer)
11186 kvm_lapic_switch_to_hv_timer(vcpu);
11187
11188 /*
11189 * If the vCPU is not runnable, a signal or another host event
11190 * of some kind is pending; service it without changing the
11191 * vCPU's activity state.
11192 */
11193 if (!kvm_arch_vcpu_runnable(vcpu))
11194 return 1;
11195 }
11196
11197 /*
11198 * Evaluate nested events before exiting the halted state. This allows
11199 * the halt state to be recorded properly in the VMCS12's activity
11200 * state field (AMD does not have a similar field and a VM-Exit always
11201 * causes a spurious wakeup from HLT).
11202 */
11203 if (is_guest_mode(vcpu)) {
11204 if (kvm_check_nested_events(vcpu) < 0)
11205 return 0;
11206 }
11207
11208 if (kvm_apic_accept_events(vcpu) < 0)
11209 return 0;
11210 switch(vcpu->arch.mp_state) {
11211 case KVM_MP_STATE_HALTED:
11212 case KVM_MP_STATE_AP_RESET_HOLD:
11213 vcpu->arch.pv.pv_unhalted = false;
11214 vcpu->arch.mp_state =
11215 KVM_MP_STATE_RUNNABLE;
11216 fallthrough;
11217 case KVM_MP_STATE_RUNNABLE:
11218 vcpu->arch.apf.halted = false;
11219 break;
11220 case KVM_MP_STATE_INIT_RECEIVED:
11221 break;
11222 default:
11223 WARN_ON_ONCE(1);
11224 break;
11225 }
11226 return 1;
11227 }
11228
kvm_vcpu_running(struct kvm_vcpu * vcpu)11229 static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu)
11230 {
11231 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
11232 !vcpu->arch.apf.halted);
11233 }
11234
11235 /* Called within kvm->srcu read side. */
vcpu_run(struct kvm_vcpu * vcpu)11236 static int vcpu_run(struct kvm_vcpu *vcpu)
11237 {
11238 int r;
11239
11240 vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
11241 vcpu->arch.l1tf_flush_l1d = true;
11242
11243 for (;;) {
11244 /*
11245 * If another guest vCPU requests a PV TLB flush in the middle
11246 * of instruction emulation, the rest of the emulation could
11247 * use a stale page translation. Assume that any code after
11248 * this point can start executing an instruction.
11249 */
11250 vcpu->arch.at_instruction_boundary = false;
11251 if (kvm_vcpu_running(vcpu)) {
11252 r = vcpu_enter_guest(vcpu);
11253 } else {
11254 r = vcpu_block(vcpu);
11255 }
11256
11257 if (r <= 0)
11258 break;
11259
11260 kvm_clear_request(KVM_REQ_UNBLOCK, vcpu);
11261 if (kvm_xen_has_pending_events(vcpu))
11262 kvm_xen_inject_pending_events(vcpu);
11263
11264 if (kvm_cpu_has_pending_timer(vcpu))
11265 kvm_inject_pending_timer_irqs(vcpu);
11266
11267 if (dm_request_for_irq_injection(vcpu) &&
11268 kvm_vcpu_ready_for_interrupt_injection(vcpu)) {
11269 r = 0;
11270 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
11271 ++vcpu->stat.request_irq_exits;
11272 break;
11273 }
11274
11275 if (__xfer_to_guest_mode_work_pending()) {
11276 kvm_vcpu_srcu_read_unlock(vcpu);
11277 r = xfer_to_guest_mode_handle_work(vcpu);
11278 kvm_vcpu_srcu_read_lock(vcpu);
11279 if (r)
11280 return r;
11281 }
11282 }
11283
11284 return r;
11285 }
11286
complete_emulated_io(struct kvm_vcpu * vcpu)11287 static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
11288 {
11289 return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
11290 }
11291
complete_emulated_pio(struct kvm_vcpu * vcpu)11292 static int complete_emulated_pio(struct kvm_vcpu *vcpu)
11293 {
11294 BUG_ON(!vcpu->arch.pio.count);
11295
11296 return complete_emulated_io(vcpu);
11297 }
11298
11299 /*
11300 * Implements the following, as a state machine:
11301 *
11302 * read:
11303 * for each fragment
11304 * for each mmio piece in the fragment
11305 * write gpa, len
11306 * exit
11307 * copy data
11308 * execute insn
11309 *
11310 * write:
11311 * for each fragment
11312 * for each mmio piece in the fragment
11313 * write gpa, len
11314 * copy data
11315 * exit
11316 */
complete_emulated_mmio(struct kvm_vcpu * vcpu)11317 static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
11318 {
11319 struct kvm_run *run = vcpu->run;
11320 struct kvm_mmio_fragment *frag;
11321 unsigned len;
11322
11323 BUG_ON(!vcpu->mmio_needed);
11324
11325 /* Complete previous fragment */
11326 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
11327 len = min(8u, frag->len);
11328 if (!vcpu->mmio_is_write)
11329 memcpy(frag->data, run->mmio.data, len);
11330
11331 if (frag->len <= 8) {
11332 /* Switch to the next fragment. */
11333 frag++;
11334 vcpu->mmio_cur_fragment++;
11335 } else {
11336 /* Go forward to the next mmio piece. */
11337 frag->data += len;
11338 frag->gpa += len;
11339 frag->len -= len;
11340 }
11341
11342 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
11343 vcpu->mmio_needed = 0;
11344
11345 /* FIXME: return into emulator if single-stepping. */
11346 if (vcpu->mmio_is_write)
11347 return 1;
11348 vcpu->mmio_read_completed = 1;
11349 return complete_emulated_io(vcpu);
11350 }
11351
11352 run->exit_reason = KVM_EXIT_MMIO;
11353 run->mmio.phys_addr = frag->gpa;
11354 if (vcpu->mmio_is_write)
11355 memcpy(run->mmio.data, frag->data, min(8u, frag->len));
11356 run->mmio.len = min(8u, frag->len);
11357 run->mmio.is_write = vcpu->mmio_is_write;
11358 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
11359 return 0;
11360 }
11361
11362 /* Swap (qemu) user FPU context for the guest FPU context. */
kvm_load_guest_fpu(struct kvm_vcpu * vcpu)11363 static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
11364 {
11365 /* Exclude PKRU, it's restored separately immediately after VM-Exit. */
11366 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
11367 trace_kvm_fpu(1);
11368 }
11369
11370 /* When vcpu_run ends, restore user space FPU context. */
kvm_put_guest_fpu(struct kvm_vcpu * vcpu)11371 static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
11372 {
11373 fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
11374 ++vcpu->stat.fpu_reload;
11375 trace_kvm_fpu(0);
11376 }
11377
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)11378 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
11379 {
11380 struct kvm_queued_exception *ex = &vcpu->arch.exception;
11381 struct kvm_run *kvm_run = vcpu->run;
11382 int r;
11383
11384 vcpu_load(vcpu);
11385 kvm_sigset_activate(vcpu);
11386 kvm_run->flags = 0;
11387 kvm_load_guest_fpu(vcpu);
11388
11389 kvm_vcpu_srcu_read_lock(vcpu);
11390 if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
11391 if (kvm_run->immediate_exit) {
11392 r = -EINTR;
11393 goto out;
11394 }
11395
11396 /*
11397 * Don't bother switching APIC timer emulation from the
11398 * hypervisor timer to the software timer, the only way for the
11399 * APIC timer to be active is if userspace stuffed vCPU state,
11400 * i.e. put the vCPU into a nonsensical state. Only an INIT
11401 * will transition the vCPU out of UNINITIALIZED (without more
11402 * state stuffing from userspace), which will reset the local
11403 * APIC and thus cancel the timer or drop the IRQ (if the timer
11404 * already expired).
11405 */
11406 kvm_vcpu_srcu_read_unlock(vcpu);
11407 kvm_vcpu_block(vcpu);
11408 kvm_vcpu_srcu_read_lock(vcpu);
11409
11410 if (kvm_apic_accept_events(vcpu) < 0) {
11411 r = 0;
11412 goto out;
11413 }
11414 r = -EAGAIN;
11415 if (signal_pending(current)) {
11416 r = -EINTR;
11417 kvm_run->exit_reason = KVM_EXIT_INTR;
11418 ++vcpu->stat.signal_exits;
11419 }
11420 goto out;
11421 }
11422
11423 if ((kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) ||
11424 (kvm_run->kvm_dirty_regs & ~KVM_SYNC_X86_VALID_FIELDS)) {
11425 r = -EINVAL;
11426 goto out;
11427 }
11428
11429 if (kvm_run->kvm_dirty_regs) {
11430 r = sync_regs(vcpu);
11431 if (r != 0)
11432 goto out;
11433 }
11434
11435 /* re-sync apic's tpr */
11436 if (!lapic_in_kernel(vcpu)) {
11437 if (kvm_set_cr8(vcpu, kvm_run->cr8) != 0) {
11438 r = -EINVAL;
11439 goto out;
11440 }
11441 }
11442
11443 /*
11444 * If userspace set a pending exception and L2 is active, convert it to
11445 * a pending VM-Exit if L1 wants to intercept the exception.
11446 */
11447 if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) &&
11448 kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector,
11449 ex->error_code)) {
11450 kvm_queue_exception_vmexit(vcpu, ex->vector,
11451 ex->has_error_code, ex->error_code,
11452 ex->has_payload, ex->payload);
11453 ex->injected = false;
11454 ex->pending = false;
11455 }
11456 vcpu->arch.exception_from_userspace = false;
11457
11458 if (unlikely(vcpu->arch.complete_userspace_io)) {
11459 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
11460 vcpu->arch.complete_userspace_io = NULL;
11461 r = cui(vcpu);
11462 if (r <= 0)
11463 goto out;
11464 } else {
11465 WARN_ON_ONCE(vcpu->arch.pio.count);
11466 WARN_ON_ONCE(vcpu->mmio_needed);
11467 }
11468
11469 if (kvm_run->immediate_exit) {
11470 r = -EINTR;
11471 goto out;
11472 }
11473
11474 r = static_call(kvm_x86_vcpu_pre_run)(vcpu);
11475 if (r <= 0)
11476 goto out;
11477
11478 r = vcpu_run(vcpu);
11479
11480 out:
11481 kvm_put_guest_fpu(vcpu);
11482 if (kvm_run->kvm_valid_regs)
11483 store_regs(vcpu);
11484 post_kvm_run_save(vcpu);
11485 kvm_vcpu_srcu_read_unlock(vcpu);
11486
11487 kvm_sigset_deactivate(vcpu);
11488 vcpu_put(vcpu);
11489 return r;
11490 }
11491
__get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)11492 static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11493 {
11494 if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
11495 /*
11496 * We are here if userspace calls get_regs() in the middle of
11497 * instruction emulation. Registers state needs to be copied
11498 * back from emulation context to vcpu. Userspace shouldn't do
11499 * that usually, but some bad designed PV devices (vmware
11500 * backdoor interface) need this to work
11501 */
11502 emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
11503 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
11504 }
11505 regs->rax = kvm_rax_read(vcpu);
11506 regs->rbx = kvm_rbx_read(vcpu);
11507 regs->rcx = kvm_rcx_read(vcpu);
11508 regs->rdx = kvm_rdx_read(vcpu);
11509 regs->rsi = kvm_rsi_read(vcpu);
11510 regs->rdi = kvm_rdi_read(vcpu);
11511 regs->rsp = kvm_rsp_read(vcpu);
11512 regs->rbp = kvm_rbp_read(vcpu);
11513 #ifdef CONFIG_X86_64
11514 regs->r8 = kvm_r8_read(vcpu);
11515 regs->r9 = kvm_r9_read(vcpu);
11516 regs->r10 = kvm_r10_read(vcpu);
11517 regs->r11 = kvm_r11_read(vcpu);
11518 regs->r12 = kvm_r12_read(vcpu);
11519 regs->r13 = kvm_r13_read(vcpu);
11520 regs->r14 = kvm_r14_read(vcpu);
11521 regs->r15 = kvm_r15_read(vcpu);
11522 #endif
11523
11524 regs->rip = kvm_rip_read(vcpu);
11525 regs->rflags = kvm_get_rflags(vcpu);
11526 }
11527
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)11528 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11529 {
11530 if (vcpu->kvm->arch.has_protected_state &&
11531 vcpu->arch.guest_state_protected)
11532 return -EINVAL;
11533
11534 vcpu_load(vcpu);
11535 __get_regs(vcpu, regs);
11536 vcpu_put(vcpu);
11537 return 0;
11538 }
11539
__set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)11540 static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11541 {
11542 vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
11543 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
11544
11545 kvm_rax_write(vcpu, regs->rax);
11546 kvm_rbx_write(vcpu, regs->rbx);
11547 kvm_rcx_write(vcpu, regs->rcx);
11548 kvm_rdx_write(vcpu, regs->rdx);
11549 kvm_rsi_write(vcpu, regs->rsi);
11550 kvm_rdi_write(vcpu, regs->rdi);
11551 kvm_rsp_write(vcpu, regs->rsp);
11552 kvm_rbp_write(vcpu, regs->rbp);
11553 #ifdef CONFIG_X86_64
11554 kvm_r8_write(vcpu, regs->r8);
11555 kvm_r9_write(vcpu, regs->r9);
11556 kvm_r10_write(vcpu, regs->r10);
11557 kvm_r11_write(vcpu, regs->r11);
11558 kvm_r12_write(vcpu, regs->r12);
11559 kvm_r13_write(vcpu, regs->r13);
11560 kvm_r14_write(vcpu, regs->r14);
11561 kvm_r15_write(vcpu, regs->r15);
11562 #endif
11563
11564 kvm_rip_write(vcpu, regs->rip);
11565 kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED);
11566
11567 vcpu->arch.exception.pending = false;
11568 vcpu->arch.exception_vmexit.pending = false;
11569
11570 kvm_make_request(KVM_REQ_EVENT, vcpu);
11571 }
11572
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)11573 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
11574 {
11575 if (vcpu->kvm->arch.has_protected_state &&
11576 vcpu->arch.guest_state_protected)
11577 return -EINVAL;
11578
11579 vcpu_load(vcpu);
11580 __set_regs(vcpu, regs);
11581 vcpu_put(vcpu);
11582 return 0;
11583 }
11584
__get_sregs_common(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11585 static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11586 {
11587 struct desc_ptr dt;
11588
11589 if (vcpu->arch.guest_state_protected)
11590 goto skip_protected_regs;
11591
11592 kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
11593 kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
11594 kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
11595 kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
11596 kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
11597 kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
11598
11599 kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
11600 kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
11601
11602 static_call(kvm_x86_get_idt)(vcpu, &dt);
11603 sregs->idt.limit = dt.size;
11604 sregs->idt.base = dt.address;
11605 static_call(kvm_x86_get_gdt)(vcpu, &dt);
11606 sregs->gdt.limit = dt.size;
11607 sregs->gdt.base = dt.address;
11608
11609 sregs->cr2 = vcpu->arch.cr2;
11610 sregs->cr3 = kvm_read_cr3(vcpu);
11611
11612 skip_protected_regs:
11613 sregs->cr0 = kvm_read_cr0(vcpu);
11614 sregs->cr4 = kvm_read_cr4(vcpu);
11615 sregs->cr8 = kvm_get_cr8(vcpu);
11616 sregs->efer = vcpu->arch.efer;
11617 sregs->apic_base = kvm_get_apic_base(vcpu);
11618 }
11619
__get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11620 static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11621 {
11622 __get_sregs_common(vcpu, sregs);
11623
11624 if (vcpu->arch.guest_state_protected)
11625 return;
11626
11627 if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft)
11628 set_bit(vcpu->arch.interrupt.nr,
11629 (unsigned long *)sregs->interrupt_bitmap);
11630 }
11631
__get_sregs2(struct kvm_vcpu * vcpu,struct kvm_sregs2 * sregs2)11632 static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
11633 {
11634 int i;
11635
11636 __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2);
11637
11638 if (vcpu->arch.guest_state_protected)
11639 return;
11640
11641 if (is_pae_paging(vcpu)) {
11642 for (i = 0 ; i < 4 ; i++)
11643 sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i);
11644 sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID;
11645 }
11646 }
11647
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11648 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
11649 struct kvm_sregs *sregs)
11650 {
11651 if (vcpu->kvm->arch.has_protected_state &&
11652 vcpu->arch.guest_state_protected)
11653 return -EINVAL;
11654
11655 vcpu_load(vcpu);
11656 __get_sregs(vcpu, sregs);
11657 vcpu_put(vcpu);
11658 return 0;
11659 }
11660
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)11661 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
11662 struct kvm_mp_state *mp_state)
11663 {
11664 int r;
11665
11666 vcpu_load(vcpu);
11667 if (kvm_mpx_supported())
11668 kvm_load_guest_fpu(vcpu);
11669
11670 r = kvm_apic_accept_events(vcpu);
11671 if (r < 0)
11672 goto out;
11673 r = 0;
11674
11675 if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED ||
11676 vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) &&
11677 vcpu->arch.pv.pv_unhalted)
11678 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
11679 else
11680 mp_state->mp_state = vcpu->arch.mp_state;
11681
11682 out:
11683 if (kvm_mpx_supported())
11684 kvm_put_guest_fpu(vcpu);
11685 vcpu_put(vcpu);
11686 return r;
11687 }
11688
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)11689 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
11690 struct kvm_mp_state *mp_state)
11691 {
11692 int ret = -EINVAL;
11693
11694 vcpu_load(vcpu);
11695
11696 switch (mp_state->mp_state) {
11697 case KVM_MP_STATE_UNINITIALIZED:
11698 case KVM_MP_STATE_HALTED:
11699 case KVM_MP_STATE_AP_RESET_HOLD:
11700 case KVM_MP_STATE_INIT_RECEIVED:
11701 case KVM_MP_STATE_SIPI_RECEIVED:
11702 if (!lapic_in_kernel(vcpu))
11703 goto out;
11704 break;
11705
11706 case KVM_MP_STATE_RUNNABLE:
11707 break;
11708
11709 default:
11710 goto out;
11711 }
11712
11713 /*
11714 * Pending INITs are reported using KVM_SET_VCPU_EVENTS, disallow
11715 * forcing the guest into INIT/SIPI if those events are supposed to be
11716 * blocked. KVM prioritizes SMI over INIT, so reject INIT/SIPI state
11717 * if an SMI is pending as well.
11718 */
11719 if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) &&
11720 (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
11721 mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
11722 goto out;
11723
11724 if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
11725 vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
11726 set_bit(KVM_APIC_SIPI, &vcpu->arch.apic->pending_events);
11727 } else
11728 vcpu->arch.mp_state = mp_state->mp_state;
11729 kvm_make_request(KVM_REQ_EVENT, vcpu);
11730
11731 ret = 0;
11732 out:
11733 vcpu_put(vcpu);
11734 return ret;
11735 }
11736
kvm_task_switch(struct kvm_vcpu * vcpu,u16 tss_selector,int idt_index,int reason,bool has_error_code,u32 error_code)11737 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
11738 int reason, bool has_error_code, u32 error_code)
11739 {
11740 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
11741 int ret;
11742
11743 init_emulate_ctxt(vcpu);
11744
11745 ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
11746 has_error_code, error_code);
11747 if (ret) {
11748 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
11749 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
11750 vcpu->run->internal.ndata = 0;
11751 return 0;
11752 }
11753
11754 kvm_rip_write(vcpu, ctxt->eip);
11755 kvm_set_rflags(vcpu, ctxt->eflags);
11756 return 1;
11757 }
11758 EXPORT_SYMBOL_GPL(kvm_task_switch);
11759
kvm_is_valid_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11760 static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11761 {
11762 if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) {
11763 /*
11764 * When EFER.LME and CR0.PG are set, the processor is in
11765 * 64-bit mode (though maybe in a 32-bit code segment).
11766 * CR4.PAE and EFER.LMA must be set.
11767 */
11768 if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA))
11769 return false;
11770 if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3))
11771 return false;
11772 } else {
11773 /*
11774 * Not in 64-bit mode: EFER.LMA is clear and the code
11775 * segment cannot be 64-bit.
11776 */
11777 if (sregs->efer & EFER_LMA || sregs->cs.l)
11778 return false;
11779 }
11780
11781 return kvm_is_valid_cr4(vcpu, sregs->cr4) &&
11782 kvm_is_valid_cr0(vcpu, sregs->cr0);
11783 }
11784
__set_sregs_common(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs,int * mmu_reset_needed,bool update_pdptrs)11785 static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
11786 int *mmu_reset_needed, bool update_pdptrs)
11787 {
11788 struct msr_data apic_base_msr;
11789 int idx;
11790 struct desc_ptr dt;
11791
11792 if (!kvm_is_valid_sregs(vcpu, sregs))
11793 return -EINVAL;
11794
11795 apic_base_msr.data = sregs->apic_base;
11796 apic_base_msr.host_initiated = true;
11797 if (kvm_set_apic_base(vcpu, &apic_base_msr))
11798 return -EINVAL;
11799
11800 if (vcpu->arch.guest_state_protected)
11801 return 0;
11802
11803 dt.size = sregs->idt.limit;
11804 dt.address = sregs->idt.base;
11805 static_call(kvm_x86_set_idt)(vcpu, &dt);
11806 dt.size = sregs->gdt.limit;
11807 dt.address = sregs->gdt.base;
11808 static_call(kvm_x86_set_gdt)(vcpu, &dt);
11809
11810 vcpu->arch.cr2 = sregs->cr2;
11811 *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
11812 vcpu->arch.cr3 = sregs->cr3;
11813 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
11814 static_call_cond(kvm_x86_post_set_cr3)(vcpu, sregs->cr3);
11815
11816 kvm_set_cr8(vcpu, sregs->cr8);
11817
11818 *mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
11819 static_call(kvm_x86_set_efer)(vcpu, sregs->efer);
11820
11821 *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
11822 static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0);
11823
11824 *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
11825 static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4);
11826
11827 if (update_pdptrs) {
11828 idx = srcu_read_lock(&vcpu->kvm->srcu);
11829 if (is_pae_paging(vcpu)) {
11830 load_pdptrs(vcpu, kvm_read_cr3(vcpu));
11831 *mmu_reset_needed = 1;
11832 }
11833 srcu_read_unlock(&vcpu->kvm->srcu, idx);
11834 }
11835
11836 kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
11837 kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
11838 kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
11839 kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
11840 kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
11841 kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
11842
11843 kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
11844 kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
11845
11846 update_cr8_intercept(vcpu);
11847
11848 /* Older userspace won't unhalt the vcpu on reset. */
11849 if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
11850 sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
11851 !is_protmode(vcpu))
11852 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
11853
11854 return 0;
11855 }
11856
__set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11857 static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
11858 {
11859 int pending_vec, max_bits;
11860 int mmu_reset_needed = 0;
11861 int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true);
11862
11863 if (ret)
11864 return ret;
11865
11866 if (mmu_reset_needed) {
11867 kvm_mmu_reset_context(vcpu);
11868 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
11869 }
11870
11871 max_bits = KVM_NR_INTERRUPTS;
11872 pending_vec = find_first_bit(
11873 (const unsigned long *)sregs->interrupt_bitmap, max_bits);
11874
11875 if (pending_vec < max_bits) {
11876 kvm_queue_interrupt(vcpu, pending_vec, false);
11877 pr_debug("Set back pending irq %d\n", pending_vec);
11878 kvm_make_request(KVM_REQ_EVENT, vcpu);
11879 }
11880 return 0;
11881 }
11882
__set_sregs2(struct kvm_vcpu * vcpu,struct kvm_sregs2 * sregs2)11883 static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
11884 {
11885 int mmu_reset_needed = 0;
11886 bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID;
11887 bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) &&
11888 !(sregs2->efer & EFER_LMA);
11889 int i, ret;
11890
11891 if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID)
11892 return -EINVAL;
11893
11894 if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected))
11895 return -EINVAL;
11896
11897 ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2,
11898 &mmu_reset_needed, !valid_pdptrs);
11899 if (ret)
11900 return ret;
11901
11902 if (valid_pdptrs) {
11903 for (i = 0; i < 4 ; i++)
11904 kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);
11905
11906 kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
11907 mmu_reset_needed = 1;
11908 vcpu->arch.pdptrs_from_userspace = true;
11909 }
11910 if (mmu_reset_needed) {
11911 kvm_mmu_reset_context(vcpu);
11912 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
11913 }
11914 return 0;
11915 }
11916
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)11917 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
11918 struct kvm_sregs *sregs)
11919 {
11920 int ret;
11921
11922 if (vcpu->kvm->arch.has_protected_state &&
11923 vcpu->arch.guest_state_protected)
11924 return -EINVAL;
11925
11926 vcpu_load(vcpu);
11927 ret = __set_sregs(vcpu, sregs);
11928 vcpu_put(vcpu);
11929 return ret;
11930 }
11931
kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm * kvm)11932 static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm)
11933 {
11934 bool set = false;
11935 struct kvm_vcpu *vcpu;
11936 unsigned long i;
11937
11938 if (!enable_apicv)
11939 return;
11940
11941 down_write(&kvm->arch.apicv_update_lock);
11942
11943 kvm_for_each_vcpu(i, vcpu, kvm) {
11944 if (vcpu->guest_debug & KVM_GUESTDBG_BLOCKIRQ) {
11945 set = true;
11946 break;
11947 }
11948 }
11949 __kvm_set_or_clear_apicv_inhibit(kvm, APICV_INHIBIT_REASON_BLOCKIRQ, set);
11950 up_write(&kvm->arch.apicv_update_lock);
11951 }
11952
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)11953 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
11954 struct kvm_guest_debug *dbg)
11955 {
11956 unsigned long rflags;
11957 int i, r;
11958
11959 if (vcpu->arch.guest_state_protected)
11960 return -EINVAL;
11961
11962 vcpu_load(vcpu);
11963
11964 if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
11965 r = -EBUSY;
11966 if (kvm_is_exception_pending(vcpu))
11967 goto out;
11968 if (dbg->control & KVM_GUESTDBG_INJECT_DB)
11969 kvm_queue_exception(vcpu, DB_VECTOR);
11970 else
11971 kvm_queue_exception(vcpu, BP_VECTOR);
11972 }
11973
11974 /*
11975 * Read rflags as long as potentially injected trace flags are still
11976 * filtered out.
11977 */
11978 rflags = kvm_get_rflags(vcpu);
11979
11980 vcpu->guest_debug = dbg->control;
11981 if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
11982 vcpu->guest_debug = 0;
11983
11984 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
11985 for (i = 0; i < KVM_NR_DB_REGS; ++i)
11986 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
11987 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
11988 } else {
11989 for (i = 0; i < KVM_NR_DB_REGS; i++)
11990 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
11991 }
11992 kvm_update_dr7(vcpu);
11993
11994 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
11995 vcpu->arch.singlestep_rip = kvm_get_linear_rip(vcpu);
11996
11997 /*
11998 * Trigger an rflags update that will inject or remove the trace
11999 * flags.
12000 */
12001 kvm_set_rflags(vcpu, rflags);
12002
12003 static_call(kvm_x86_update_exception_bitmap)(vcpu);
12004
12005 kvm_arch_vcpu_guestdbg_update_apicv_inhibit(vcpu->kvm);
12006
12007 r = 0;
12008
12009 out:
12010 vcpu_put(vcpu);
12011 return r;
12012 }
12013
12014 /*
12015 * Translate a guest virtual address to a guest physical address.
12016 */
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)12017 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
12018 struct kvm_translation *tr)
12019 {
12020 unsigned long vaddr = tr->linear_address;
12021 gpa_t gpa;
12022 int idx;
12023
12024 vcpu_load(vcpu);
12025
12026 idx = srcu_read_lock(&vcpu->kvm->srcu);
12027 gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
12028 srcu_read_unlock(&vcpu->kvm->srcu, idx);
12029 tr->physical_address = gpa;
12030 tr->valid = gpa != INVALID_GPA;
12031 tr->writeable = 1;
12032 tr->usermode = 0;
12033
12034 vcpu_put(vcpu);
12035 return 0;
12036 }
12037
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)12038 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
12039 {
12040 struct fxregs_state *fxsave;
12041
12042 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
12043 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
12044
12045 vcpu_load(vcpu);
12046
12047 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
12048 memcpy(fpu->fpr, fxsave->st_space, 128);
12049 fpu->fcw = fxsave->cwd;
12050 fpu->fsw = fxsave->swd;
12051 fpu->ftwx = fxsave->twd;
12052 fpu->last_opcode = fxsave->fop;
12053 fpu->last_ip = fxsave->rip;
12054 fpu->last_dp = fxsave->rdp;
12055 memcpy(fpu->xmm, fxsave->xmm_space, sizeof(fxsave->xmm_space));
12056
12057 vcpu_put(vcpu);
12058 return 0;
12059 }
12060
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)12061 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
12062 {
12063 struct fxregs_state *fxsave;
12064
12065 if (fpstate_is_confidential(&vcpu->arch.guest_fpu))
12066 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
12067
12068 vcpu_load(vcpu);
12069
12070 fxsave = &vcpu->arch.guest_fpu.fpstate->regs.fxsave;
12071
12072 memcpy(fxsave->st_space, fpu->fpr, 128);
12073 fxsave->cwd = fpu->fcw;
12074 fxsave->swd = fpu->fsw;
12075 fxsave->twd = fpu->ftwx;
12076 fxsave->fop = fpu->last_opcode;
12077 fxsave->rip = fpu->last_ip;
12078 fxsave->rdp = fpu->last_dp;
12079 memcpy(fxsave->xmm_space, fpu->xmm, sizeof(fxsave->xmm_space));
12080
12081 vcpu_put(vcpu);
12082 return 0;
12083 }
12084
store_regs(struct kvm_vcpu * vcpu)12085 static void store_regs(struct kvm_vcpu *vcpu)
12086 {
12087 BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES);
12088
12089 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS)
12090 __get_regs(vcpu, &vcpu->run->s.regs.regs);
12091
12092 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS)
12093 __get_sregs(vcpu, &vcpu->run->s.regs.sregs);
12094
12095 if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS)
12096 kvm_vcpu_ioctl_x86_get_vcpu_events(
12097 vcpu, &vcpu->run->s.regs.events);
12098 }
12099
sync_regs(struct kvm_vcpu * vcpu)12100 static int sync_regs(struct kvm_vcpu *vcpu)
12101 {
12102 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) {
12103 __set_regs(vcpu, &vcpu->run->s.regs.regs);
12104 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS;
12105 }
12106
12107 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) {
12108 struct kvm_sregs sregs = vcpu->run->s.regs.sregs;
12109
12110 if (__set_sregs(vcpu, &sregs))
12111 return -EINVAL;
12112
12113 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS;
12114 }
12115
12116 if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) {
12117 struct kvm_vcpu_events events = vcpu->run->s.regs.events;
12118
12119 if (kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events))
12120 return -EINVAL;
12121
12122 vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_EVENTS;
12123 }
12124
12125 return 0;
12126 }
12127
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)12128 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
12129 {
12130 if (kvm_check_tsc_unstable() && kvm->created_vcpus)
12131 pr_warn_once("SMP vm created on host with unstable TSC; "
12132 "guest TSC will not be reliable\n");
12133
12134 if (!kvm->arch.max_vcpu_ids)
12135 kvm->arch.max_vcpu_ids = KVM_MAX_VCPU_IDS;
12136
12137 if (id >= kvm->arch.max_vcpu_ids)
12138 return -EINVAL;
12139
12140 return static_call(kvm_x86_vcpu_precreate)(kvm);
12141 }
12142
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)12143 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
12144 {
12145 struct page *page;
12146 int r;
12147
12148 vcpu->arch.last_vmentry_cpu = -1;
12149 vcpu->arch.regs_avail = ~0;
12150 vcpu->arch.regs_dirty = ~0;
12151
12152 kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm);
12153
12154 if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu))
12155 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
12156 else
12157 vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
12158
12159 r = kvm_mmu_create(vcpu);
12160 if (r < 0)
12161 return r;
12162
12163 r = kvm_create_lapic(vcpu);
12164 if (r < 0)
12165 goto fail_mmu_destroy;
12166
12167 r = -ENOMEM;
12168
12169 page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
12170 if (!page)
12171 goto fail_free_lapic;
12172 vcpu->arch.pio_data = page_address(page);
12173
12174 vcpu->arch.mce_banks = kcalloc(KVM_MAX_MCE_BANKS * 4, sizeof(u64),
12175 GFP_KERNEL_ACCOUNT);
12176 vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64),
12177 GFP_KERNEL_ACCOUNT);
12178 if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks)
12179 goto fail_free_mce_banks;
12180 vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
12181
12182 if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask,
12183 GFP_KERNEL_ACCOUNT))
12184 goto fail_free_mce_banks;
12185
12186 if (!alloc_emulate_ctxt(vcpu))
12187 goto free_wbinvd_dirty_mask;
12188
12189 if (!fpu_alloc_guest_fpstate(&vcpu->arch.guest_fpu)) {
12190 pr_err("failed to allocate vcpu's fpu\n");
12191 goto free_emulate_ctxt;
12192 }
12193
12194 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
12195 vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
12196
12197 vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT;
12198
12199 kvm_async_pf_hash_reset(vcpu);
12200
12201 vcpu->arch.perf_capabilities = kvm_caps.supported_perf_cap;
12202 kvm_pmu_init(vcpu);
12203
12204 vcpu->arch.pending_external_vector = -1;
12205 vcpu->arch.preempted_in_kernel = false;
12206
12207 #if IS_ENABLED(CONFIG_HYPERV)
12208 vcpu->arch.hv_root_tdp = INVALID_PAGE;
12209 #endif
12210
12211 r = static_call(kvm_x86_vcpu_create)(vcpu);
12212 if (r)
12213 goto free_guest_fpu;
12214
12215 vcpu->arch.arch_capabilities = kvm_get_arch_capabilities();
12216 vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT;
12217 kvm_xen_init_vcpu(vcpu);
12218 kvm_vcpu_mtrr_init(vcpu);
12219 vcpu_load(vcpu);
12220 kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz);
12221 kvm_vcpu_reset(vcpu, false);
12222 kvm_init_mmu(vcpu);
12223 vcpu_put(vcpu);
12224 return 0;
12225
12226 free_guest_fpu:
12227 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
12228 free_emulate_ctxt:
12229 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
12230 free_wbinvd_dirty_mask:
12231 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
12232 fail_free_mce_banks:
12233 kfree(vcpu->arch.mce_banks);
12234 kfree(vcpu->arch.mci_ctl2_banks);
12235 free_page((unsigned long)vcpu->arch.pio_data);
12236 fail_free_lapic:
12237 kvm_free_lapic(vcpu);
12238 fail_mmu_destroy:
12239 kvm_mmu_destroy(vcpu);
12240 return r;
12241 }
12242
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)12243 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
12244 {
12245 struct kvm *kvm = vcpu->kvm;
12246
12247 if (mutex_lock_killable(&vcpu->mutex))
12248 return;
12249 vcpu_load(vcpu);
12250 kvm_synchronize_tsc(vcpu, NULL);
12251 vcpu_put(vcpu);
12252
12253 /* poll control enabled by default */
12254 vcpu->arch.msr_kvm_poll_control = 1;
12255
12256 mutex_unlock(&vcpu->mutex);
12257
12258 if (kvmclock_periodic_sync && vcpu->vcpu_idx == 0)
12259 schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
12260 KVMCLOCK_SYNC_PERIOD);
12261 }
12262
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)12263 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
12264 {
12265 int idx;
12266
12267 kvmclock_reset(vcpu);
12268
12269 static_call(kvm_x86_vcpu_free)(vcpu);
12270
12271 kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);
12272 free_cpumask_var(vcpu->arch.wbinvd_dirty_mask);
12273 fpu_free_guest_fpstate(&vcpu->arch.guest_fpu);
12274
12275 kvm_xen_destroy_vcpu(vcpu);
12276 kvm_hv_vcpu_uninit(vcpu);
12277 kvm_pmu_destroy(vcpu);
12278 kfree(vcpu->arch.mce_banks);
12279 kfree(vcpu->arch.mci_ctl2_banks);
12280 kvm_free_lapic(vcpu);
12281 idx = srcu_read_lock(&vcpu->kvm->srcu);
12282 kvm_mmu_destroy(vcpu);
12283 srcu_read_unlock(&vcpu->kvm->srcu, idx);
12284 free_page((unsigned long)vcpu->arch.pio_data);
12285 kvfree(vcpu->arch.cpuid_entries);
12286 }
12287
kvm_vcpu_reset(struct kvm_vcpu * vcpu,bool init_event)12288 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
12289 {
12290 struct kvm_cpuid_entry2 *cpuid_0x1;
12291 unsigned long old_cr0 = kvm_read_cr0(vcpu);
12292 unsigned long new_cr0;
12293
12294 /*
12295 * Several of the "set" flows, e.g. ->set_cr0(), read other registers
12296 * to handle side effects. RESET emulation hits those flows and relies
12297 * on emulated/virtualized registers, including those that are loaded
12298 * into hardware, to be zeroed at vCPU creation. Use CRs as a sentinel
12299 * to detect improper or missing initialization.
12300 */
12301 WARN_ON_ONCE(!init_event &&
12302 (old_cr0 || kvm_read_cr3(vcpu) || kvm_read_cr4(vcpu)));
12303
12304 /*
12305 * SVM doesn't unconditionally VM-Exit on INIT and SHUTDOWN, thus it's
12306 * possible to INIT the vCPU while L2 is active. Force the vCPU back
12307 * into L1 as EFER.SVME is cleared on INIT (along with all other EFER
12308 * bits), i.e. virtualization is disabled.
12309 */
12310 if (is_guest_mode(vcpu))
12311 kvm_leave_nested(vcpu);
12312
12313 kvm_lapic_reset(vcpu, init_event);
12314
12315 WARN_ON_ONCE(is_guest_mode(vcpu) || is_smm(vcpu));
12316 vcpu->arch.hflags = 0;
12317
12318 vcpu->arch.smi_pending = 0;
12319 vcpu->arch.smi_count = 0;
12320 atomic_set(&vcpu->arch.nmi_queued, 0);
12321 vcpu->arch.nmi_pending = 0;
12322 vcpu->arch.nmi_injected = false;
12323 kvm_clear_interrupt_queue(vcpu);
12324 kvm_clear_exception_queue(vcpu);
12325
12326 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
12327 kvm_update_dr0123(vcpu);
12328 vcpu->arch.dr6 = DR6_ACTIVE_LOW;
12329 vcpu->arch.dr7 = DR7_FIXED_1;
12330 kvm_update_dr7(vcpu);
12331
12332 vcpu->arch.cr2 = 0;
12333
12334 kvm_make_request(KVM_REQ_EVENT, vcpu);
12335 vcpu->arch.apf.msr_en_val = 0;
12336 vcpu->arch.apf.msr_int_val = 0;
12337 vcpu->arch.st.msr_val = 0;
12338
12339 kvmclock_reset(vcpu);
12340
12341 kvm_clear_async_pf_completion_queue(vcpu);
12342 kvm_async_pf_hash_reset(vcpu);
12343 vcpu->arch.apf.halted = false;
12344
12345 if (vcpu->arch.guest_fpu.fpstate && kvm_mpx_supported()) {
12346 struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate;
12347
12348 /*
12349 * All paths that lead to INIT are required to load the guest's
12350 * FPU state (because most paths are buried in KVM_RUN).
12351 */
12352 if (init_event)
12353 kvm_put_guest_fpu(vcpu);
12354
12355 fpstate_clear_xstate_component(fpstate, XFEATURE_BNDREGS);
12356 fpstate_clear_xstate_component(fpstate, XFEATURE_BNDCSR);
12357
12358 if (init_event)
12359 kvm_load_guest_fpu(vcpu);
12360 }
12361
12362 if (!init_event) {
12363 vcpu->arch.smbase = 0x30000;
12364
12365 vcpu->arch.msr_misc_features_enables = 0;
12366 vcpu->arch.ia32_misc_enable_msr = MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL |
12367 MSR_IA32_MISC_ENABLE_BTS_UNAVAIL;
12368
12369 __kvm_set_xcr(vcpu, 0, XFEATURE_MASK_FP);
12370 __kvm_set_msr(vcpu, MSR_IA32_XSS, 0, true);
12371 }
12372
12373 /* All GPRs except RDX (handled below) are zeroed on RESET/INIT. */
12374 memset(vcpu->arch.regs, 0, sizeof(vcpu->arch.regs));
12375 kvm_register_mark_dirty(vcpu, VCPU_REGS_RSP);
12376
12377 /*
12378 * Fall back to KVM's default Family/Model/Stepping of 0x600 (P6/Athlon)
12379 * if no CPUID match is found. Note, it's impossible to get a match at
12380 * RESET since KVM emulates RESET before exposing the vCPU to userspace,
12381 * i.e. it's impossible for kvm_find_cpuid_entry() to find a valid entry
12382 * on RESET. But, go through the motions in case that's ever remedied.
12383 */
12384 cpuid_0x1 = kvm_find_cpuid_entry(vcpu, 1);
12385 kvm_rdx_write(vcpu, cpuid_0x1 ? cpuid_0x1->eax : 0x600);
12386
12387 static_call(kvm_x86_vcpu_reset)(vcpu, init_event);
12388
12389 kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
12390 kvm_rip_write(vcpu, 0xfff0);
12391
12392 vcpu->arch.cr3 = 0;
12393 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
12394
12395 /*
12396 * CR0.CD/NW are set on RESET, preserved on INIT. Note, some versions
12397 * of Intel's SDM list CD/NW as being set on INIT, but they contradict
12398 * (or qualify) that with a footnote stating that CD/NW are preserved.
12399 */
12400 new_cr0 = X86_CR0_ET;
12401 if (init_event)
12402 new_cr0 |= (old_cr0 & (X86_CR0_NW | X86_CR0_CD));
12403 else
12404 new_cr0 |= X86_CR0_NW | X86_CR0_CD;
12405
12406 static_call(kvm_x86_set_cr0)(vcpu, new_cr0);
12407 static_call(kvm_x86_set_cr4)(vcpu, 0);
12408 static_call(kvm_x86_set_efer)(vcpu, 0);
12409 static_call(kvm_x86_update_exception_bitmap)(vcpu);
12410
12411 /*
12412 * On the standard CR0/CR4/EFER modification paths, there are several
12413 * complex conditions determining whether the MMU has to be reset and/or
12414 * which PCIDs have to be flushed. However, CR0.WP and the paging-related
12415 * bits in CR4 and EFER are irrelevant if CR0.PG was '0'; and a reset+flush
12416 * is needed anyway if CR0.PG was '1' (which can only happen for INIT, as
12417 * CR0 will be '0' prior to RESET). So we only need to check CR0.PG here.
12418 */
12419 if (old_cr0 & X86_CR0_PG) {
12420 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
12421 kvm_mmu_reset_context(vcpu);
12422 }
12423
12424 /*
12425 * Intel's SDM states that all TLB entries are flushed on INIT. AMD's
12426 * APM states the TLBs are untouched by INIT, but it also states that
12427 * the TLBs are flushed on "External initialization of the processor."
12428 * Flush the guest TLB regardless of vendor, there is no meaningful
12429 * benefit in relying on the guest to flush the TLB immediately after
12430 * INIT. A spurious TLB flush is benign and likely negligible from a
12431 * performance perspective.
12432 */
12433 if (init_event)
12434 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
12435 }
12436 EXPORT_SYMBOL_GPL(kvm_vcpu_reset);
12437
kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu * vcpu,u8 vector)12438 void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
12439 {
12440 struct kvm_segment cs;
12441
12442 kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
12443 cs.selector = vector << 8;
12444 cs.base = vector << 12;
12445 kvm_set_segment(vcpu, &cs, VCPU_SREG_CS);
12446 kvm_rip_write(vcpu, 0);
12447 }
12448 EXPORT_SYMBOL_GPL(kvm_vcpu_deliver_sipi_vector);
12449
kvm_arch_hardware_enable(void)12450 int kvm_arch_hardware_enable(void)
12451 {
12452 struct kvm *kvm;
12453 struct kvm_vcpu *vcpu;
12454 unsigned long i;
12455 int ret;
12456 u64 local_tsc;
12457 u64 max_tsc = 0;
12458 bool stable, backwards_tsc = false;
12459
12460 kvm_user_return_msr_cpu_online();
12461
12462 ret = kvm_x86_check_processor_compatibility();
12463 if (ret)
12464 return ret;
12465
12466 ret = static_call(kvm_x86_hardware_enable)();
12467 if (ret != 0)
12468 return ret;
12469
12470 local_tsc = rdtsc();
12471 stable = !kvm_check_tsc_unstable();
12472 list_for_each_entry(kvm, &vm_list, vm_list) {
12473 kvm_for_each_vcpu(i, vcpu, kvm) {
12474 if (!stable && vcpu->cpu == smp_processor_id())
12475 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
12476 if (stable && vcpu->arch.last_host_tsc > local_tsc) {
12477 backwards_tsc = true;
12478 if (vcpu->arch.last_host_tsc > max_tsc)
12479 max_tsc = vcpu->arch.last_host_tsc;
12480 }
12481 }
12482 }
12483
12484 /*
12485 * Sometimes, even reliable TSCs go backwards. This happens on
12486 * platforms that reset TSC during suspend or hibernate actions, but
12487 * maintain synchronization. We must compensate. Fortunately, we can
12488 * detect that condition here, which happens early in CPU bringup,
12489 * before any KVM threads can be running. Unfortunately, we can't
12490 * bring the TSCs fully up to date with real time, as we aren't yet far
12491 * enough into CPU bringup that we know how much real time has actually
12492 * elapsed; our helper function, ktime_get_boottime_ns() will be using boot
12493 * variables that haven't been updated yet.
12494 *
12495 * So we simply find the maximum observed TSC above, then record the
12496 * adjustment to TSC in each VCPU. When the VCPU later gets loaded,
12497 * the adjustment will be applied. Note that we accumulate
12498 * adjustments, in case multiple suspend cycles happen before some VCPU
12499 * gets a chance to run again. In the event that no KVM threads get a
12500 * chance to run, we will miss the entire elapsed period, as we'll have
12501 * reset last_host_tsc, so VCPUs will not have the TSC adjusted and may
12502 * loose cycle time. This isn't too big a deal, since the loss will be
12503 * uniform across all VCPUs (not to mention the scenario is extremely
12504 * unlikely). It is possible that a second hibernate recovery happens
12505 * much faster than a first, causing the observed TSC here to be
12506 * smaller; this would require additional padding adjustment, which is
12507 * why we set last_host_tsc to the local tsc observed here.
12508 *
12509 * N.B. - this code below runs only on platforms with reliable TSC,
12510 * as that is the only way backwards_tsc is set above. Also note
12511 * that this runs for ALL vcpus, which is not a bug; all VCPUs should
12512 * have the same delta_cyc adjustment applied if backwards_tsc
12513 * is detected. Note further, this adjustment is only done once,
12514 * as we reset last_host_tsc on all VCPUs to stop this from being
12515 * called multiple times (one for each physical CPU bringup).
12516 *
12517 * Platforms with unreliable TSCs don't have to deal with this, they
12518 * will be compensated by the logic in vcpu_load, which sets the TSC to
12519 * catchup mode. This will catchup all VCPUs to real time, but cannot
12520 * guarantee that they stay in perfect synchronization.
12521 */
12522 if (backwards_tsc) {
12523 u64 delta_cyc = max_tsc - local_tsc;
12524 list_for_each_entry(kvm, &vm_list, vm_list) {
12525 kvm->arch.backwards_tsc_observed = true;
12526 kvm_for_each_vcpu(i, vcpu, kvm) {
12527 vcpu->arch.tsc_offset_adjustment += delta_cyc;
12528 vcpu->arch.last_host_tsc = local_tsc;
12529 kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
12530 }
12531
12532 /*
12533 * We have to disable TSC offset matching.. if you were
12534 * booting a VM while issuing an S4 host suspend....
12535 * you may have some problem. Solving this issue is
12536 * left as an exercise to the reader.
12537 */
12538 kvm->arch.last_tsc_nsec = 0;
12539 kvm->arch.last_tsc_write = 0;
12540 }
12541
12542 }
12543 return 0;
12544 }
12545
kvm_arch_hardware_disable(void)12546 void kvm_arch_hardware_disable(void)
12547 {
12548 static_call(kvm_x86_hardware_disable)();
12549 drop_user_return_notifiers();
12550 }
12551
kvm_vcpu_is_reset_bsp(struct kvm_vcpu * vcpu)12552 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu)
12553 {
12554 return vcpu->kvm->arch.bsp_vcpu_id == vcpu->vcpu_id;
12555 }
12556
kvm_vcpu_is_bsp(struct kvm_vcpu * vcpu)12557 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
12558 {
12559 return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
12560 }
12561
kvm_arch_sched_in(struct kvm_vcpu * vcpu,int cpu)12562 void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
12563 {
12564 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
12565
12566 vcpu->arch.l1tf_flush_l1d = true;
12567 if (pmu->version && unlikely(pmu->event_count)) {
12568 pmu->need_cleanup = true;
12569 kvm_make_request(KVM_REQ_PMU, vcpu);
12570 }
12571 static_call(kvm_x86_sched_in)(vcpu, cpu);
12572 }
12573
kvm_arch_free_vm(struct kvm * kvm)12574 void kvm_arch_free_vm(struct kvm *kvm)
12575 {
12576 #if IS_ENABLED(CONFIG_HYPERV)
12577 kfree(kvm->arch.hv_pa_pg);
12578 #endif
12579 __kvm_arch_free_vm(kvm);
12580 }
12581
12582
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)12583 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
12584 {
12585 int ret;
12586 unsigned long flags;
12587
12588 if (!kvm_is_vm_type_supported(type))
12589 return -EINVAL;
12590
12591 kvm->arch.vm_type = type;
12592 kvm->arch.has_private_mem =
12593 (type == KVM_X86_SW_PROTECTED_VM);
12594
12595 ret = kvm_page_track_init(kvm);
12596 if (ret)
12597 goto out;
12598
12599 kvm_mmu_init_vm(kvm);
12600
12601 ret = static_call(kvm_x86_vm_init)(kvm);
12602 if (ret)
12603 goto out_uninit_mmu;
12604
12605 INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list);
12606 atomic_set(&kvm->arch.noncoherent_dma_count, 0);
12607
12608 /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
12609 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
12610 /* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
12611 set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
12612 &kvm->arch.irq_sources_bitmap);
12613
12614 raw_spin_lock_init(&kvm->arch.tsc_write_lock);
12615 mutex_init(&kvm->arch.apic_map_lock);
12616 seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
12617 kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
12618
12619 raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
12620 pvclock_update_vm_gtod_copy(kvm);
12621 raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
12622
12623 kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz;
12624 kvm->arch.guest_can_read_msr_platform_info = true;
12625 kvm->arch.enable_pmu = enable_pmu;
12626
12627 #if IS_ENABLED(CONFIG_HYPERV)
12628 spin_lock_init(&kvm->arch.hv_root_tdp_lock);
12629 kvm->arch.hv_root_tdp = INVALID_PAGE;
12630 #endif
12631
12632 INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn);
12633 INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn);
12634
12635 kvm_apicv_init(kvm);
12636 kvm_hv_init_vm(kvm);
12637 kvm_xen_init_vm(kvm);
12638
12639 return 0;
12640
12641 out_uninit_mmu:
12642 kvm_mmu_uninit_vm(kvm);
12643 kvm_page_track_cleanup(kvm);
12644 out:
12645 return ret;
12646 }
12647
kvm_arch_post_init_vm(struct kvm * kvm)12648 int kvm_arch_post_init_vm(struct kvm *kvm)
12649 {
12650 return kvm_mmu_post_init_vm(kvm);
12651 }
12652
kvm_unload_vcpu_mmu(struct kvm_vcpu * vcpu)12653 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
12654 {
12655 vcpu_load(vcpu);
12656 kvm_mmu_unload(vcpu);
12657 vcpu_put(vcpu);
12658 }
12659
kvm_unload_vcpu_mmus(struct kvm * kvm)12660 static void kvm_unload_vcpu_mmus(struct kvm *kvm)
12661 {
12662 unsigned long i;
12663 struct kvm_vcpu *vcpu;
12664
12665 kvm_for_each_vcpu(i, vcpu, kvm) {
12666 kvm_clear_async_pf_completion_queue(vcpu);
12667 kvm_unload_vcpu_mmu(vcpu);
12668 }
12669 }
12670
kvm_arch_sync_events(struct kvm * kvm)12671 void kvm_arch_sync_events(struct kvm *kvm)
12672 {
12673 cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work);
12674 cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work);
12675 kvm_free_pit(kvm);
12676 }
12677
12678 /**
12679 * __x86_set_memory_region: Setup KVM internal memory slot
12680 *
12681 * @kvm: the kvm pointer to the VM.
12682 * @id: the slot ID to setup.
12683 * @gpa: the GPA to install the slot (unused when @size == 0).
12684 * @size: the size of the slot. Set to zero to uninstall a slot.
12685 *
12686 * This function helps to setup a KVM internal memory slot. Specify
12687 * @size > 0 to install a new slot, while @size == 0 to uninstall a
12688 * slot. The return code can be one of the following:
12689 *
12690 * HVA: on success (uninstall will return a bogus HVA)
12691 * -errno: on error
12692 *
12693 * The caller should always use IS_ERR() to check the return value
12694 * before use. Note, the KVM internal memory slots are guaranteed to
12695 * remain valid and unchanged until the VM is destroyed, i.e., the
12696 * GPA->HVA translation will not change. However, the HVA is a user
12697 * address, i.e. its accessibility is not guaranteed, and must be
12698 * accessed via __copy_{to,from}_user().
12699 */
__x86_set_memory_region(struct kvm * kvm,int id,gpa_t gpa,u32 size)12700 void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
12701 u32 size)
12702 {
12703 int i, r;
12704 unsigned long hva, old_npages;
12705 struct kvm_memslots *slots = kvm_memslots(kvm);
12706 struct kvm_memory_slot *slot;
12707
12708 /* Called with kvm->slots_lock held. */
12709 if (WARN_ON(id >= KVM_MEM_SLOTS_NUM))
12710 return ERR_PTR_USR(-EINVAL);
12711
12712 slot = id_to_memslot(slots, id);
12713 if (size) {
12714 if (slot && slot->npages)
12715 return ERR_PTR_USR(-EEXIST);
12716
12717 /*
12718 * MAP_SHARED to prevent internal slot pages from being moved
12719 * by fork()/COW.
12720 */
12721 hva = vm_mmap(NULL, 0, size, PROT_READ | PROT_WRITE,
12722 MAP_SHARED | MAP_ANONYMOUS, 0);
12723 if (IS_ERR_VALUE(hva))
12724 return (void __user *)hva;
12725 } else {
12726 if (!slot || !slot->npages)
12727 return NULL;
12728
12729 old_npages = slot->npages;
12730 hva = slot->userspace_addr;
12731 }
12732
12733 for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
12734 struct kvm_userspace_memory_region2 m;
12735
12736 m.slot = id | (i << 16);
12737 m.flags = 0;
12738 m.guest_phys_addr = gpa;
12739 m.userspace_addr = hva;
12740 m.memory_size = size;
12741 r = __kvm_set_memory_region(kvm, &m);
12742 if (r < 0)
12743 return ERR_PTR_USR(r);
12744 }
12745
12746 if (!size)
12747 vm_munmap(hva, old_npages * PAGE_SIZE);
12748
12749 return (void __user *)hva;
12750 }
12751 EXPORT_SYMBOL_GPL(__x86_set_memory_region);
12752
kvm_arch_pre_destroy_vm(struct kvm * kvm)12753 void kvm_arch_pre_destroy_vm(struct kvm *kvm)
12754 {
12755 kvm_mmu_pre_destroy_vm(kvm);
12756 }
12757
kvm_arch_destroy_vm(struct kvm * kvm)12758 void kvm_arch_destroy_vm(struct kvm *kvm)
12759 {
12760 if (current->mm == kvm->mm) {
12761 /*
12762 * Free memory regions allocated on behalf of userspace,
12763 * unless the memory map has changed due to process exit
12764 * or fd copying.
12765 */
12766 mutex_lock(&kvm->slots_lock);
12767 __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
12768 0, 0);
12769 __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
12770 0, 0);
12771 __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0);
12772 mutex_unlock(&kvm->slots_lock);
12773 }
12774 kvm_unload_vcpu_mmus(kvm);
12775 static_call_cond(kvm_x86_vm_destroy)(kvm);
12776 kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1));
12777 kvm_pic_destroy(kvm);
12778 kvm_ioapic_destroy(kvm);
12779 kvm_destroy_vcpus(kvm);
12780 kvfree(rcu_dereference_check(kvm->arch.apic_map, 1));
12781 kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1));
12782 kvm_mmu_uninit_vm(kvm);
12783 kvm_page_track_cleanup(kvm);
12784 kvm_xen_destroy_vm(kvm);
12785 kvm_hv_destroy_vm(kvm);
12786 }
12787
memslot_rmap_free(struct kvm_memory_slot * slot)12788 static void memslot_rmap_free(struct kvm_memory_slot *slot)
12789 {
12790 int i;
12791
12792 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
12793 vfree(slot->arch.rmap[i]);
12794 slot->arch.rmap[i] = NULL;
12795 }
12796 }
12797
kvm_arch_free_memslot(struct kvm * kvm,struct kvm_memory_slot * slot)12798 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
12799 {
12800 int i;
12801
12802 memslot_rmap_free(slot);
12803
12804 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
12805 vfree(slot->arch.lpage_info[i - 1]);
12806 slot->arch.lpage_info[i - 1] = NULL;
12807 }
12808
12809 kvm_page_track_free_memslot(slot);
12810 }
12811
memslot_rmap_alloc(struct kvm_memory_slot * slot,unsigned long npages)12812 int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages)
12813 {
12814 const int sz = sizeof(*slot->arch.rmap[0]);
12815 int i;
12816
12817 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
12818 int level = i + 1;
12819 int lpages = __kvm_mmu_slot_lpages(slot, npages, level);
12820
12821 if (slot->arch.rmap[i])
12822 continue;
12823
12824 slot->arch.rmap[i] = __vcalloc(lpages, sz, GFP_KERNEL_ACCOUNT);
12825 if (!slot->arch.rmap[i]) {
12826 memslot_rmap_free(slot);
12827 return -ENOMEM;
12828 }
12829 }
12830
12831 return 0;
12832 }
12833
kvm_alloc_memslot_metadata(struct kvm * kvm,struct kvm_memory_slot * slot)12834 static int kvm_alloc_memslot_metadata(struct kvm *kvm,
12835 struct kvm_memory_slot *slot)
12836 {
12837 unsigned long npages = slot->npages;
12838 int i, r;
12839
12840 /*
12841 * Clear out the previous array pointers for the KVM_MR_MOVE case. The
12842 * old arrays will be freed by __kvm_set_memory_region() if installing
12843 * the new memslot is successful.
12844 */
12845 memset(&slot->arch, 0, sizeof(slot->arch));
12846
12847 if (kvm_memslots_have_rmaps(kvm)) {
12848 r = memslot_rmap_alloc(slot, npages);
12849 if (r)
12850 return r;
12851 }
12852
12853 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
12854 struct kvm_lpage_info *linfo;
12855 unsigned long ugfn;
12856 int lpages;
12857 int level = i + 1;
12858
12859 lpages = __kvm_mmu_slot_lpages(slot, npages, level);
12860
12861 linfo = __vcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT);
12862 if (!linfo)
12863 goto out_free;
12864
12865 slot->arch.lpage_info[i - 1] = linfo;
12866
12867 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
12868 linfo[0].disallow_lpage = 1;
12869 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
12870 linfo[lpages - 1].disallow_lpage = 1;
12871 ugfn = slot->userspace_addr >> PAGE_SHIFT;
12872 /*
12873 * If the gfn and userspace address are not aligned wrt each
12874 * other, disable large page support for this slot.
12875 */
12876 if ((slot->base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1)) {
12877 unsigned long j;
12878
12879 for (j = 0; j < lpages; ++j)
12880 linfo[j].disallow_lpage = 1;
12881 }
12882 }
12883
12884 #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
12885 kvm_mmu_init_memslot_memory_attributes(kvm, slot);
12886 #endif
12887
12888 if (kvm_page_track_create_memslot(kvm, slot, npages))
12889 goto out_free;
12890
12891 return 0;
12892
12893 out_free:
12894 memslot_rmap_free(slot);
12895
12896 for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) {
12897 vfree(slot->arch.lpage_info[i - 1]);
12898 slot->arch.lpage_info[i - 1] = NULL;
12899 }
12900 return -ENOMEM;
12901 }
12902
kvm_arch_memslots_updated(struct kvm * kvm,u64 gen)12903 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
12904 {
12905 struct kvm_vcpu *vcpu;
12906 unsigned long i;
12907
12908 /*
12909 * memslots->generation has been incremented.
12910 * mmio generation may have reached its maximum value.
12911 */
12912 kvm_mmu_invalidate_mmio_sptes(kvm, gen);
12913
12914 /* Force re-initialization of steal_time cache */
12915 kvm_for_each_vcpu(i, vcpu, kvm)
12916 kvm_vcpu_kick(vcpu);
12917 }
12918
kvm_arch_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)12919 int kvm_arch_prepare_memory_region(struct kvm *kvm,
12920 const struct kvm_memory_slot *old,
12921 struct kvm_memory_slot *new,
12922 enum kvm_mr_change change)
12923 {
12924 /*
12925 * KVM doesn't support moving memslots when there are external page
12926 * trackers attached to the VM, i.e. if KVMGT is in use.
12927 */
12928 if (change == KVM_MR_MOVE && kvm_page_track_has_external_user(kvm))
12929 return -EINVAL;
12930
12931 if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) {
12932 if ((new->base_gfn + new->npages - 1) > kvm_mmu_max_gfn())
12933 return -EINVAL;
12934
12935 return kvm_alloc_memslot_metadata(kvm, new);
12936 }
12937
12938 if (change == KVM_MR_FLAGS_ONLY)
12939 memcpy(&new->arch, &old->arch, sizeof(old->arch));
12940 else if (WARN_ON_ONCE(change != KVM_MR_DELETE))
12941 return -EIO;
12942
12943 return 0;
12944 }
12945
12946
kvm_mmu_update_cpu_dirty_logging(struct kvm * kvm,bool enable)12947 static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable)
12948 {
12949 int nr_slots;
12950
12951 if (!kvm_x86_ops.cpu_dirty_log_size)
12952 return;
12953
12954 nr_slots = atomic_read(&kvm->nr_memslots_dirty_logging);
12955 if ((enable && nr_slots == 1) || !nr_slots)
12956 kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING);
12957 }
12958
kvm_mmu_slot_apply_flags(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)12959 static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
12960 struct kvm_memory_slot *old,
12961 const struct kvm_memory_slot *new,
12962 enum kvm_mr_change change)
12963 {
12964 u32 old_flags = old ? old->flags : 0;
12965 u32 new_flags = new ? new->flags : 0;
12966 bool log_dirty_pages = new_flags & KVM_MEM_LOG_DIRTY_PAGES;
12967
12968 /*
12969 * Update CPU dirty logging if dirty logging is being toggled. This
12970 * applies to all operations.
12971 */
12972 if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES)
12973 kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages);
12974
12975 /*
12976 * Nothing more to do for RO slots (which can't be dirtied and can't be
12977 * made writable) or CREATE/MOVE/DELETE of a slot.
12978 *
12979 * For a memslot with dirty logging disabled:
12980 * CREATE: No dirty mappings will already exist.
12981 * MOVE/DELETE: The old mappings will already have been cleaned up by
12982 * kvm_arch_flush_shadow_memslot()
12983 *
12984 * For a memslot with dirty logging enabled:
12985 * CREATE: No shadow pages exist, thus nothing to write-protect
12986 * and no dirty bits to clear.
12987 * MOVE/DELETE: The old mappings will already have been cleaned up by
12988 * kvm_arch_flush_shadow_memslot().
12989 */
12990 if ((change != KVM_MR_FLAGS_ONLY) || (new_flags & KVM_MEM_READONLY))
12991 return;
12992
12993 /*
12994 * READONLY and non-flags changes were filtered out above, and the only
12995 * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty
12996 * logging isn't being toggled on or off.
12997 */
12998 if (WARN_ON_ONCE(!((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES)))
12999 return;
13000
13001 if (!log_dirty_pages) {
13002 /*
13003 * Dirty logging tracks sptes in 4k granularity, meaning that
13004 * large sptes have to be split. If live migration succeeds,
13005 * the guest in the source machine will be destroyed and large
13006 * sptes will be created in the destination. However, if the
13007 * guest continues to run in the source machine (for example if
13008 * live migration fails), small sptes will remain around and
13009 * cause bad performance.
13010 *
13011 * Scan sptes if dirty logging has been stopped, dropping those
13012 * which can be collapsed into a single large-page spte. Later
13013 * page faults will create the large-page sptes.
13014 */
13015 kvm_mmu_zap_collapsible_sptes(kvm, new);
13016 } else {
13017 /*
13018 * Initially-all-set does not require write protecting any page,
13019 * because they're all assumed to be dirty.
13020 */
13021 if (kvm_dirty_log_manual_protect_and_init_set(kvm))
13022 return;
13023
13024 if (READ_ONCE(eager_page_split))
13025 kvm_mmu_slot_try_split_huge_pages(kvm, new, PG_LEVEL_4K);
13026
13027 if (kvm_x86_ops.cpu_dirty_log_size) {
13028 kvm_mmu_slot_leaf_clear_dirty(kvm, new);
13029 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M);
13030 } else {
13031 kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K);
13032 }
13033
13034 /*
13035 * Unconditionally flush the TLBs after enabling dirty logging.
13036 * A flush is almost always going to be necessary (see below),
13037 * and unconditionally flushing allows the helpers to omit
13038 * the subtly complex checks when removing write access.
13039 *
13040 * Do the flush outside of mmu_lock to reduce the amount of
13041 * time mmu_lock is held. Flushing after dropping mmu_lock is
13042 * safe as KVM only needs to guarantee the slot is fully
13043 * write-protected before returning to userspace, i.e. before
13044 * userspace can consume the dirty status.
13045 *
13046 * Flushing outside of mmu_lock requires KVM to be careful when
13047 * making decisions based on writable status of an SPTE, e.g. a
13048 * !writable SPTE doesn't guarantee a CPU can't perform writes.
13049 *
13050 * Specifically, KVM also write-protects guest page tables to
13051 * monitor changes when using shadow paging, and must guarantee
13052 * no CPUs can write to those page before mmu_lock is dropped.
13053 * Because CPUs may have stale TLB entries at this point, a
13054 * !writable SPTE doesn't guarantee CPUs can't perform writes.
13055 *
13056 * KVM also allows making SPTES writable outside of mmu_lock,
13057 * e.g. to allow dirty logging without taking mmu_lock.
13058 *
13059 * To handle these scenarios, KVM uses a separate software-only
13060 * bit (MMU-writable) to track if a SPTE is !writable due to
13061 * a guest page table being write-protected (KVM clears the
13062 * MMU-writable flag when write-protecting for shadow paging).
13063 *
13064 * The use of MMU-writable is also the primary motivation for
13065 * the unconditional flush. Because KVM must guarantee that a
13066 * CPU doesn't contain stale, writable TLB entries for a
13067 * !MMU-writable SPTE, KVM must flush if it encounters any
13068 * MMU-writable SPTE regardless of whether the actual hardware
13069 * writable bit was set. I.e. KVM is almost guaranteed to need
13070 * to flush, while unconditionally flushing allows the "remove
13071 * write access" helpers to ignore MMU-writable entirely.
13072 *
13073 * See is_writable_pte() for more details (the case involving
13074 * access-tracked SPTEs is particularly relevant).
13075 */
13076 kvm_flush_remote_tlbs_memslot(kvm, new);
13077 }
13078 }
13079
kvm_arch_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)13080 void kvm_arch_commit_memory_region(struct kvm *kvm,
13081 struct kvm_memory_slot *old,
13082 const struct kvm_memory_slot *new,
13083 enum kvm_mr_change change)
13084 {
13085 if (change == KVM_MR_DELETE)
13086 kvm_page_track_delete_slot(kvm, old);
13087
13088 if (!kvm->arch.n_requested_mmu_pages &&
13089 (change == KVM_MR_CREATE || change == KVM_MR_DELETE)) {
13090 unsigned long nr_mmu_pages;
13091
13092 nr_mmu_pages = kvm->nr_memslot_pages / KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO;
13093 nr_mmu_pages = max(nr_mmu_pages, KVM_MIN_ALLOC_MMU_PAGES);
13094 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
13095 }
13096
13097 kvm_mmu_slot_apply_flags(kvm, old, new, change);
13098
13099 /* Free the arrays associated with the old memslot. */
13100 if (change == KVM_MR_MOVE)
13101 kvm_arch_free_memslot(kvm, old);
13102 }
13103
kvm_guest_apic_has_interrupt(struct kvm_vcpu * vcpu)13104 static inline bool kvm_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
13105 {
13106 return (is_guest_mode(vcpu) &&
13107 static_call(kvm_x86_guest_apic_has_interrupt)(vcpu));
13108 }
13109
kvm_vcpu_has_events(struct kvm_vcpu * vcpu)13110 static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu)
13111 {
13112 if (!list_empty_careful(&vcpu->async_pf.done))
13113 return true;
13114
13115 if (kvm_apic_has_pending_init_or_sipi(vcpu) &&
13116 kvm_apic_init_sipi_allowed(vcpu))
13117 return true;
13118
13119 if (vcpu->arch.pv.pv_unhalted)
13120 return true;
13121
13122 if (kvm_is_exception_pending(vcpu))
13123 return true;
13124
13125 if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
13126 (vcpu->arch.nmi_pending &&
13127 static_call(kvm_x86_nmi_allowed)(vcpu, false)))
13128 return true;
13129
13130 #ifdef CONFIG_KVM_SMM
13131 if (kvm_test_request(KVM_REQ_SMI, vcpu) ||
13132 (vcpu->arch.smi_pending &&
13133 static_call(kvm_x86_smi_allowed)(vcpu, false)))
13134 return true;
13135 #endif
13136
13137 if (kvm_test_request(KVM_REQ_PMI, vcpu))
13138 return true;
13139
13140 if (kvm_arch_interrupt_allowed(vcpu) &&
13141 (kvm_cpu_has_interrupt(vcpu) ||
13142 kvm_guest_apic_has_interrupt(vcpu)))
13143 return true;
13144
13145 if (kvm_hv_has_stimer_pending(vcpu))
13146 return true;
13147
13148 if (is_guest_mode(vcpu) &&
13149 kvm_x86_ops.nested_ops->has_events &&
13150 kvm_x86_ops.nested_ops->has_events(vcpu))
13151 return true;
13152
13153 if (kvm_xen_has_pending_events(vcpu))
13154 return true;
13155
13156 return false;
13157 }
13158
kvm_arch_vcpu_runnable(struct kvm_vcpu * vcpu)13159 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
13160 {
13161 return kvm_vcpu_running(vcpu) || kvm_vcpu_has_events(vcpu);
13162 }
13163
kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu * vcpu)13164 bool kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)
13165 {
13166 return kvm_vcpu_apicv_active(vcpu) &&
13167 static_call(kvm_x86_dy_apicv_has_pending_interrupt)(vcpu);
13168 }
13169
kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu * vcpu)13170 bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu)
13171 {
13172 return vcpu->arch.preempted_in_kernel;
13173 }
13174
kvm_arch_dy_runnable(struct kvm_vcpu * vcpu)13175 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
13176 {
13177 if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
13178 return true;
13179
13180 if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
13181 #ifdef CONFIG_KVM_SMM
13182 kvm_test_request(KVM_REQ_SMI, vcpu) ||
13183 #endif
13184 kvm_test_request(KVM_REQ_EVENT, vcpu))
13185 return true;
13186
13187 return kvm_arch_dy_has_pending_interrupt(vcpu);
13188 }
13189
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)13190 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
13191 {
13192 if (vcpu->arch.guest_state_protected)
13193 return true;
13194
13195 return static_call(kvm_x86_get_cpl)(vcpu) == 0;
13196 }
13197
kvm_arch_vcpu_get_ip(struct kvm_vcpu * vcpu)13198 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
13199 {
13200 return kvm_rip_read(vcpu);
13201 }
13202
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)13203 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
13204 {
13205 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
13206 }
13207
kvm_arch_interrupt_allowed(struct kvm_vcpu * vcpu)13208 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
13209 {
13210 return static_call(kvm_x86_interrupt_allowed)(vcpu, false);
13211 }
13212
kvm_get_linear_rip(struct kvm_vcpu * vcpu)13213 unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu)
13214 {
13215 /* Can't read the RIP when guest state is protected, just return 0 */
13216 if (vcpu->arch.guest_state_protected)
13217 return 0;
13218
13219 if (is_64_bit_mode(vcpu))
13220 return kvm_rip_read(vcpu);
13221 return (u32)(get_segment_base(vcpu, VCPU_SREG_CS) +
13222 kvm_rip_read(vcpu));
13223 }
13224 EXPORT_SYMBOL_GPL(kvm_get_linear_rip);
13225
kvm_is_linear_rip(struct kvm_vcpu * vcpu,unsigned long linear_rip)13226 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
13227 {
13228 return kvm_get_linear_rip(vcpu) == linear_rip;
13229 }
13230 EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
13231
kvm_get_rflags(struct kvm_vcpu * vcpu)13232 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
13233 {
13234 unsigned long rflags;
13235
13236 rflags = static_call(kvm_x86_get_rflags)(vcpu);
13237 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
13238 rflags &= ~X86_EFLAGS_TF;
13239 return rflags;
13240 }
13241 EXPORT_SYMBOL_GPL(kvm_get_rflags);
13242
__kvm_set_rflags(struct kvm_vcpu * vcpu,unsigned long rflags)13243 static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
13244 {
13245 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
13246 kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
13247 rflags |= X86_EFLAGS_TF;
13248 static_call(kvm_x86_set_rflags)(vcpu, rflags);
13249 }
13250
kvm_set_rflags(struct kvm_vcpu * vcpu,unsigned long rflags)13251 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
13252 {
13253 __kvm_set_rflags(vcpu, rflags);
13254 kvm_make_request(KVM_REQ_EVENT, vcpu);
13255 }
13256 EXPORT_SYMBOL_GPL(kvm_set_rflags);
13257
kvm_async_pf_hash_fn(gfn_t gfn)13258 static inline u32 kvm_async_pf_hash_fn(gfn_t gfn)
13259 {
13260 BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU));
13261
13262 return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU));
13263 }
13264
kvm_async_pf_next_probe(u32 key)13265 static inline u32 kvm_async_pf_next_probe(u32 key)
13266 {
13267 return (key + 1) & (ASYNC_PF_PER_VCPU - 1);
13268 }
13269
kvm_add_async_pf_gfn(struct kvm_vcpu * vcpu,gfn_t gfn)13270 static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13271 {
13272 u32 key = kvm_async_pf_hash_fn(gfn);
13273
13274 while (vcpu->arch.apf.gfns[key] != ~0)
13275 key = kvm_async_pf_next_probe(key);
13276
13277 vcpu->arch.apf.gfns[key] = gfn;
13278 }
13279
kvm_async_pf_gfn_slot(struct kvm_vcpu * vcpu,gfn_t gfn)13280 static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn)
13281 {
13282 int i;
13283 u32 key = kvm_async_pf_hash_fn(gfn);
13284
13285 for (i = 0; i < ASYNC_PF_PER_VCPU &&
13286 (vcpu->arch.apf.gfns[key] != gfn &&
13287 vcpu->arch.apf.gfns[key] != ~0); i++)
13288 key = kvm_async_pf_next_probe(key);
13289
13290 return key;
13291 }
13292
kvm_find_async_pf_gfn(struct kvm_vcpu * vcpu,gfn_t gfn)13293 bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13294 {
13295 return vcpu->arch.apf.gfns[kvm_async_pf_gfn_slot(vcpu, gfn)] == gfn;
13296 }
13297
kvm_del_async_pf_gfn(struct kvm_vcpu * vcpu,gfn_t gfn)13298 static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
13299 {
13300 u32 i, j, k;
13301
13302 i = j = kvm_async_pf_gfn_slot(vcpu, gfn);
13303
13304 if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn))
13305 return;
13306
13307 while (true) {
13308 vcpu->arch.apf.gfns[i] = ~0;
13309 do {
13310 j = kvm_async_pf_next_probe(j);
13311 if (vcpu->arch.apf.gfns[j] == ~0)
13312 return;
13313 k = kvm_async_pf_hash_fn(vcpu->arch.apf.gfns[j]);
13314 /*
13315 * k lies cyclically in ]i,j]
13316 * | i.k.j |
13317 * |....j i.k.| or |.k..j i...|
13318 */
13319 } while ((i <= j) ? (i < k && k <= j) : (i < k || k <= j));
13320 vcpu->arch.apf.gfns[i] = vcpu->arch.apf.gfns[j];
13321 i = j;
13322 }
13323 }
13324
apf_put_user_notpresent(struct kvm_vcpu * vcpu)13325 static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu)
13326 {
13327 u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT;
13328
13329 return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason,
13330 sizeof(reason));
13331 }
13332
apf_put_user_ready(struct kvm_vcpu * vcpu,u32 token)13333 static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token)
13334 {
13335 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
13336
13337 return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
13338 &token, offset, sizeof(token));
13339 }
13340
apf_pageready_slot_free(struct kvm_vcpu * vcpu)13341 static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
13342 {
13343 unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
13344 u32 val;
13345
13346 if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
13347 &val, offset, sizeof(val)))
13348 return false;
13349
13350 return !val;
13351 }
13352
kvm_can_deliver_async_pf(struct kvm_vcpu * vcpu)13353 static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
13354 {
13355
13356 if (!kvm_pv_async_pf_enabled(vcpu))
13357 return false;
13358
13359 if (vcpu->arch.apf.send_user_only &&
13360 static_call(kvm_x86_get_cpl)(vcpu) == 0)
13361 return false;
13362
13363 if (is_guest_mode(vcpu)) {
13364 /*
13365 * L1 needs to opt into the special #PF vmexits that are
13366 * used to deliver async page faults.
13367 */
13368 return vcpu->arch.apf.delivery_as_pf_vmexit;
13369 } else {
13370 /*
13371 * Play it safe in case the guest temporarily disables paging.
13372 * The real mode IDT in particular is unlikely to have a #PF
13373 * exception setup.
13374 */
13375 return is_paging(vcpu);
13376 }
13377 }
13378
kvm_can_do_async_pf(struct kvm_vcpu * vcpu)13379 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu)
13380 {
13381 if (unlikely(!lapic_in_kernel(vcpu) ||
13382 kvm_event_needs_reinjection(vcpu) ||
13383 kvm_is_exception_pending(vcpu)))
13384 return false;
13385
13386 if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu))
13387 return false;
13388
13389 /*
13390 * If interrupts are off we cannot even use an artificial
13391 * halt state.
13392 */
13393 return kvm_arch_interrupt_allowed(vcpu);
13394 }
13395
kvm_arch_async_page_not_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)13396 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
13397 struct kvm_async_pf *work)
13398 {
13399 struct x86_exception fault;
13400
13401 trace_kvm_async_pf_not_present(work->arch.token, work->cr2_or_gpa);
13402 kvm_add_async_pf_gfn(vcpu, work->arch.gfn);
13403
13404 if (kvm_can_deliver_async_pf(vcpu) &&
13405 !apf_put_user_notpresent(vcpu)) {
13406 fault.vector = PF_VECTOR;
13407 fault.error_code_valid = true;
13408 fault.error_code = 0;
13409 fault.nested_page_fault = false;
13410 fault.address = work->arch.token;
13411 fault.async_page_fault = true;
13412 kvm_inject_page_fault(vcpu, &fault);
13413 return true;
13414 } else {
13415 /*
13416 * It is not possible to deliver a paravirtualized asynchronous
13417 * page fault, but putting the guest in an artificial halt state
13418 * can be beneficial nevertheless: if an interrupt arrives, we
13419 * can deliver it timely and perhaps the guest will schedule
13420 * another process. When the instruction that triggered a page
13421 * fault is retried, hopefully the page will be ready in the host.
13422 */
13423 kvm_make_request(KVM_REQ_APF_HALT, vcpu);
13424 return false;
13425 }
13426 }
13427
kvm_arch_async_page_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)13428 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
13429 struct kvm_async_pf *work)
13430 {
13431 struct kvm_lapic_irq irq = {
13432 .delivery_mode = APIC_DM_FIXED,
13433 .vector = vcpu->arch.apf.vec
13434 };
13435
13436 if (work->wakeup_all)
13437 work->arch.token = ~0; /* broadcast wakeup */
13438 else
13439 kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
13440 trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);
13441
13442 if ((work->wakeup_all || work->notpresent_injected) &&
13443 kvm_pv_async_pf_enabled(vcpu) &&
13444 !apf_put_user_ready(vcpu, work->arch.token)) {
13445 vcpu->arch.apf.pageready_pending = true;
13446 kvm_apic_set_irq(vcpu, &irq, NULL);
13447 }
13448
13449 vcpu->arch.apf.halted = false;
13450 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
13451 }
13452
kvm_arch_async_page_present_queued(struct kvm_vcpu * vcpu)13453 void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu)
13454 {
13455 kvm_make_request(KVM_REQ_APF_READY, vcpu);
13456 if (!vcpu->arch.apf.pageready_pending)
13457 kvm_vcpu_kick(vcpu);
13458 }
13459
kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu * vcpu)13460 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
13461 {
13462 if (!kvm_pv_async_pf_enabled(vcpu))
13463 return true;
13464 else
13465 return kvm_lapic_enabled(vcpu) && apf_pageready_slot_free(vcpu);
13466 }
13467
kvm_arch_start_assignment(struct kvm * kvm)13468 void kvm_arch_start_assignment(struct kvm *kvm)
13469 {
13470 if (atomic_inc_return(&kvm->arch.assigned_device_count) == 1)
13471 static_call_cond(kvm_x86_pi_start_assignment)(kvm);
13472 }
13473 EXPORT_SYMBOL_GPL(kvm_arch_start_assignment);
13474
kvm_arch_end_assignment(struct kvm * kvm)13475 void kvm_arch_end_assignment(struct kvm *kvm)
13476 {
13477 atomic_dec(&kvm->arch.assigned_device_count);
13478 }
13479 EXPORT_SYMBOL_GPL(kvm_arch_end_assignment);
13480
kvm_arch_has_assigned_device(struct kvm * kvm)13481 bool noinstr kvm_arch_has_assigned_device(struct kvm *kvm)
13482 {
13483 return raw_atomic_read(&kvm->arch.assigned_device_count);
13484 }
13485 EXPORT_SYMBOL_GPL(kvm_arch_has_assigned_device);
13486
kvm_noncoherent_dma_assignment_start_or_stop(struct kvm * kvm)13487 static void kvm_noncoherent_dma_assignment_start_or_stop(struct kvm *kvm)
13488 {
13489 /*
13490 * Non-coherent DMA assignment and de-assignment will affect
13491 * whether KVM honors guest MTRRs and cause changes in memtypes
13492 * in TDP.
13493 * So, pass %true unconditionally to indicate non-coherent DMA was,
13494 * or will be involved, and that zapping SPTEs might be necessary.
13495 */
13496 if (__kvm_mmu_honors_guest_mtrrs(true))
13497 kvm_zap_gfn_range(kvm, gpa_to_gfn(0), gpa_to_gfn(~0ULL));
13498 }
13499
kvm_arch_register_noncoherent_dma(struct kvm * kvm)13500 void kvm_arch_register_noncoherent_dma(struct kvm *kvm)
13501 {
13502 if (atomic_inc_return(&kvm->arch.noncoherent_dma_count) == 1)
13503 kvm_noncoherent_dma_assignment_start_or_stop(kvm);
13504 }
13505 EXPORT_SYMBOL_GPL(kvm_arch_register_noncoherent_dma);
13506
kvm_arch_unregister_noncoherent_dma(struct kvm * kvm)13507 void kvm_arch_unregister_noncoherent_dma(struct kvm *kvm)
13508 {
13509 if (!atomic_dec_return(&kvm->arch.noncoherent_dma_count))
13510 kvm_noncoherent_dma_assignment_start_or_stop(kvm);
13511 }
13512 EXPORT_SYMBOL_GPL(kvm_arch_unregister_noncoherent_dma);
13513
kvm_arch_has_noncoherent_dma(struct kvm * kvm)13514 bool kvm_arch_has_noncoherent_dma(struct kvm *kvm)
13515 {
13516 return atomic_read(&kvm->arch.noncoherent_dma_count);
13517 }
13518 EXPORT_SYMBOL_GPL(kvm_arch_has_noncoherent_dma);
13519
kvm_arch_has_irq_bypass(void)13520 bool kvm_arch_has_irq_bypass(void)
13521 {
13522 return enable_apicv && irq_remapping_cap(IRQ_POSTING_CAP);
13523 }
13524
kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)13525 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
13526 struct irq_bypass_producer *prod)
13527 {
13528 struct kvm_kernel_irqfd *irqfd =
13529 container_of(cons, struct kvm_kernel_irqfd, consumer);
13530 int ret;
13531
13532 irqfd->producer = prod;
13533 kvm_arch_start_assignment(irqfd->kvm);
13534 ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm,
13535 prod->irq, irqfd->gsi, 1);
13536
13537 if (ret)
13538 kvm_arch_end_assignment(irqfd->kvm);
13539
13540 return ret;
13541 }
13542
kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer * cons,struct irq_bypass_producer * prod)13543 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
13544 struct irq_bypass_producer *prod)
13545 {
13546 int ret;
13547 struct kvm_kernel_irqfd *irqfd =
13548 container_of(cons, struct kvm_kernel_irqfd, consumer);
13549
13550 WARN_ON(irqfd->producer != prod);
13551 irqfd->producer = NULL;
13552
13553 /*
13554 * When producer of consumer is unregistered, we change back to
13555 * remapped mode, so we can re-use the current implementation
13556 * when the irq is masked/disabled or the consumer side (KVM
13557 * int this case doesn't want to receive the interrupts.
13558 */
13559 ret = static_call(kvm_x86_pi_update_irte)(irqfd->kvm, prod->irq, irqfd->gsi, 0);
13560 if (ret)
13561 printk(KERN_INFO "irq bypass consumer (token %p) unregistration"
13562 " fails: %d\n", irqfd->consumer.token, ret);
13563
13564 kvm_arch_end_assignment(irqfd->kvm);
13565 }
13566
kvm_arch_update_irqfd_routing(struct kvm * kvm,unsigned int host_irq,uint32_t guest_irq,bool set)13567 int kvm_arch_update_irqfd_routing(struct kvm *kvm, unsigned int host_irq,
13568 uint32_t guest_irq, bool set)
13569 {
13570 return static_call(kvm_x86_pi_update_irte)(kvm, host_irq, guest_irq, set);
13571 }
13572
kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry * old,struct kvm_kernel_irq_routing_entry * new)13573 bool kvm_arch_irqfd_route_changed(struct kvm_kernel_irq_routing_entry *old,
13574 struct kvm_kernel_irq_routing_entry *new)
13575 {
13576 if (new->type != KVM_IRQ_ROUTING_MSI)
13577 return true;
13578
13579 return !!memcmp(&old->msi, &new->msi, sizeof(new->msi));
13580 }
13581
kvm_vector_hashing_enabled(void)13582 bool kvm_vector_hashing_enabled(void)
13583 {
13584 return vector_hashing;
13585 }
13586
kvm_arch_no_poll(struct kvm_vcpu * vcpu)13587 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
13588 {
13589 return (vcpu->arch.msr_kvm_poll_control & 1) == 0;
13590 }
13591 EXPORT_SYMBOL_GPL(kvm_arch_no_poll);
13592
13593
kvm_spec_ctrl_test_value(u64 value)13594 int kvm_spec_ctrl_test_value(u64 value)
13595 {
13596 /*
13597 * test that setting IA32_SPEC_CTRL to given value
13598 * is allowed by the host processor
13599 */
13600
13601 u64 saved_value;
13602 unsigned long flags;
13603 int ret = 0;
13604
13605 local_irq_save(flags);
13606
13607 if (rdmsrl_safe(MSR_IA32_SPEC_CTRL, &saved_value))
13608 ret = 1;
13609 else if (wrmsrl_safe(MSR_IA32_SPEC_CTRL, value))
13610 ret = 1;
13611 else
13612 wrmsrl(MSR_IA32_SPEC_CTRL, saved_value);
13613
13614 local_irq_restore(flags);
13615
13616 return ret;
13617 }
13618 EXPORT_SYMBOL_GPL(kvm_spec_ctrl_test_value);
13619
kvm_fixup_and_inject_pf_error(struct kvm_vcpu * vcpu,gva_t gva,u16 error_code)13620 void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code)
13621 {
13622 struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
13623 struct x86_exception fault;
13624 u64 access = error_code &
13625 (PFERR_WRITE_MASK | PFERR_FETCH_MASK | PFERR_USER_MASK);
13626
13627 if (!(error_code & PFERR_PRESENT_MASK) ||
13628 mmu->gva_to_gpa(vcpu, mmu, gva, access, &fault) != INVALID_GPA) {
13629 /*
13630 * If vcpu->arch.walk_mmu->gva_to_gpa succeeded, the page
13631 * tables probably do not match the TLB. Just proceed
13632 * with the error code that the processor gave.
13633 */
13634 fault.vector = PF_VECTOR;
13635 fault.error_code_valid = true;
13636 fault.error_code = error_code;
13637 fault.nested_page_fault = false;
13638 fault.address = gva;
13639 fault.async_page_fault = false;
13640 }
13641 vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
13642 }
13643 EXPORT_SYMBOL_GPL(kvm_fixup_and_inject_pf_error);
13644
13645 /*
13646 * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns
13647 * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value
13648 * indicates whether exit to userspace is needed.
13649 */
kvm_handle_memory_failure(struct kvm_vcpu * vcpu,int r,struct x86_exception * e)13650 int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r,
13651 struct x86_exception *e)
13652 {
13653 if (r == X86EMUL_PROPAGATE_FAULT) {
13654 if (KVM_BUG_ON(!e, vcpu->kvm))
13655 return -EIO;
13656
13657 kvm_inject_emulated_page_fault(vcpu, e);
13658 return 1;
13659 }
13660
13661 /*
13662 * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED
13663 * while handling a VMX instruction KVM could've handled the request
13664 * correctly by exiting to userspace and performing I/O but there
13665 * doesn't seem to be a real use-case behind such requests, just return
13666 * KVM_EXIT_INTERNAL_ERROR for now.
13667 */
13668 kvm_prepare_emulation_failure_exit(vcpu);
13669
13670 return 0;
13671 }
13672 EXPORT_SYMBOL_GPL(kvm_handle_memory_failure);
13673
kvm_handle_invpcid(struct kvm_vcpu * vcpu,unsigned long type,gva_t gva)13674 int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
13675 {
13676 bool pcid_enabled;
13677 struct x86_exception e;
13678 struct {
13679 u64 pcid;
13680 u64 gla;
13681 } operand;
13682 int r;
13683
13684 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e);
13685 if (r != X86EMUL_CONTINUE)
13686 return kvm_handle_memory_failure(vcpu, r, &e);
13687
13688 if (operand.pcid >> 12 != 0) {
13689 kvm_inject_gp(vcpu, 0);
13690 return 1;
13691 }
13692
13693 pcid_enabled = kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE);
13694
13695 switch (type) {
13696 case INVPCID_TYPE_INDIV_ADDR:
13697 /*
13698 * LAM doesn't apply to addresses that are inputs to TLB
13699 * invalidation.
13700 */
13701 if ((!pcid_enabled && (operand.pcid != 0)) ||
13702 is_noncanonical_address(operand.gla, vcpu)) {
13703 kvm_inject_gp(vcpu, 0);
13704 return 1;
13705 }
13706 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
13707 return kvm_skip_emulated_instruction(vcpu);
13708
13709 case INVPCID_TYPE_SINGLE_CTXT:
13710 if (!pcid_enabled && (operand.pcid != 0)) {
13711 kvm_inject_gp(vcpu, 0);
13712 return 1;
13713 }
13714
13715 kvm_invalidate_pcid(vcpu, operand.pcid);
13716 return kvm_skip_emulated_instruction(vcpu);
13717
13718 case INVPCID_TYPE_ALL_NON_GLOBAL:
13719 /*
13720 * Currently, KVM doesn't mark global entries in the shadow
13721 * page tables, so a non-global flush just degenerates to a
13722 * global flush. If needed, we could optimize this later by
13723 * keeping track of global entries in shadow page tables.
13724 */
13725
13726 fallthrough;
13727 case INVPCID_TYPE_ALL_INCL_GLOBAL:
13728 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
13729 return kvm_skip_emulated_instruction(vcpu);
13730
13731 default:
13732 kvm_inject_gp(vcpu, 0);
13733 return 1;
13734 }
13735 }
13736 EXPORT_SYMBOL_GPL(kvm_handle_invpcid);
13737
complete_sev_es_emulated_mmio(struct kvm_vcpu * vcpu)13738 static int complete_sev_es_emulated_mmio(struct kvm_vcpu *vcpu)
13739 {
13740 struct kvm_run *run = vcpu->run;
13741 struct kvm_mmio_fragment *frag;
13742 unsigned int len;
13743
13744 BUG_ON(!vcpu->mmio_needed);
13745
13746 /* Complete previous fragment */
13747 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment];
13748 len = min(8u, frag->len);
13749 if (!vcpu->mmio_is_write)
13750 memcpy(frag->data, run->mmio.data, len);
13751
13752 if (frag->len <= 8) {
13753 /* Switch to the next fragment. */
13754 frag++;
13755 vcpu->mmio_cur_fragment++;
13756 } else {
13757 /* Go forward to the next mmio piece. */
13758 frag->data += len;
13759 frag->gpa += len;
13760 frag->len -= len;
13761 }
13762
13763 if (vcpu->mmio_cur_fragment >= vcpu->mmio_nr_fragments) {
13764 vcpu->mmio_needed = 0;
13765
13766 // VMG change, at this point, we're always done
13767 // RIP has already been advanced
13768 return 1;
13769 }
13770
13771 // More MMIO is needed
13772 run->mmio.phys_addr = frag->gpa;
13773 run->mmio.len = min(8u, frag->len);
13774 run->mmio.is_write = vcpu->mmio_is_write;
13775 if (run->mmio.is_write)
13776 memcpy(run->mmio.data, frag->data, min(8u, frag->len));
13777 run->exit_reason = KVM_EXIT_MMIO;
13778
13779 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13780
13781 return 0;
13782 }
13783
kvm_sev_es_mmio_write(struct kvm_vcpu * vcpu,gpa_t gpa,unsigned int bytes,void * data)13784 int kvm_sev_es_mmio_write(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
13785 void *data)
13786 {
13787 int handled;
13788 struct kvm_mmio_fragment *frag;
13789
13790 if (!data)
13791 return -EINVAL;
13792
13793 handled = write_emultor.read_write_mmio(vcpu, gpa, bytes, data);
13794 if (handled == bytes)
13795 return 1;
13796
13797 bytes -= handled;
13798 gpa += handled;
13799 data += handled;
13800
13801 /*TODO: Check if need to increment number of frags */
13802 frag = vcpu->mmio_fragments;
13803 vcpu->mmio_nr_fragments = 1;
13804 frag->len = bytes;
13805 frag->gpa = gpa;
13806 frag->data = data;
13807
13808 vcpu->mmio_needed = 1;
13809 vcpu->mmio_cur_fragment = 0;
13810
13811 vcpu->run->mmio.phys_addr = gpa;
13812 vcpu->run->mmio.len = min(8u, frag->len);
13813 vcpu->run->mmio.is_write = 1;
13814 memcpy(vcpu->run->mmio.data, frag->data, min(8u, frag->len));
13815 vcpu->run->exit_reason = KVM_EXIT_MMIO;
13816
13817 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13818
13819 return 0;
13820 }
13821 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_write);
13822
kvm_sev_es_mmio_read(struct kvm_vcpu * vcpu,gpa_t gpa,unsigned int bytes,void * data)13823 int kvm_sev_es_mmio_read(struct kvm_vcpu *vcpu, gpa_t gpa, unsigned int bytes,
13824 void *data)
13825 {
13826 int handled;
13827 struct kvm_mmio_fragment *frag;
13828
13829 if (!data)
13830 return -EINVAL;
13831
13832 handled = read_emultor.read_write_mmio(vcpu, gpa, bytes, data);
13833 if (handled == bytes)
13834 return 1;
13835
13836 bytes -= handled;
13837 gpa += handled;
13838 data += handled;
13839
13840 /*TODO: Check if need to increment number of frags */
13841 frag = vcpu->mmio_fragments;
13842 vcpu->mmio_nr_fragments = 1;
13843 frag->len = bytes;
13844 frag->gpa = gpa;
13845 frag->data = data;
13846
13847 vcpu->mmio_needed = 1;
13848 vcpu->mmio_cur_fragment = 0;
13849
13850 vcpu->run->mmio.phys_addr = gpa;
13851 vcpu->run->mmio.len = min(8u, frag->len);
13852 vcpu->run->mmio.is_write = 0;
13853 vcpu->run->exit_reason = KVM_EXIT_MMIO;
13854
13855 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_mmio;
13856
13857 return 0;
13858 }
13859 EXPORT_SYMBOL_GPL(kvm_sev_es_mmio_read);
13860
advance_sev_es_emulated_pio(struct kvm_vcpu * vcpu,unsigned count,int size)13861 static void advance_sev_es_emulated_pio(struct kvm_vcpu *vcpu, unsigned count, int size)
13862 {
13863 vcpu->arch.sev_pio_count -= count;
13864 vcpu->arch.sev_pio_data += count * size;
13865 }
13866
13867 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13868 unsigned int port);
13869
complete_sev_es_emulated_outs(struct kvm_vcpu * vcpu)13870 static int complete_sev_es_emulated_outs(struct kvm_vcpu *vcpu)
13871 {
13872 int size = vcpu->arch.pio.size;
13873 int port = vcpu->arch.pio.port;
13874
13875 vcpu->arch.pio.count = 0;
13876 if (vcpu->arch.sev_pio_count)
13877 return kvm_sev_es_outs(vcpu, size, port);
13878 return 1;
13879 }
13880
kvm_sev_es_outs(struct kvm_vcpu * vcpu,unsigned int size,unsigned int port)13881 static int kvm_sev_es_outs(struct kvm_vcpu *vcpu, unsigned int size,
13882 unsigned int port)
13883 {
13884 for (;;) {
13885 unsigned int count =
13886 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
13887 int ret = emulator_pio_out(vcpu, size, port, vcpu->arch.sev_pio_data, count);
13888
13889 /* memcpy done already by emulator_pio_out. */
13890 advance_sev_es_emulated_pio(vcpu, count, size);
13891 if (!ret)
13892 break;
13893
13894 /* Emulation done by the kernel. */
13895 if (!vcpu->arch.sev_pio_count)
13896 return 1;
13897 }
13898
13899 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_outs;
13900 return 0;
13901 }
13902
13903 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13904 unsigned int port);
13905
complete_sev_es_emulated_ins(struct kvm_vcpu * vcpu)13906 static int complete_sev_es_emulated_ins(struct kvm_vcpu *vcpu)
13907 {
13908 unsigned count = vcpu->arch.pio.count;
13909 int size = vcpu->arch.pio.size;
13910 int port = vcpu->arch.pio.port;
13911
13912 complete_emulator_pio_in(vcpu, vcpu->arch.sev_pio_data);
13913 advance_sev_es_emulated_pio(vcpu, count, size);
13914 if (vcpu->arch.sev_pio_count)
13915 return kvm_sev_es_ins(vcpu, size, port);
13916 return 1;
13917 }
13918
kvm_sev_es_ins(struct kvm_vcpu * vcpu,unsigned int size,unsigned int port)13919 static int kvm_sev_es_ins(struct kvm_vcpu *vcpu, unsigned int size,
13920 unsigned int port)
13921 {
13922 for (;;) {
13923 unsigned int count =
13924 min_t(unsigned int, PAGE_SIZE / size, vcpu->arch.sev_pio_count);
13925 if (!emulator_pio_in(vcpu, size, port, vcpu->arch.sev_pio_data, count))
13926 break;
13927
13928 /* Emulation done by the kernel. */
13929 advance_sev_es_emulated_pio(vcpu, count, size);
13930 if (!vcpu->arch.sev_pio_count)
13931 return 1;
13932 }
13933
13934 vcpu->arch.complete_userspace_io = complete_sev_es_emulated_ins;
13935 return 0;
13936 }
13937
kvm_sev_es_string_io(struct kvm_vcpu * vcpu,unsigned int size,unsigned int port,void * data,unsigned int count,int in)13938 int kvm_sev_es_string_io(struct kvm_vcpu *vcpu, unsigned int size,
13939 unsigned int port, void *data, unsigned int count,
13940 int in)
13941 {
13942 vcpu->arch.sev_pio_data = data;
13943 vcpu->arch.sev_pio_count = count;
13944 return in ? kvm_sev_es_ins(vcpu, size, port)
13945 : kvm_sev_es_outs(vcpu, size, port);
13946 }
13947 EXPORT_SYMBOL_GPL(kvm_sev_es_string_io);
13948
13949 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_entry);
13950 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
13951 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_fast_mmio);
13952 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
13953 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
13954 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
13955 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
13956 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter);
13957 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
13958 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
13959 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
13960 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter_failed);
13961 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
13962 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
13963 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
13964 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
13965 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window_update);
13966 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
13967 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update);
13968 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access);
13969 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi);
13970 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log);
13971 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_kick_vcpu_slowpath);
13972 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_doorbell);
13973 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq);
13974 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter);
13975 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit);
13976 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_enter);
13977 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_msr_protocol_exit);
13978
kvm_x86_init(void)13979 static int __init kvm_x86_init(void)
13980 {
13981 kvm_mmu_x86_module_init();
13982 mitigate_smt_rsb &= boot_cpu_has_bug(X86_BUG_SMT_RSB) && cpu_smt_possible();
13983 return 0;
13984 }
13985 module_init(kvm_x86_init);
13986
kvm_x86_exit(void)13987 static void __exit kvm_x86_exit(void)
13988 {
13989 WARN_ON_ONCE(static_branch_unlikely(&kvm_has_noapic_vcpu));
13990 }
13991 module_exit(kvm_x86_exit);
13992