1 /*
2 * ARM implementation of KVM hooks
3 *
4 * Copyright Christoffer Dall 2009-2010
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
8 *
9 */
10
11 #include "qemu/osdep.h"
12 #include <sys/ioctl.h>
13
14 #include <linux/kvm.h>
15
16 #include "qemu-common.h"
17 #include "qemu/timer.h"
18 #include "qemu/error-report.h"
19 #include "qemu/main-loop.h"
20 #include "qom/object.h"
21 #include "qapi/error.h"
22 #include "sysemu/sysemu.h"
23 #include "sysemu/kvm.h"
24 #include "sysemu/kvm_int.h"
25 #include "kvm_arm.h"
26 #include "cpu.h"
27 #include "trace.h"
28 #include "internals.h"
29 #include "hw/pci/pci.h"
30 #include "exec/memattrs.h"
31 #include "exec/address-spaces.h"
32 #include "hw/boards.h"
33 #include "hw/irq.h"
34 #include "qemu/log.h"
35
36 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
37 KVM_CAP_LAST_INFO
38 };
39
40 static bool cap_has_mp_state;
41 static bool cap_has_inject_serror_esr;
42 static bool cap_has_inject_ext_dabt;
43
44 static ARMHostCPUFeatures arm_host_cpu_features;
45
kvm_arm_vcpu_init(CPUState * cs)46 int kvm_arm_vcpu_init(CPUState *cs)
47 {
48 ARMCPU *cpu = ARM_CPU(cs);
49 struct kvm_vcpu_init init;
50
51 init.target = cpu->kvm_target;
52 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features));
53
54 return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_INIT, &init);
55 }
56
kvm_arm_vcpu_finalize(CPUState * cs,int feature)57 int kvm_arm_vcpu_finalize(CPUState *cs, int feature)
58 {
59 return kvm_vcpu_ioctl(cs, KVM_ARM_VCPU_FINALIZE, &feature);
60 }
61
kvm_arm_init_serror_injection(CPUState * cs)62 void kvm_arm_init_serror_injection(CPUState *cs)
63 {
64 cap_has_inject_serror_esr = kvm_check_extension(cs->kvm_state,
65 KVM_CAP_ARM_INJECT_SERROR_ESR);
66 }
67
kvm_arm_create_scratch_host_vcpu(const uint32_t * cpus_to_try,int * fdarray,struct kvm_vcpu_init * init)68 bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
69 int *fdarray,
70 struct kvm_vcpu_init *init)
71 {
72 int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1;
73
74 kvmfd = qemu_open_old("/dev/kvm", O_RDWR);
75 if (kvmfd < 0) {
76 goto err;
77 }
78 vmfd = ioctl(kvmfd, KVM_CREATE_VM, 0);
79 if (vmfd < 0) {
80 goto err;
81 }
82 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
83 if (cpufd < 0) {
84 goto err;
85 }
86
87 if (!init) {
88 /* Caller doesn't want the VCPU to be initialized, so skip it */
89 goto finish;
90 }
91
92 if (init->target == -1) {
93 struct kvm_vcpu_init preferred;
94
95 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, &preferred);
96 if (!ret) {
97 init->target = preferred.target;
98 }
99 }
100 if (ret >= 0) {
101 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
102 if (ret < 0) {
103 goto err;
104 }
105 } else if (cpus_to_try) {
106 /* Old kernel which doesn't know about the
107 * PREFERRED_TARGET ioctl: we know it will only support
108 * creating one kind of guest CPU which is its preferred
109 * CPU type.
110 */
111 struct kvm_vcpu_init try;
112
113 while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) {
114 try.target = *cpus_to_try++;
115 memcpy(try.features, init->features, sizeof(init->features));
116 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, &try);
117 if (ret >= 0) {
118 break;
119 }
120 }
121 if (ret < 0) {
122 goto err;
123 }
124 init->target = try.target;
125 } else {
126 /* Treat a NULL cpus_to_try argument the same as an empty
127 * list, which means we will fail the call since this must
128 * be an old kernel which doesn't support PREFERRED_TARGET.
129 */
130 goto err;
131 }
132
133 finish:
134 fdarray[0] = kvmfd;
135 fdarray[1] = vmfd;
136 fdarray[2] = cpufd;
137
138 return true;
139
140 err:
141 if (cpufd >= 0) {
142 close(cpufd);
143 }
144 if (vmfd >= 0) {
145 close(vmfd);
146 }
147 if (kvmfd >= 0) {
148 close(kvmfd);
149 }
150
151 return false;
152 }
153
kvm_arm_destroy_scratch_host_vcpu(int * fdarray)154 void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
155 {
156 int i;
157
158 for (i = 2; i >= 0; i--) {
159 close(fdarray[i]);
160 }
161 }
162
kvm_arm_set_cpu_features_from_host(ARMCPU * cpu)163 void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu)
164 {
165 CPUARMState *env = &cpu->env;
166
167 if (!arm_host_cpu_features.dtb_compatible) {
168 if (!kvm_enabled() ||
169 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) {
170 /* We can't report this error yet, so flag that we need to
171 * in arm_cpu_realizefn().
172 */
173 cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE;
174 cpu->host_cpu_probe_failed = true;
175 return;
176 }
177 }
178
179 cpu->kvm_target = arm_host_cpu_features.target;
180 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
181 cpu->isar = arm_host_cpu_features.isar;
182 env->features = arm_host_cpu_features.features;
183 }
184
kvm_no_adjvtime_get(Object * obj,Error ** errp)185 static bool kvm_no_adjvtime_get(Object *obj, Error **errp)
186 {
187 return !ARM_CPU(obj)->kvm_adjvtime;
188 }
189
kvm_no_adjvtime_set(Object * obj,bool value,Error ** errp)190 static void kvm_no_adjvtime_set(Object *obj, bool value, Error **errp)
191 {
192 ARM_CPU(obj)->kvm_adjvtime = !value;
193 }
194
kvm_steal_time_get(Object * obj,Error ** errp)195 static bool kvm_steal_time_get(Object *obj, Error **errp)
196 {
197 return ARM_CPU(obj)->kvm_steal_time != ON_OFF_AUTO_OFF;
198 }
199
kvm_steal_time_set(Object * obj,bool value,Error ** errp)200 static void kvm_steal_time_set(Object *obj, bool value, Error **errp)
201 {
202 ARM_CPU(obj)->kvm_steal_time = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
203 }
204
205 /* KVM VCPU properties should be prefixed with "kvm-". */
kvm_arm_add_vcpu_properties(Object * obj)206 void kvm_arm_add_vcpu_properties(Object *obj)
207 {
208 ARMCPU *cpu = ARM_CPU(obj);
209 CPUARMState *env = &cpu->env;
210
211 if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
212 cpu->kvm_adjvtime = true;
213 object_property_add_bool(obj, "kvm-no-adjvtime", kvm_no_adjvtime_get,
214 kvm_no_adjvtime_set);
215 object_property_set_description(obj, "kvm-no-adjvtime",
216 "Set on to disable the adjustment of "
217 "the virtual counter. VM stopped time "
218 "will be counted.");
219 }
220
221 cpu->kvm_steal_time = ON_OFF_AUTO_AUTO;
222 object_property_add_bool(obj, "kvm-steal-time", kvm_steal_time_get,
223 kvm_steal_time_set);
224 object_property_set_description(obj, "kvm-steal-time",
225 "Set off to disable KVM steal time.");
226 }
227
kvm_arm_pmu_supported(void)228 bool kvm_arm_pmu_supported(void)
229 {
230 return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3);
231 }
232
kvm_arm_get_max_vm_ipa_size(MachineState * ms)233 int kvm_arm_get_max_vm_ipa_size(MachineState *ms)
234 {
235 KVMState *s = KVM_STATE(ms->accelerator);
236 int ret;
237
238 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE);
239 return ret > 0 ? ret : 40;
240 }
241
kvm_arch_init(MachineState * ms,KVMState * s)242 int kvm_arch_init(MachineState *ms, KVMState *s)
243 {
244 int ret = 0;
245 /* For ARM interrupt delivery is always asynchronous,
246 * whether we are using an in-kernel VGIC or not.
247 */
248 kvm_async_interrupts_allowed = true;
249
250 /*
251 * PSCI wakes up secondary cores, so we always need to
252 * have vCPUs waiting in kernel space
253 */
254 kvm_halt_in_kernel_allowed = true;
255
256 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
257
258 if (ms->smp.cpus > 256 &&
259 !kvm_check_extension(s, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2)) {
260 error_report("Using more than 256 vcpus requires a host kernel "
261 "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2");
262 ret = -EINVAL;
263 }
264
265 if (kvm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) {
266 if (kvm_vm_enable_cap(s, KVM_CAP_ARM_NISV_TO_USER, 0)) {
267 error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap");
268 } else {
269 /* Set status for supporting the external dabt injection */
270 cap_has_inject_ext_dabt = kvm_check_extension(s,
271 KVM_CAP_ARM_INJECT_EXT_DABT);
272 }
273 }
274
275 return ret;
276 }
277
kvm_arch_vcpu_id(CPUState * cpu)278 unsigned long kvm_arch_vcpu_id(CPUState *cpu)
279 {
280 return cpu->cpu_index;
281 }
282
283 /* We track all the KVM devices which need their memory addresses
284 * passing to the kernel in a list of these structures.
285 * When board init is complete we run through the list and
286 * tell the kernel the base addresses of the memory regions.
287 * We use a MemoryListener to track mapping and unmapping of
288 * the regions during board creation, so the board models don't
289 * need to do anything special for the KVM case.
290 *
291 * Sometimes the address must be OR'ed with some other fields
292 * (for example for KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION).
293 * @kda_addr_ormask aims at storing the value of those fields.
294 */
295 typedef struct KVMDevice {
296 struct kvm_arm_device_addr kda;
297 struct kvm_device_attr kdattr;
298 uint64_t kda_addr_ormask;
299 MemoryRegion *mr;
300 QSLIST_ENTRY(KVMDevice) entries;
301 int dev_fd;
302 } KVMDevice;
303
304 static QSLIST_HEAD(, KVMDevice) kvm_devices_head;
305
kvm_arm_devlistener_add(MemoryListener * listener,MemoryRegionSection * section)306 static void kvm_arm_devlistener_add(MemoryListener *listener,
307 MemoryRegionSection *section)
308 {
309 KVMDevice *kd;
310
311 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
312 if (section->mr == kd->mr) {
313 kd->kda.addr = section->offset_within_address_space;
314 }
315 }
316 }
317
kvm_arm_devlistener_del(MemoryListener * listener,MemoryRegionSection * section)318 static void kvm_arm_devlistener_del(MemoryListener *listener,
319 MemoryRegionSection *section)
320 {
321 KVMDevice *kd;
322
323 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
324 if (section->mr == kd->mr) {
325 kd->kda.addr = -1;
326 }
327 }
328 }
329
330 static MemoryListener devlistener = {
331 .region_add = kvm_arm_devlistener_add,
332 .region_del = kvm_arm_devlistener_del,
333 };
334
kvm_arm_set_device_addr(KVMDevice * kd)335 static void kvm_arm_set_device_addr(KVMDevice *kd)
336 {
337 struct kvm_device_attr *attr = &kd->kdattr;
338 int ret;
339
340 /* If the device control API is available and we have a device fd on the
341 * KVMDevice struct, let's use the newer API
342 */
343 if (kd->dev_fd >= 0) {
344 uint64_t addr = kd->kda.addr;
345
346 addr |= kd->kda_addr_ormask;
347 attr->addr = (uintptr_t)&addr;
348 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr);
349 } else {
350 ret = kvm_vm_ioctl(kvm_state, KVM_ARM_SET_DEVICE_ADDR, &kd->kda);
351 }
352
353 if (ret < 0) {
354 fprintf(stderr, "Failed to set device address: %s\n",
355 strerror(-ret));
356 abort();
357 }
358 }
359
kvm_arm_machine_init_done(Notifier * notifier,void * data)360 static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
361 {
362 KVMDevice *kd, *tkd;
363
364 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
365 if (kd->kda.addr != -1) {
366 kvm_arm_set_device_addr(kd);
367 }
368 memory_region_unref(kd->mr);
369 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries);
370 g_free(kd);
371 }
372 memory_listener_unregister(&devlistener);
373 }
374
375 static Notifier notify = {
376 .notify = kvm_arm_machine_init_done,
377 };
378
kvm_arm_register_device(MemoryRegion * mr,uint64_t devid,uint64_t group,uint64_t attr,int dev_fd,uint64_t addr_ormask)379 void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group,
380 uint64_t attr, int dev_fd, uint64_t addr_ormask)
381 {
382 KVMDevice *kd;
383
384 if (!kvm_irqchip_in_kernel()) {
385 return;
386 }
387
388 if (QSLIST_EMPTY(&kvm_devices_head)) {
389 memory_listener_register(&devlistener, &address_space_memory);
390 qemu_add_machine_init_done_notifier(¬ify);
391 }
392 kd = g_new0(KVMDevice, 1);
393 kd->mr = mr;
394 kd->kda.id = devid;
395 kd->kda.addr = -1;
396 kd->kdattr.flags = 0;
397 kd->kdattr.group = group;
398 kd->kdattr.attr = attr;
399 kd->dev_fd = dev_fd;
400 kd->kda_addr_ormask = addr_ormask;
401 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
402 memory_region_ref(kd->mr);
403 }
404
compare_u64(const void * a,const void * b)405 static int compare_u64(const void *a, const void *b)
406 {
407 if (*(uint64_t *)a > *(uint64_t *)b) {
408 return 1;
409 }
410 if (*(uint64_t *)a < *(uint64_t *)b) {
411 return -1;
412 }
413 return 0;
414 }
415
416 /*
417 * cpreg_values are sorted in ascending order by KVM register ID
418 * (see kvm_arm_init_cpreg_list). This allows us to cheaply find
419 * the storage for a KVM register by ID with a binary search.
420 */
kvm_arm_get_cpreg_ptr(ARMCPU * cpu,uint64_t regidx)421 static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU *cpu, uint64_t regidx)
422 {
423 uint64_t *res;
424
425 res = bsearch(®idx, cpu->cpreg_indexes, cpu->cpreg_array_len,
426 sizeof(uint64_t), compare_u64);
427 assert(res);
428
429 return &cpu->cpreg_values[res - cpu->cpreg_indexes];
430 }
431
432 /* Initialize the ARMCPU cpreg list according to the kernel's
433 * definition of what CPU registers it knows about (and throw away
434 * the previous TCG-created cpreg list).
435 */
kvm_arm_init_cpreg_list(ARMCPU * cpu)436 int kvm_arm_init_cpreg_list(ARMCPU *cpu)
437 {
438 struct kvm_reg_list rl;
439 struct kvm_reg_list *rlp;
440 int i, ret, arraylen;
441 CPUState *cs = CPU(cpu);
442
443 rl.n = 0;
444 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
445 if (ret != -E2BIG) {
446 return ret;
447 }
448 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
449 rlp->n = rl.n;
450 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
451 if (ret) {
452 goto out;
453 }
454 /* Sort the list we get back from the kernel, since cpreg_tuples
455 * must be in strictly ascending order.
456 */
457 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
458
459 for (i = 0, arraylen = 0; i < rlp->n; i++) {
460 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
461 continue;
462 }
463 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
464 case KVM_REG_SIZE_U32:
465 case KVM_REG_SIZE_U64:
466 break;
467 default:
468 fprintf(stderr, "Can't handle size of register in kernel list\n");
469 ret = -EINVAL;
470 goto out;
471 }
472
473 arraylen++;
474 }
475
476 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
477 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
478 cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
479 arraylen);
480 cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
481 arraylen);
482 cpu->cpreg_array_len = arraylen;
483 cpu->cpreg_vmstate_array_len = arraylen;
484
485 for (i = 0, arraylen = 0; i < rlp->n; i++) {
486 uint64_t regidx = rlp->reg[i];
487 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
488 continue;
489 }
490 cpu->cpreg_indexes[arraylen] = regidx;
491 arraylen++;
492 }
493 assert(cpu->cpreg_array_len == arraylen);
494
495 if (!write_kvmstate_to_list(cpu)) {
496 /* Shouldn't happen unless kernel is inconsistent about
497 * what registers exist.
498 */
499 fprintf(stderr, "Initial read of kernel register state failed\n");
500 ret = -EINVAL;
501 goto out;
502 }
503
504 out:
505 g_free(rlp);
506 return ret;
507 }
508
write_kvmstate_to_list(ARMCPU * cpu)509 bool write_kvmstate_to_list(ARMCPU *cpu)
510 {
511 CPUState *cs = CPU(cpu);
512 int i;
513 bool ok = true;
514
515 for (i = 0; i < cpu->cpreg_array_len; i++) {
516 struct kvm_one_reg r;
517 uint64_t regidx = cpu->cpreg_indexes[i];
518 uint32_t v32;
519 int ret;
520
521 r.id = regidx;
522
523 switch (regidx & KVM_REG_SIZE_MASK) {
524 case KVM_REG_SIZE_U32:
525 r.addr = (uintptr_t)&v32;
526 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
527 if (!ret) {
528 cpu->cpreg_values[i] = v32;
529 }
530 break;
531 case KVM_REG_SIZE_U64:
532 r.addr = (uintptr_t)(cpu->cpreg_values + i);
533 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &r);
534 break;
535 default:
536 abort();
537 }
538 if (ret) {
539 ok = false;
540 }
541 }
542 return ok;
543 }
544
write_list_to_kvmstate(ARMCPU * cpu,int level)545 bool write_list_to_kvmstate(ARMCPU *cpu, int level)
546 {
547 CPUState *cs = CPU(cpu);
548 int i;
549 bool ok = true;
550
551 for (i = 0; i < cpu->cpreg_array_len; i++) {
552 struct kvm_one_reg r;
553 uint64_t regidx = cpu->cpreg_indexes[i];
554 uint32_t v32;
555 int ret;
556
557 if (kvm_arm_cpreg_level(regidx) > level) {
558 continue;
559 }
560
561 r.id = regidx;
562 switch (regidx & KVM_REG_SIZE_MASK) {
563 case KVM_REG_SIZE_U32:
564 v32 = cpu->cpreg_values[i];
565 r.addr = (uintptr_t)&v32;
566 break;
567 case KVM_REG_SIZE_U64:
568 r.addr = (uintptr_t)(cpu->cpreg_values + i);
569 break;
570 default:
571 abort();
572 }
573 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &r);
574 if (ret) {
575 /* We might fail for "unknown register" and also for
576 * "you tried to set a register which is constant with
577 * a different value from what it actually contains".
578 */
579 ok = false;
580 }
581 }
582 return ok;
583 }
584
kvm_arm_cpu_pre_save(ARMCPU * cpu)585 void kvm_arm_cpu_pre_save(ARMCPU *cpu)
586 {
587 /* KVM virtual time adjustment */
588 if (cpu->kvm_vtime_dirty) {
589 *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT) = cpu->kvm_vtime;
590 }
591 }
592
kvm_arm_cpu_post_load(ARMCPU * cpu)593 void kvm_arm_cpu_post_load(ARMCPU *cpu)
594 {
595 /* KVM virtual time adjustment */
596 if (cpu->kvm_adjvtime) {
597 cpu->kvm_vtime = *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT);
598 cpu->kvm_vtime_dirty = true;
599 }
600 }
601
kvm_arm_reset_vcpu(ARMCPU * cpu)602 void kvm_arm_reset_vcpu(ARMCPU *cpu)
603 {
604 int ret;
605
606 /* Re-init VCPU so that all registers are set to
607 * their respective reset values.
608 */
609 ret = kvm_arm_vcpu_init(CPU(cpu));
610 if (ret < 0) {
611 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
612 abort();
613 }
614 if (!write_kvmstate_to_list(cpu)) {
615 fprintf(stderr, "write_kvmstate_to_list failed\n");
616 abort();
617 }
618 /*
619 * Sync the reset values also into the CPUState. This is necessary
620 * because the next thing we do will be a kvm_arch_put_registers()
621 * which will update the list values from the CPUState before copying
622 * the list values back to KVM. It's OK to ignore failure returns here
623 * for the same reason we do so in kvm_arch_get_registers().
624 */
625 write_list_to_cpustate(cpu);
626 }
627
628 /*
629 * Update KVM's MP_STATE based on what QEMU thinks it is
630 */
kvm_arm_sync_mpstate_to_kvm(ARMCPU * cpu)631 int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
632 {
633 if (cap_has_mp_state) {
634 struct kvm_mp_state mp_state = {
635 .mp_state = (cpu->power_state == PSCI_OFF) ?
636 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
637 };
638 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
639 if (ret) {
640 fprintf(stderr, "%s: failed to set MP_STATE %d/%s\n",
641 __func__, ret, strerror(-ret));
642 return -1;
643 }
644 }
645
646 return 0;
647 }
648
649 /*
650 * Sync the KVM MP_STATE into QEMU
651 */
kvm_arm_sync_mpstate_to_qemu(ARMCPU * cpu)652 int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
653 {
654 if (cap_has_mp_state) {
655 struct kvm_mp_state mp_state;
656 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
657 if (ret) {
658 fprintf(stderr, "%s: failed to get MP_STATE %d/%s\n",
659 __func__, ret, strerror(-ret));
660 abort();
661 }
662 cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ?
663 PSCI_OFF : PSCI_ON;
664 }
665
666 return 0;
667 }
668
kvm_arm_get_virtual_time(CPUState * cs)669 void kvm_arm_get_virtual_time(CPUState *cs)
670 {
671 ARMCPU *cpu = ARM_CPU(cs);
672 struct kvm_one_reg reg = {
673 .id = KVM_REG_ARM_TIMER_CNT,
674 .addr = (uintptr_t)&cpu->kvm_vtime,
675 };
676 int ret;
677
678 if (cpu->kvm_vtime_dirty) {
679 return;
680 }
681
682 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, ®);
683 if (ret) {
684 error_report("Failed to get KVM_REG_ARM_TIMER_CNT");
685 abort();
686 }
687
688 cpu->kvm_vtime_dirty = true;
689 }
690
kvm_arm_put_virtual_time(CPUState * cs)691 void kvm_arm_put_virtual_time(CPUState *cs)
692 {
693 ARMCPU *cpu = ARM_CPU(cs);
694 struct kvm_one_reg reg = {
695 .id = KVM_REG_ARM_TIMER_CNT,
696 .addr = (uintptr_t)&cpu->kvm_vtime,
697 };
698 int ret;
699
700 if (!cpu->kvm_vtime_dirty) {
701 return;
702 }
703
704 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®);
705 if (ret) {
706 error_report("Failed to set KVM_REG_ARM_TIMER_CNT");
707 abort();
708 }
709
710 cpu->kvm_vtime_dirty = false;
711 }
712
kvm_put_vcpu_events(ARMCPU * cpu)713 int kvm_put_vcpu_events(ARMCPU *cpu)
714 {
715 CPUARMState *env = &cpu->env;
716 struct kvm_vcpu_events events;
717 int ret;
718
719 if (!kvm_has_vcpu_events()) {
720 return 0;
721 }
722
723 memset(&events, 0, sizeof(events));
724 events.exception.serror_pending = env->serror.pending;
725
726 /* Inject SError to guest with specified syndrome if host kernel
727 * supports it, otherwise inject SError without syndrome.
728 */
729 if (cap_has_inject_serror_esr) {
730 events.exception.serror_has_esr = env->serror.has_esr;
731 events.exception.serror_esr = env->serror.esr;
732 }
733
734 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
735 if (ret) {
736 error_report("failed to put vcpu events");
737 }
738
739 return ret;
740 }
741
kvm_get_vcpu_events(ARMCPU * cpu)742 int kvm_get_vcpu_events(ARMCPU *cpu)
743 {
744 CPUARMState *env = &cpu->env;
745 struct kvm_vcpu_events events;
746 int ret;
747
748 if (!kvm_has_vcpu_events()) {
749 return 0;
750 }
751
752 memset(&events, 0, sizeof(events));
753 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
754 if (ret) {
755 error_report("failed to get vcpu events");
756 return ret;
757 }
758
759 env->serror.pending = events.exception.serror_pending;
760 env->serror.has_esr = events.exception.serror_has_esr;
761 env->serror.esr = events.exception.serror_esr;
762
763 return 0;
764 }
765
kvm_arch_pre_run(CPUState * cs,struct kvm_run * run)766 void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
767 {
768 ARMCPU *cpu = ARM_CPU(cs);
769 CPUARMState *env = &cpu->env;
770
771 if (unlikely(env->ext_dabt_raised)) {
772 /*
773 * Verifying that the ext DABT has been properly injected,
774 * otherwise risking indefinitely re-running the faulting instruction
775 * Covering a very narrow case for kernels 5.5..5.5.4
776 * when injected abort was misconfigured to be
777 * an IMPLEMENTATION DEFINED exception (for 32-bit EL1)
778 */
779 if (!arm_feature(env, ARM_FEATURE_AARCH64) &&
780 unlikely(!kvm_arm_verify_ext_dabt_pending(cs))) {
781
782 error_report("Data abort exception with no valid ISS generated by "
783 "guest memory access. KVM unable to emulate faulting "
784 "instruction. Failed to inject an external data abort "
785 "into the guest.");
786 abort();
787 }
788 /* Clear the status */
789 env->ext_dabt_raised = 0;
790 }
791 }
792
kvm_arch_post_run(CPUState * cs,struct kvm_run * run)793 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
794 {
795 ARMCPU *cpu;
796 uint32_t switched_level;
797
798 if (kvm_irqchip_in_kernel()) {
799 /*
800 * We only need to sync timer states with user-space interrupt
801 * controllers, so return early and save cycles if we don't.
802 */
803 return MEMTXATTRS_UNSPECIFIED;
804 }
805
806 cpu = ARM_CPU(cs);
807
808 /* Synchronize our shadowed in-kernel device irq lines with the kvm ones */
809 if (run->s.regs.device_irq_level != cpu->device_irq_level) {
810 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
811
812 qemu_mutex_lock_iothread();
813
814 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
815 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
816 !!(run->s.regs.device_irq_level &
817 KVM_ARM_DEV_EL1_VTIMER));
818 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
819 }
820
821 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
822 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
823 !!(run->s.regs.device_irq_level &
824 KVM_ARM_DEV_EL1_PTIMER));
825 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
826 }
827
828 if (switched_level & KVM_ARM_DEV_PMU) {
829 qemu_set_irq(cpu->pmu_interrupt,
830 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU));
831 switched_level &= ~KVM_ARM_DEV_PMU;
832 }
833
834 if (switched_level) {
835 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
836 __func__, switched_level);
837 }
838
839 /* We also mark unknown levels as processed to not waste cycles */
840 cpu->device_irq_level = run->s.regs.device_irq_level;
841 qemu_mutex_unlock_iothread();
842 }
843
844 return MEMTXATTRS_UNSPECIFIED;
845 }
846
kvm_arm_vm_state_change(void * opaque,int running,RunState state)847 void kvm_arm_vm_state_change(void *opaque, int running, RunState state)
848 {
849 CPUState *cs = opaque;
850 ARMCPU *cpu = ARM_CPU(cs);
851
852 if (running) {
853 if (cpu->kvm_adjvtime) {
854 kvm_arm_put_virtual_time(cs);
855 }
856 } else {
857 if (cpu->kvm_adjvtime) {
858 kvm_arm_get_virtual_time(cs);
859 }
860 }
861 }
862
863 /**
864 * kvm_arm_handle_dabt_nisv:
865 * @cs: CPUState
866 * @esr_iss: ISS encoding (limited) for the exception from Data Abort
867 * ISV bit set to '0b0' -> no valid instruction syndrome
868 * @fault_ipa: faulting address for the synchronous data abort
869 *
870 * Returns: 0 if the exception has been handled, < 0 otherwise
871 */
kvm_arm_handle_dabt_nisv(CPUState * cs,uint64_t esr_iss,uint64_t fault_ipa)872 static int kvm_arm_handle_dabt_nisv(CPUState *cs, uint64_t esr_iss,
873 uint64_t fault_ipa)
874 {
875 ARMCPU *cpu = ARM_CPU(cs);
876 CPUARMState *env = &cpu->env;
877 /*
878 * Request KVM to inject the external data abort into the guest
879 */
880 if (cap_has_inject_ext_dabt) {
881 struct kvm_vcpu_events events = { };
882 /*
883 * The external data abort event will be handled immediately by KVM
884 * using the address fault that triggered the exit on given VCPU.
885 * Requesting injection of the external data abort does not rely
886 * on any other VCPU state. Therefore, in this particular case, the VCPU
887 * synchronization can be exceptionally skipped.
888 */
889 events.exception.ext_dabt_pending = 1;
890 /* KVM_CAP_ARM_INJECT_EXT_DABT implies KVM_CAP_VCPU_EVENTS */
891 if (!kvm_vcpu_ioctl(cs, KVM_SET_VCPU_EVENTS, &events)) {
892 env->ext_dabt_raised = 1;
893 return 0;
894 }
895 } else {
896 error_report("Data abort exception triggered by guest memory access "
897 "at physical address: 0x" TARGET_FMT_lx,
898 (target_ulong)fault_ipa);
899 error_printf("KVM unable to emulate faulting instruction.\n");
900 }
901 return -1;
902 }
903
kvm_arch_handle_exit(CPUState * cs,struct kvm_run * run)904 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
905 {
906 int ret = 0;
907
908 switch (run->exit_reason) {
909 case KVM_EXIT_DEBUG:
910 if (kvm_arm_handle_debug(cs, &run->debug.arch)) {
911 ret = EXCP_DEBUG;
912 } /* otherwise return to guest */
913 break;
914 case KVM_EXIT_ARM_NISV:
915 /* External DABT with no valid iss to decode */
916 ret = kvm_arm_handle_dabt_nisv(cs, run->arm_nisv.esr_iss,
917 run->arm_nisv.fault_ipa);
918 break;
919 default:
920 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
921 __func__, run->exit_reason);
922 break;
923 }
924 return ret;
925 }
926
kvm_arch_stop_on_emulation_error(CPUState * cs)927 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
928 {
929 return true;
930 }
931
kvm_arch_process_async_events(CPUState * cs)932 int kvm_arch_process_async_events(CPUState *cs)
933 {
934 return 0;
935 }
936
kvm_arch_update_guest_debug(CPUState * cs,struct kvm_guest_debug * dbg)937 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
938 {
939 if (kvm_sw_breakpoints_active(cs)) {
940 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
941 }
942 if (kvm_arm_hw_debug_active(cs)) {
943 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
944 kvm_arm_copy_hw_debug_data(&dbg->arch);
945 }
946 }
947
kvm_arch_init_irq_routing(KVMState * s)948 void kvm_arch_init_irq_routing(KVMState *s)
949 {
950 }
951
kvm_arch_irqchip_create(KVMState * s)952 int kvm_arch_irqchip_create(KVMState *s)
953 {
954 if (kvm_kernel_irqchip_split()) {
955 perror("-machine kernel_irqchip=split is not supported on ARM.");
956 exit(1);
957 }
958
959 /* If we can create the VGIC using the newer device control API, we
960 * let the device do this when it initializes itself, otherwise we
961 * fall back to the old API */
962 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
963 }
964
kvm_arm_vgic_probe(void)965 int kvm_arm_vgic_probe(void)
966 {
967 int val = 0;
968
969 if (kvm_create_device(kvm_state,
970 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
971 val |= KVM_ARM_VGIC_V3;
972 }
973 if (kvm_create_device(kvm_state,
974 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
975 val |= KVM_ARM_VGIC_V2;
976 }
977 return val;
978 }
979
kvm_arm_set_irq(int cpu,int irqtype,int irq,int level)980 int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
981 {
982 int kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
983 int cpu_idx1 = cpu % 256;
984 int cpu_idx2 = cpu / 256;
985
986 kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
987 (cpu_idx2 << KVM_ARM_IRQ_VCPU2_SHIFT);
988
989 return kvm_set_irq(kvm_state, kvm_irq, !!level);
990 }
991
kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry * route,uint64_t address,uint32_t data,PCIDevice * dev)992 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
993 uint64_t address, uint32_t data, PCIDevice *dev)
994 {
995 AddressSpace *as = pci_device_iommu_address_space(dev);
996 hwaddr xlat, len, doorbell_gpa;
997 MemoryRegionSection mrs;
998 MemoryRegion *mr;
999 int ret = 1;
1000
1001 if (as == &address_space_memory) {
1002 return 0;
1003 }
1004
1005 /* MSI doorbell address is translated by an IOMMU */
1006
1007 rcu_read_lock();
1008 mr = address_space_translate(as, address, &xlat, &len, true,
1009 MEMTXATTRS_UNSPECIFIED);
1010 if (!mr) {
1011 goto unlock;
1012 }
1013 mrs = memory_region_find(mr, xlat, 1);
1014 if (!mrs.mr) {
1015 goto unlock;
1016 }
1017
1018 doorbell_gpa = mrs.offset_within_address_space;
1019 memory_region_unref(mrs.mr);
1020
1021 route->u.msi.address_lo = doorbell_gpa;
1022 route->u.msi.address_hi = doorbell_gpa >> 32;
1023
1024 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa);
1025
1026 ret = 0;
1027
1028 unlock:
1029 rcu_read_unlock();
1030 return ret;
1031 }
1032
kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry * route,int vector,PCIDevice * dev)1033 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
1034 int vector, PCIDevice *dev)
1035 {
1036 return 0;
1037 }
1038
kvm_arch_release_virq_post(int virq)1039 int kvm_arch_release_virq_post(int virq)
1040 {
1041 return 0;
1042 }
1043
kvm_arch_msi_data_to_gsi(uint32_t data)1044 int kvm_arch_msi_data_to_gsi(uint32_t data)
1045 {
1046 return (data - 32) & 0xffff;
1047 }
1048