1 /*
2 * ARM implementation of KVM hooks
3 *
4 * Copyright Christoffer Dall 2009-2010
5 * Copyright Mian-M. Hamayun 2013, Virtual Open Systems
6 * Copyright Alex Bennée 2014, Linaro
7 *
8 * This work is licensed under the terms of the GNU GPL, version 2 or later.
9 * See the COPYING file in the top-level directory.
10 *
11 */
12
13 #include "qemu/osdep.h"
14 #include <sys/ioctl.h>
15
16 #include <linux/kvm.h>
17
18 #include "qemu/timer.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qom/object.h"
22 #include "qapi/error.h"
23 #include "sysemu/sysemu.h"
24 #include "sysemu/runstate.h"
25 #include "sysemu/kvm.h"
26 #include "sysemu/kvm_int.h"
27 #include "kvm_arm.h"
28 #include "cpu.h"
29 #include "trace.h"
30 #include "internals.h"
31 #include "hw/pci/pci.h"
32 #include "exec/memattrs.h"
33 #include "exec/address-spaces.h"
34 #include "exec/gdbstub.h"
35 #include "hw/boards.h"
36 #include "hw/irq.h"
37 #include "qapi/visitor.h"
38 #include "qemu/log.h"
39 #include "hw/acpi/acpi.h"
40 #include "hw/acpi/ghes.h"
41 #include "target/arm/gtimer.h"
42
43 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
44 KVM_CAP_LAST_INFO
45 };
46
47 static bool cap_has_mp_state;
48 static bool cap_has_inject_serror_esr;
49 static bool cap_has_inject_ext_dabt;
50
51 /**
52 * ARMHostCPUFeatures: information about the host CPU (identified
53 * by asking the host kernel)
54 */
55 typedef struct ARMHostCPUFeatures {
56 ARMISARegisters isar;
57 uint64_t features;
58 uint32_t target;
59 const char *dtb_compatible;
60 } ARMHostCPUFeatures;
61
62 static ARMHostCPUFeatures arm_host_cpu_features;
63
64 /**
65 * kvm_arm_vcpu_init:
66 * @cpu: ARMCPU
67 *
68 * Initialize (or reinitialize) the VCPU by invoking the
69 * KVM_ARM_VCPU_INIT ioctl with the CPU type and feature
70 * bitmask specified in the CPUState.
71 *
72 * Returns: 0 if success else < 0 error code
73 */
kvm_arm_vcpu_init(ARMCPU * cpu)74 static int kvm_arm_vcpu_init(ARMCPU *cpu)
75 {
76 struct kvm_vcpu_init init;
77
78 init.target = cpu->kvm_target;
79 memcpy(init.features, cpu->kvm_init_features, sizeof(init.features));
80
81 return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_INIT, &init);
82 }
83
84 /**
85 * kvm_arm_vcpu_finalize:
86 * @cpu: ARMCPU
87 * @feature: feature to finalize
88 *
89 * Finalizes the configuration of the specified VCPU feature by
90 * invoking the KVM_ARM_VCPU_FINALIZE ioctl. Features requiring
91 * this are documented in the "KVM_ARM_VCPU_FINALIZE" section of
92 * KVM's API documentation.
93 *
94 * Returns: 0 if success else < 0 error code
95 */
kvm_arm_vcpu_finalize(ARMCPU * cpu,int feature)96 static int kvm_arm_vcpu_finalize(ARMCPU *cpu, int feature)
97 {
98 return kvm_vcpu_ioctl(CPU(cpu), KVM_ARM_VCPU_FINALIZE, &feature);
99 }
100
kvm_arm_create_scratch_host_vcpu(const uint32_t * cpus_to_try,int * fdarray,struct kvm_vcpu_init * init)101 bool kvm_arm_create_scratch_host_vcpu(const uint32_t *cpus_to_try,
102 int *fdarray,
103 struct kvm_vcpu_init *init)
104 {
105 int ret = 0, kvmfd = -1, vmfd = -1, cpufd = -1;
106 int max_vm_pa_size;
107
108 kvmfd = qemu_open_old("/dev/kvm", O_RDWR);
109 if (kvmfd < 0) {
110 goto err;
111 }
112 max_vm_pa_size = ioctl(kvmfd, KVM_CHECK_EXTENSION, KVM_CAP_ARM_VM_IPA_SIZE);
113 if (max_vm_pa_size < 0) {
114 max_vm_pa_size = 0;
115 }
116 do {
117 vmfd = ioctl(kvmfd, KVM_CREATE_VM, max_vm_pa_size);
118 } while (vmfd == -1 && errno == EINTR);
119 if (vmfd < 0) {
120 goto err;
121 }
122 cpufd = ioctl(vmfd, KVM_CREATE_VCPU, 0);
123 if (cpufd < 0) {
124 goto err;
125 }
126
127 if (!init) {
128 /* Caller doesn't want the VCPU to be initialized, so skip it */
129 goto finish;
130 }
131
132 if (init->target == -1) {
133 struct kvm_vcpu_init preferred;
134
135 ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, &preferred);
136 if (!ret) {
137 init->target = preferred.target;
138 }
139 }
140 if (ret >= 0) {
141 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
142 if (ret < 0) {
143 goto err;
144 }
145 } else if (cpus_to_try) {
146 /* Old kernel which doesn't know about the
147 * PREFERRED_TARGET ioctl: we know it will only support
148 * creating one kind of guest CPU which is its preferred
149 * CPU type.
150 */
151 struct kvm_vcpu_init try;
152
153 while (*cpus_to_try != QEMU_KVM_ARM_TARGET_NONE) {
154 try.target = *cpus_to_try++;
155 memcpy(try.features, init->features, sizeof(init->features));
156 ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, &try);
157 if (ret >= 0) {
158 break;
159 }
160 }
161 if (ret < 0) {
162 goto err;
163 }
164 init->target = try.target;
165 } else {
166 /* Treat a NULL cpus_to_try argument the same as an empty
167 * list, which means we will fail the call since this must
168 * be an old kernel which doesn't support PREFERRED_TARGET.
169 */
170 goto err;
171 }
172
173 finish:
174 fdarray[0] = kvmfd;
175 fdarray[1] = vmfd;
176 fdarray[2] = cpufd;
177
178 return true;
179
180 err:
181 if (cpufd >= 0) {
182 close(cpufd);
183 }
184 if (vmfd >= 0) {
185 close(vmfd);
186 }
187 if (kvmfd >= 0) {
188 close(kvmfd);
189 }
190
191 return false;
192 }
193
kvm_arm_destroy_scratch_host_vcpu(int * fdarray)194 void kvm_arm_destroy_scratch_host_vcpu(int *fdarray)
195 {
196 int i;
197
198 for (i = 2; i >= 0; i--) {
199 close(fdarray[i]);
200 }
201 }
202
read_sys_reg32(int fd,uint32_t * pret,uint64_t id)203 static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id)
204 {
205 uint64_t ret;
206 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret };
207 int err;
208
209 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
210 err = ioctl(fd, KVM_GET_ONE_REG, &idreg);
211 if (err < 0) {
212 return -1;
213 }
214 *pret = ret;
215 return 0;
216 }
217
read_sys_reg64(int fd,uint64_t * pret,uint64_t id)218 static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id)
219 {
220 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret };
221
222 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
223 return ioctl(fd, KVM_GET_ONE_REG, &idreg);
224 }
225
kvm_arm_pauth_supported(void)226 static bool kvm_arm_pauth_supported(void)
227 {
228 return (kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_ADDRESS) &&
229 kvm_check_extension(kvm_state, KVM_CAP_ARM_PTRAUTH_GENERIC));
230 }
231
kvm_arm_get_host_cpu_features(ARMHostCPUFeatures * ahcf)232 static bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
233 {
234 /* Identify the feature bits corresponding to the host CPU, and
235 * fill out the ARMHostCPUClass fields accordingly. To do this
236 * we have to create a scratch VM, create a single CPU inside it,
237 * and then query that CPU for the relevant ID registers.
238 */
239 int fdarray[3];
240 bool sve_supported;
241 bool pmu_supported = false;
242 uint64_t features = 0;
243 int err;
244
245 /* Old kernels may not know about the PREFERRED_TARGET ioctl: however
246 * we know these will only support creating one kind of guest CPU,
247 * which is its preferred CPU type. Fortunately these old kernels
248 * support only a very limited number of CPUs.
249 */
250 static const uint32_t cpus_to_try[] = {
251 KVM_ARM_TARGET_AEM_V8,
252 KVM_ARM_TARGET_FOUNDATION_V8,
253 KVM_ARM_TARGET_CORTEX_A57,
254 QEMU_KVM_ARM_TARGET_NONE
255 };
256 /*
257 * target = -1 informs kvm_arm_create_scratch_host_vcpu()
258 * to use the preferred target
259 */
260 struct kvm_vcpu_init init = { .target = -1, };
261
262 /*
263 * Ask for SVE if supported, so that we can query ID_AA64ZFR0,
264 * which is otherwise RAZ.
265 */
266 sve_supported = kvm_arm_sve_supported();
267 if (sve_supported) {
268 init.features[0] |= 1 << KVM_ARM_VCPU_SVE;
269 }
270
271 /*
272 * Ask for Pointer Authentication if supported, so that we get
273 * the unsanitized field values for AA64ISAR1_EL1.
274 */
275 if (kvm_arm_pauth_supported()) {
276 init.features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS |
277 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC);
278 }
279
280 if (kvm_arm_pmu_supported()) {
281 init.features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
282 pmu_supported = true;
283 }
284
285 if (!kvm_arm_create_scratch_host_vcpu(cpus_to_try, fdarray, &init)) {
286 return false;
287 }
288
289 ahcf->target = init.target;
290 ahcf->dtb_compatible = "arm,arm-v8";
291
292 err = read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr0,
293 ARM64_SYS_REG(3, 0, 0, 4, 0));
294 if (unlikely(err < 0)) {
295 /*
296 * Before v4.15, the kernel only exposed a limited number of system
297 * registers, not including any of the interesting AArch64 ID regs.
298 * For the most part we could leave these fields as zero with minimal
299 * effect, since this does not affect the values seen by the guest.
300 *
301 * However, it could cause problems down the line for QEMU,
302 * so provide a minimal v8.0 default.
303 *
304 * ??? Could read MIDR and use knowledge from cpu64.c.
305 * ??? Could map a page of memory into our temp guest and
306 * run the tiniest of hand-crafted kernels to extract
307 * the values seen by the guest.
308 * ??? Either of these sounds like too much effort just
309 * to work around running a modern host kernel.
310 */
311 ahcf->isar.id_aa64pfr0 = 0x00000011; /* EL1&0, AArch64 only */
312 err = 0;
313 } else {
314 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr1,
315 ARM64_SYS_REG(3, 0, 0, 4, 1));
316 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64smfr0,
317 ARM64_SYS_REG(3, 0, 0, 4, 5));
318 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr0,
319 ARM64_SYS_REG(3, 0, 0, 5, 0));
320 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr1,
321 ARM64_SYS_REG(3, 0, 0, 5, 1));
322 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar0,
323 ARM64_SYS_REG(3, 0, 0, 6, 0));
324 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar1,
325 ARM64_SYS_REG(3, 0, 0, 6, 1));
326 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar2,
327 ARM64_SYS_REG(3, 0, 0, 6, 2));
328 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr0,
329 ARM64_SYS_REG(3, 0, 0, 7, 0));
330 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr1,
331 ARM64_SYS_REG(3, 0, 0, 7, 1));
332 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr2,
333 ARM64_SYS_REG(3, 0, 0, 7, 2));
334 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr3,
335 ARM64_SYS_REG(3, 0, 0, 7, 3));
336
337 /*
338 * Note that if AArch32 support is not present in the host,
339 * the AArch32 sysregs are present to be read, but will
340 * return UNKNOWN values. This is neither better nor worse
341 * than skipping the reads and leaving 0, as we must avoid
342 * considering the values in every case.
343 */
344 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr0,
345 ARM64_SYS_REG(3, 0, 0, 1, 0));
346 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr1,
347 ARM64_SYS_REG(3, 0, 0, 1, 1));
348 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_dfr0,
349 ARM64_SYS_REG(3, 0, 0, 1, 2));
350 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr0,
351 ARM64_SYS_REG(3, 0, 0, 1, 4));
352 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr1,
353 ARM64_SYS_REG(3, 0, 0, 1, 5));
354 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr2,
355 ARM64_SYS_REG(3, 0, 0, 1, 6));
356 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr3,
357 ARM64_SYS_REG(3, 0, 0, 1, 7));
358 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar0,
359 ARM64_SYS_REG(3, 0, 0, 2, 0));
360 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar1,
361 ARM64_SYS_REG(3, 0, 0, 2, 1));
362 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar2,
363 ARM64_SYS_REG(3, 0, 0, 2, 2));
364 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar3,
365 ARM64_SYS_REG(3, 0, 0, 2, 3));
366 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar4,
367 ARM64_SYS_REG(3, 0, 0, 2, 4));
368 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar5,
369 ARM64_SYS_REG(3, 0, 0, 2, 5));
370 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr4,
371 ARM64_SYS_REG(3, 0, 0, 2, 6));
372 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar6,
373 ARM64_SYS_REG(3, 0, 0, 2, 7));
374
375 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr0,
376 ARM64_SYS_REG(3, 0, 0, 3, 0));
377 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr1,
378 ARM64_SYS_REG(3, 0, 0, 3, 1));
379 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr2,
380 ARM64_SYS_REG(3, 0, 0, 3, 2));
381 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr2,
382 ARM64_SYS_REG(3, 0, 0, 3, 4));
383 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_dfr1,
384 ARM64_SYS_REG(3, 0, 0, 3, 5));
385 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr5,
386 ARM64_SYS_REG(3, 0, 0, 3, 6));
387
388 /*
389 * DBGDIDR is a bit complicated because the kernel doesn't
390 * provide an accessor for it in 64-bit mode, which is what this
391 * scratch VM is in, and there's no architected "64-bit sysreg
392 * which reads the same as the 32-bit register" the way there is
393 * for other ID registers. Instead we synthesize a value from the
394 * AArch64 ID_AA64DFR0, the same way the kernel code in
395 * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does.
396 * We only do this if the CPU supports AArch32 at EL1.
397 */
398 if (FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL1) >= 2) {
399 int wrps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, WRPS);
400 int brps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, BRPS);
401 int ctx_cmps =
402 FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, CTX_CMPS);
403 int version = 6; /* ARMv8 debug architecture */
404 bool has_el3 =
405 !!FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL3);
406 uint32_t dbgdidr = 0;
407
408 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps);
409 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps);
410 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps);
411 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version);
412 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3);
413 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3);
414 dbgdidr |= (1 << 15); /* RES1 bit */
415 ahcf->isar.dbgdidr = dbgdidr;
416 }
417
418 if (pmu_supported) {
419 /* PMCR_EL0 is only accessible if the vCPU has feature PMU_V3 */
420 err |= read_sys_reg64(fdarray[2], &ahcf->isar.reset_pmcr_el0,
421 ARM64_SYS_REG(3, 3, 9, 12, 0));
422 }
423
424 if (sve_supported) {
425 /*
426 * There is a range of kernels between kernel commit 73433762fcae
427 * and f81cb2c3ad41 which have a bug where the kernel doesn't
428 * expose SYS_ID_AA64ZFR0_EL1 via the ONE_REG API unless the VM has
429 * enabled SVE support, which resulted in an error rather than RAZ.
430 * So only read the register if we set KVM_ARM_VCPU_SVE above.
431 */
432 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64zfr0,
433 ARM64_SYS_REG(3, 0, 0, 4, 4));
434 }
435 }
436
437 kvm_arm_destroy_scratch_host_vcpu(fdarray);
438
439 if (err < 0) {
440 return false;
441 }
442
443 /*
444 * We can assume any KVM supporting CPU is at least a v8
445 * with VFPv4+Neon; this in turn implies most of the other
446 * feature bits.
447 */
448 features |= 1ULL << ARM_FEATURE_V8;
449 features |= 1ULL << ARM_FEATURE_NEON;
450 features |= 1ULL << ARM_FEATURE_AARCH64;
451 features |= 1ULL << ARM_FEATURE_PMU;
452 features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
453
454 ahcf->features = features;
455
456 return true;
457 }
458
kvm_arm_set_cpu_features_from_host(ARMCPU * cpu)459 void kvm_arm_set_cpu_features_from_host(ARMCPU *cpu)
460 {
461 CPUARMState *env = &cpu->env;
462
463 if (!arm_host_cpu_features.dtb_compatible) {
464 if (!kvm_enabled() ||
465 !kvm_arm_get_host_cpu_features(&arm_host_cpu_features)) {
466 /* We can't report this error yet, so flag that we need to
467 * in arm_cpu_realizefn().
468 */
469 cpu->kvm_target = QEMU_KVM_ARM_TARGET_NONE;
470 cpu->host_cpu_probe_failed = true;
471 return;
472 }
473 }
474
475 cpu->kvm_target = arm_host_cpu_features.target;
476 cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
477 cpu->isar = arm_host_cpu_features.isar;
478 env->features = arm_host_cpu_features.features;
479 }
480
kvm_no_adjvtime_get(Object * obj,Error ** errp)481 static bool kvm_no_adjvtime_get(Object *obj, Error **errp)
482 {
483 return !ARM_CPU(obj)->kvm_adjvtime;
484 }
485
kvm_no_adjvtime_set(Object * obj,bool value,Error ** errp)486 static void kvm_no_adjvtime_set(Object *obj, bool value, Error **errp)
487 {
488 ARM_CPU(obj)->kvm_adjvtime = !value;
489 }
490
kvm_steal_time_get(Object * obj,Error ** errp)491 static bool kvm_steal_time_get(Object *obj, Error **errp)
492 {
493 return ARM_CPU(obj)->kvm_steal_time != ON_OFF_AUTO_OFF;
494 }
495
kvm_steal_time_set(Object * obj,bool value,Error ** errp)496 static void kvm_steal_time_set(Object *obj, bool value, Error **errp)
497 {
498 ARM_CPU(obj)->kvm_steal_time = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
499 }
500
501 /* KVM VCPU properties should be prefixed with "kvm-". */
kvm_arm_add_vcpu_properties(ARMCPU * cpu)502 void kvm_arm_add_vcpu_properties(ARMCPU *cpu)
503 {
504 CPUARMState *env = &cpu->env;
505 Object *obj = OBJECT(cpu);
506
507 if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
508 cpu->kvm_adjvtime = true;
509 object_property_add_bool(obj, "kvm-no-adjvtime", kvm_no_adjvtime_get,
510 kvm_no_adjvtime_set);
511 object_property_set_description(obj, "kvm-no-adjvtime",
512 "Set on to disable the adjustment of "
513 "the virtual counter. VM stopped time "
514 "will be counted.");
515 }
516
517 cpu->kvm_steal_time = ON_OFF_AUTO_AUTO;
518 object_property_add_bool(obj, "kvm-steal-time", kvm_steal_time_get,
519 kvm_steal_time_set);
520 object_property_set_description(obj, "kvm-steal-time",
521 "Set off to disable KVM steal time.");
522 }
523
kvm_arm_pmu_supported(void)524 bool kvm_arm_pmu_supported(void)
525 {
526 return kvm_check_extension(kvm_state, KVM_CAP_ARM_PMU_V3);
527 }
528
kvm_arm_get_max_vm_ipa_size(MachineState * ms,bool * fixed_ipa)529 int kvm_arm_get_max_vm_ipa_size(MachineState *ms, bool *fixed_ipa)
530 {
531 KVMState *s = KVM_STATE(ms->accelerator);
532 int ret;
533
534 ret = kvm_check_extension(s, KVM_CAP_ARM_VM_IPA_SIZE);
535 *fixed_ipa = ret <= 0;
536
537 return ret > 0 ? ret : 40;
538 }
539
kvm_arch_get_default_type(MachineState * ms)540 int kvm_arch_get_default_type(MachineState *ms)
541 {
542 bool fixed_ipa;
543 int size = kvm_arm_get_max_vm_ipa_size(ms, &fixed_ipa);
544 return fixed_ipa ? 0 : size;
545 }
546
kvm_arch_init(MachineState * ms,KVMState * s)547 int kvm_arch_init(MachineState *ms, KVMState *s)
548 {
549 int ret = 0;
550 /* For ARM interrupt delivery is always asynchronous,
551 * whether we are using an in-kernel VGIC or not.
552 */
553 kvm_async_interrupts_allowed = true;
554
555 /*
556 * PSCI wakes up secondary cores, so we always need to
557 * have vCPUs waiting in kernel space
558 */
559 kvm_halt_in_kernel_allowed = true;
560
561 cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
562
563 /* Check whether user space can specify guest syndrome value */
564 cap_has_inject_serror_esr =
565 kvm_check_extension(s, KVM_CAP_ARM_INJECT_SERROR_ESR);
566
567 if (ms->smp.cpus > 256 &&
568 !kvm_check_extension(s, KVM_CAP_ARM_IRQ_LINE_LAYOUT_2)) {
569 error_report("Using more than 256 vcpus requires a host kernel "
570 "with KVM_CAP_ARM_IRQ_LINE_LAYOUT_2");
571 ret = -EINVAL;
572 }
573
574 if (kvm_check_extension(s, KVM_CAP_ARM_NISV_TO_USER)) {
575 if (kvm_vm_enable_cap(s, KVM_CAP_ARM_NISV_TO_USER, 0)) {
576 error_report("Failed to enable KVM_CAP_ARM_NISV_TO_USER cap");
577 } else {
578 /* Set status for supporting the external dabt injection */
579 cap_has_inject_ext_dabt = kvm_check_extension(s,
580 KVM_CAP_ARM_INJECT_EXT_DABT);
581 }
582 }
583
584 if (s->kvm_eager_split_size) {
585 uint32_t sizes;
586
587 sizes = kvm_vm_check_extension(s, KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES);
588 if (!sizes) {
589 s->kvm_eager_split_size = 0;
590 warn_report("Eager Page Split support not available");
591 } else if (!(s->kvm_eager_split_size & sizes)) {
592 error_report("Eager Page Split requested chunk size not valid");
593 ret = -EINVAL;
594 } else {
595 ret = kvm_vm_enable_cap(s, KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE, 0,
596 s->kvm_eager_split_size);
597 if (ret < 0) {
598 error_report("Enabling of Eager Page Split failed: %s",
599 strerror(-ret));
600 }
601 }
602 }
603
604 max_hw_wps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_WPS);
605 hw_watchpoints = g_array_sized_new(true, true,
606 sizeof(HWWatchpoint), max_hw_wps);
607
608 max_hw_bps = kvm_check_extension(s, KVM_CAP_GUEST_DEBUG_HW_BPS);
609 hw_breakpoints = g_array_sized_new(true, true,
610 sizeof(HWBreakpoint), max_hw_bps);
611
612 return ret;
613 }
614
kvm_arch_vcpu_id(CPUState * cpu)615 unsigned long kvm_arch_vcpu_id(CPUState *cpu)
616 {
617 return cpu->cpu_index;
618 }
619
620 /* We track all the KVM devices which need their memory addresses
621 * passing to the kernel in a list of these structures.
622 * When board init is complete we run through the list and
623 * tell the kernel the base addresses of the memory regions.
624 * We use a MemoryListener to track mapping and unmapping of
625 * the regions during board creation, so the board models don't
626 * need to do anything special for the KVM case.
627 *
628 * Sometimes the address must be OR'ed with some other fields
629 * (for example for KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION).
630 * @kda_addr_ormask aims at storing the value of those fields.
631 */
632 typedef struct KVMDevice {
633 struct kvm_arm_device_addr kda;
634 struct kvm_device_attr kdattr;
635 uint64_t kda_addr_ormask;
636 MemoryRegion *mr;
637 QSLIST_ENTRY(KVMDevice) entries;
638 int dev_fd;
639 } KVMDevice;
640
641 static QSLIST_HEAD(, KVMDevice) kvm_devices_head;
642
kvm_arm_devlistener_add(MemoryListener * listener,MemoryRegionSection * section)643 static void kvm_arm_devlistener_add(MemoryListener *listener,
644 MemoryRegionSection *section)
645 {
646 KVMDevice *kd;
647
648 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
649 if (section->mr == kd->mr) {
650 kd->kda.addr = section->offset_within_address_space;
651 }
652 }
653 }
654
kvm_arm_devlistener_del(MemoryListener * listener,MemoryRegionSection * section)655 static void kvm_arm_devlistener_del(MemoryListener *listener,
656 MemoryRegionSection *section)
657 {
658 KVMDevice *kd;
659
660 QSLIST_FOREACH(kd, &kvm_devices_head, entries) {
661 if (section->mr == kd->mr) {
662 kd->kda.addr = -1;
663 }
664 }
665 }
666
667 static MemoryListener devlistener = {
668 .name = "kvm-arm",
669 .region_add = kvm_arm_devlistener_add,
670 .region_del = kvm_arm_devlistener_del,
671 .priority = MEMORY_LISTENER_PRIORITY_MIN,
672 };
673
kvm_arm_set_device_addr(KVMDevice * kd)674 static void kvm_arm_set_device_addr(KVMDevice *kd)
675 {
676 struct kvm_device_attr *attr = &kd->kdattr;
677 int ret;
678
679 /* If the device control API is available and we have a device fd on the
680 * KVMDevice struct, let's use the newer API
681 */
682 if (kd->dev_fd >= 0) {
683 uint64_t addr = kd->kda.addr;
684
685 addr |= kd->kda_addr_ormask;
686 attr->addr = (uintptr_t)&addr;
687 ret = kvm_device_ioctl(kd->dev_fd, KVM_SET_DEVICE_ATTR, attr);
688 } else {
689 ret = kvm_vm_ioctl(kvm_state, KVM_ARM_SET_DEVICE_ADDR, &kd->kda);
690 }
691
692 if (ret < 0) {
693 fprintf(stderr, "Failed to set device address: %s\n",
694 strerror(-ret));
695 abort();
696 }
697 }
698
kvm_arm_machine_init_done(Notifier * notifier,void * data)699 static void kvm_arm_machine_init_done(Notifier *notifier, void *data)
700 {
701 KVMDevice *kd, *tkd;
702
703 QSLIST_FOREACH_SAFE(kd, &kvm_devices_head, entries, tkd) {
704 if (kd->kda.addr != -1) {
705 kvm_arm_set_device_addr(kd);
706 }
707 memory_region_unref(kd->mr);
708 QSLIST_REMOVE_HEAD(&kvm_devices_head, entries);
709 g_free(kd);
710 }
711 memory_listener_unregister(&devlistener);
712 }
713
714 static Notifier notify = {
715 .notify = kvm_arm_machine_init_done,
716 };
717
kvm_arm_register_device(MemoryRegion * mr,uint64_t devid,uint64_t group,uint64_t attr,int dev_fd,uint64_t addr_ormask)718 void kvm_arm_register_device(MemoryRegion *mr, uint64_t devid, uint64_t group,
719 uint64_t attr, int dev_fd, uint64_t addr_ormask)
720 {
721 KVMDevice *kd;
722
723 if (!kvm_irqchip_in_kernel()) {
724 return;
725 }
726
727 if (QSLIST_EMPTY(&kvm_devices_head)) {
728 memory_listener_register(&devlistener, &address_space_memory);
729 qemu_add_machine_init_done_notifier(¬ify);
730 }
731 kd = g_new0(KVMDevice, 1);
732 kd->mr = mr;
733 kd->kda.id = devid;
734 kd->kda.addr = -1;
735 kd->kdattr.flags = 0;
736 kd->kdattr.group = group;
737 kd->kdattr.attr = attr;
738 kd->dev_fd = dev_fd;
739 kd->kda_addr_ormask = addr_ormask;
740 QSLIST_INSERT_HEAD(&kvm_devices_head, kd, entries);
741 memory_region_ref(kd->mr);
742 }
743
compare_u64(const void * a,const void * b)744 static int compare_u64(const void *a, const void *b)
745 {
746 if (*(uint64_t *)a > *(uint64_t *)b) {
747 return 1;
748 }
749 if (*(uint64_t *)a < *(uint64_t *)b) {
750 return -1;
751 }
752 return 0;
753 }
754
755 /*
756 * cpreg_values are sorted in ascending order by KVM register ID
757 * (see kvm_arm_init_cpreg_list). This allows us to cheaply find
758 * the storage for a KVM register by ID with a binary search.
759 */
kvm_arm_get_cpreg_ptr(ARMCPU * cpu,uint64_t regidx)760 static uint64_t *kvm_arm_get_cpreg_ptr(ARMCPU *cpu, uint64_t regidx)
761 {
762 uint64_t *res;
763
764 res = bsearch(®idx, cpu->cpreg_indexes, cpu->cpreg_array_len,
765 sizeof(uint64_t), compare_u64);
766 assert(res);
767
768 return &cpu->cpreg_values[res - cpu->cpreg_indexes];
769 }
770
771 /**
772 * kvm_arm_reg_syncs_via_cpreg_list:
773 * @regidx: KVM register index
774 *
775 * Return true if this KVM register should be synchronized via the
776 * cpreg list of arbitrary system registers, false if it is synchronized
777 * by hand using code in kvm_arch_get/put_registers().
778 */
kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)779 static bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
780 {
781 switch (regidx & KVM_REG_ARM_COPROC_MASK) {
782 case KVM_REG_ARM_CORE:
783 case KVM_REG_ARM64_SVE:
784 return false;
785 default:
786 return true;
787 }
788 }
789
790 /**
791 * kvm_arm_init_cpreg_list:
792 * @cpu: ARMCPU
793 *
794 * Initialize the ARMCPU cpreg list according to the kernel's
795 * definition of what CPU registers it knows about (and throw away
796 * the previous TCG-created cpreg list).
797 *
798 * Returns: 0 if success, else < 0 error code
799 */
kvm_arm_init_cpreg_list(ARMCPU * cpu)800 static int kvm_arm_init_cpreg_list(ARMCPU *cpu)
801 {
802 struct kvm_reg_list rl;
803 struct kvm_reg_list *rlp;
804 int i, ret, arraylen;
805 CPUState *cs = CPU(cpu);
806
807 rl.n = 0;
808 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
809 if (ret != -E2BIG) {
810 return ret;
811 }
812 rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
813 rlp->n = rl.n;
814 ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
815 if (ret) {
816 goto out;
817 }
818 /* Sort the list we get back from the kernel, since cpreg_tuples
819 * must be in strictly ascending order.
820 */
821 qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
822
823 for (i = 0, arraylen = 0; i < rlp->n; i++) {
824 if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
825 continue;
826 }
827 switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
828 case KVM_REG_SIZE_U32:
829 case KVM_REG_SIZE_U64:
830 break;
831 default:
832 fprintf(stderr, "Can't handle size of register in kernel list\n");
833 ret = -EINVAL;
834 goto out;
835 }
836
837 arraylen++;
838 }
839
840 cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
841 cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
842 cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
843 arraylen);
844 cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
845 arraylen);
846 cpu->cpreg_array_len = arraylen;
847 cpu->cpreg_vmstate_array_len = arraylen;
848
849 for (i = 0, arraylen = 0; i < rlp->n; i++) {
850 uint64_t regidx = rlp->reg[i];
851 if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
852 continue;
853 }
854 cpu->cpreg_indexes[arraylen] = regidx;
855 arraylen++;
856 }
857 assert(cpu->cpreg_array_len == arraylen);
858
859 if (!write_kvmstate_to_list(cpu)) {
860 /* Shouldn't happen unless kernel is inconsistent about
861 * what registers exist.
862 */
863 fprintf(stderr, "Initial read of kernel register state failed\n");
864 ret = -EINVAL;
865 goto out;
866 }
867
868 out:
869 g_free(rlp);
870 return ret;
871 }
872
873 /**
874 * kvm_arm_cpreg_level:
875 * @regidx: KVM register index
876 *
877 * Return the level of this coprocessor/system register. Return value is
878 * either KVM_PUT_RUNTIME_STATE, KVM_PUT_RESET_STATE, or KVM_PUT_FULL_STATE.
879 */
kvm_arm_cpreg_level(uint64_t regidx)880 static int kvm_arm_cpreg_level(uint64_t regidx)
881 {
882 /*
883 * All system registers are assumed to be level KVM_PUT_RUNTIME_STATE.
884 * If a register should be written less often, you must add it here
885 * with a state of either KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE.
886 */
887 switch (regidx) {
888 case KVM_REG_ARM_TIMER_CNT:
889 case KVM_REG_ARM_PTIMER_CNT:
890 return KVM_PUT_FULL_STATE;
891 }
892 return KVM_PUT_RUNTIME_STATE;
893 }
894
write_kvmstate_to_list(ARMCPU * cpu)895 bool write_kvmstate_to_list(ARMCPU *cpu)
896 {
897 CPUState *cs = CPU(cpu);
898 int i;
899 bool ok = true;
900
901 for (i = 0; i < cpu->cpreg_array_len; i++) {
902 uint64_t regidx = cpu->cpreg_indexes[i];
903 uint32_t v32;
904 int ret;
905
906 switch (regidx & KVM_REG_SIZE_MASK) {
907 case KVM_REG_SIZE_U32:
908 ret = kvm_get_one_reg(cs, regidx, &v32);
909 if (!ret) {
910 cpu->cpreg_values[i] = v32;
911 }
912 break;
913 case KVM_REG_SIZE_U64:
914 ret = kvm_get_one_reg(cs, regidx, cpu->cpreg_values + i);
915 break;
916 default:
917 g_assert_not_reached();
918 }
919 if (ret) {
920 ok = false;
921 }
922 }
923 return ok;
924 }
925
write_list_to_kvmstate(ARMCPU * cpu,int level)926 bool write_list_to_kvmstate(ARMCPU *cpu, int level)
927 {
928 CPUState *cs = CPU(cpu);
929 int i;
930 bool ok = true;
931
932 for (i = 0; i < cpu->cpreg_array_len; i++) {
933 uint64_t regidx = cpu->cpreg_indexes[i];
934 uint32_t v32;
935 int ret;
936
937 if (kvm_arm_cpreg_level(regidx) > level) {
938 continue;
939 }
940
941 switch (regidx & KVM_REG_SIZE_MASK) {
942 case KVM_REG_SIZE_U32:
943 v32 = cpu->cpreg_values[i];
944 ret = kvm_set_one_reg(cs, regidx, &v32);
945 break;
946 case KVM_REG_SIZE_U64:
947 ret = kvm_set_one_reg(cs, regidx, cpu->cpreg_values + i);
948 break;
949 default:
950 g_assert_not_reached();
951 }
952 if (ret) {
953 /* We might fail for "unknown register" and also for
954 * "you tried to set a register which is constant with
955 * a different value from what it actually contains".
956 */
957 ok = false;
958 }
959 }
960 return ok;
961 }
962
kvm_arm_cpu_pre_save(ARMCPU * cpu)963 void kvm_arm_cpu_pre_save(ARMCPU *cpu)
964 {
965 /* KVM virtual time adjustment */
966 if (cpu->kvm_vtime_dirty) {
967 *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT) = cpu->kvm_vtime;
968 }
969 }
970
kvm_arm_cpu_post_load(ARMCPU * cpu)971 void kvm_arm_cpu_post_load(ARMCPU *cpu)
972 {
973 /* KVM virtual time adjustment */
974 if (cpu->kvm_adjvtime) {
975 cpu->kvm_vtime = *kvm_arm_get_cpreg_ptr(cpu, KVM_REG_ARM_TIMER_CNT);
976 cpu->kvm_vtime_dirty = true;
977 }
978 }
979
kvm_arm_reset_vcpu(ARMCPU * cpu)980 void kvm_arm_reset_vcpu(ARMCPU *cpu)
981 {
982 int ret;
983
984 /* Re-init VCPU so that all registers are set to
985 * their respective reset values.
986 */
987 ret = kvm_arm_vcpu_init(cpu);
988 if (ret < 0) {
989 fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
990 abort();
991 }
992 if (!write_kvmstate_to_list(cpu)) {
993 fprintf(stderr, "write_kvmstate_to_list failed\n");
994 abort();
995 }
996 /*
997 * Sync the reset values also into the CPUState. This is necessary
998 * because the next thing we do will be a kvm_arch_put_registers()
999 * which will update the list values from the CPUState before copying
1000 * the list values back to KVM. It's OK to ignore failure returns here
1001 * for the same reason we do so in kvm_arch_get_registers().
1002 */
1003 write_list_to_cpustate(cpu);
1004 }
1005
1006 /*
1007 * Update KVM's MP_STATE based on what QEMU thinks it is
1008 */
kvm_arm_sync_mpstate_to_kvm(ARMCPU * cpu)1009 static int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
1010 {
1011 if (cap_has_mp_state) {
1012 struct kvm_mp_state mp_state = {
1013 .mp_state = (cpu->power_state == PSCI_OFF) ?
1014 KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
1015 };
1016 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
1017 }
1018 return 0;
1019 }
1020
1021 /*
1022 * Sync the KVM MP_STATE into QEMU
1023 */
kvm_arm_sync_mpstate_to_qemu(ARMCPU * cpu)1024 static int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
1025 {
1026 if (cap_has_mp_state) {
1027 struct kvm_mp_state mp_state;
1028 int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
1029 if (ret) {
1030 return ret;
1031 }
1032 cpu->power_state = (mp_state.mp_state == KVM_MP_STATE_STOPPED) ?
1033 PSCI_OFF : PSCI_ON;
1034 }
1035 return 0;
1036 }
1037
1038 /**
1039 * kvm_arm_get_virtual_time:
1040 * @cpu: ARMCPU
1041 *
1042 * Gets the VCPU's virtual counter and stores it in the KVM CPU state.
1043 */
kvm_arm_get_virtual_time(ARMCPU * cpu)1044 static void kvm_arm_get_virtual_time(ARMCPU *cpu)
1045 {
1046 int ret;
1047
1048 if (cpu->kvm_vtime_dirty) {
1049 return;
1050 }
1051
1052 ret = kvm_get_one_reg(CPU(cpu), KVM_REG_ARM_TIMER_CNT, &cpu->kvm_vtime);
1053 if (ret) {
1054 error_report("Failed to get KVM_REG_ARM_TIMER_CNT");
1055 abort();
1056 }
1057
1058 cpu->kvm_vtime_dirty = true;
1059 }
1060
1061 /**
1062 * kvm_arm_put_virtual_time:
1063 * @cpu: ARMCPU
1064 *
1065 * Sets the VCPU's virtual counter to the value stored in the KVM CPU state.
1066 */
kvm_arm_put_virtual_time(ARMCPU * cpu)1067 static void kvm_arm_put_virtual_time(ARMCPU *cpu)
1068 {
1069 int ret;
1070
1071 if (!cpu->kvm_vtime_dirty) {
1072 return;
1073 }
1074
1075 ret = kvm_set_one_reg(CPU(cpu), KVM_REG_ARM_TIMER_CNT, &cpu->kvm_vtime);
1076 if (ret) {
1077 error_report("Failed to set KVM_REG_ARM_TIMER_CNT");
1078 abort();
1079 }
1080
1081 cpu->kvm_vtime_dirty = false;
1082 }
1083
1084 /**
1085 * kvm_put_vcpu_events:
1086 * @cpu: ARMCPU
1087 *
1088 * Put VCPU related state to kvm.
1089 *
1090 * Returns: 0 if success else < 0 error code
1091 */
kvm_put_vcpu_events(ARMCPU * cpu)1092 static int kvm_put_vcpu_events(ARMCPU *cpu)
1093 {
1094 CPUARMState *env = &cpu->env;
1095 struct kvm_vcpu_events events;
1096 int ret;
1097
1098 if (!kvm_has_vcpu_events()) {
1099 return 0;
1100 }
1101
1102 memset(&events, 0, sizeof(events));
1103 events.exception.serror_pending = env->serror.pending;
1104
1105 /* Inject SError to guest with specified syndrome if host kernel
1106 * supports it, otherwise inject SError without syndrome.
1107 */
1108 if (cap_has_inject_serror_esr) {
1109 events.exception.serror_has_esr = env->serror.has_esr;
1110 events.exception.serror_esr = env->serror.esr;
1111 }
1112
1113 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
1114 if (ret) {
1115 error_report("failed to put vcpu events");
1116 }
1117
1118 return ret;
1119 }
1120
1121 /**
1122 * kvm_get_vcpu_events:
1123 * @cpu: ARMCPU
1124 *
1125 * Get VCPU related state from kvm.
1126 *
1127 * Returns: 0 if success else < 0 error code
1128 */
kvm_get_vcpu_events(ARMCPU * cpu)1129 static int kvm_get_vcpu_events(ARMCPU *cpu)
1130 {
1131 CPUARMState *env = &cpu->env;
1132 struct kvm_vcpu_events events;
1133 int ret;
1134
1135 if (!kvm_has_vcpu_events()) {
1136 return 0;
1137 }
1138
1139 memset(&events, 0, sizeof(events));
1140 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
1141 if (ret) {
1142 error_report("failed to get vcpu events");
1143 return ret;
1144 }
1145
1146 env->serror.pending = events.exception.serror_pending;
1147 env->serror.has_esr = events.exception.serror_has_esr;
1148 env->serror.esr = events.exception.serror_esr;
1149
1150 return 0;
1151 }
1152
1153 #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0)
1154 #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2)
1155
1156 /*
1157 * ESR_EL1
1158 * ISS encoding
1159 * AARCH64: DFSC, bits [5:0]
1160 * AARCH32:
1161 * TTBCR.EAE == 0
1162 * FS[4] - DFSR[10]
1163 * FS[3:0] - DFSR[3:0]
1164 * TTBCR.EAE == 1
1165 * FS, bits [5:0]
1166 */
1167 #define ESR_DFSC(aarch64, lpae, v) \
1168 ((aarch64 || (lpae)) ? ((v) & 0x3F) \
1169 : (((v) >> 6) | ((v) & 0x1F)))
1170
1171 #define ESR_DFSC_EXTABT(aarch64, lpae) \
1172 ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8)
1173
1174 /**
1175 * kvm_arm_verify_ext_dabt_pending:
1176 * @cpu: ARMCPU
1177 *
1178 * Verify the fault status code wrt the Ext DABT injection
1179 *
1180 * Returns: true if the fault status code is as expected, false otherwise
1181 */
kvm_arm_verify_ext_dabt_pending(ARMCPU * cpu)1182 static bool kvm_arm_verify_ext_dabt_pending(ARMCPU *cpu)
1183 {
1184 CPUState *cs = CPU(cpu);
1185 uint64_t dfsr_val;
1186
1187 if (!kvm_get_one_reg(cs, ARM64_REG_ESR_EL1, &dfsr_val)) {
1188 CPUARMState *env = &cpu->env;
1189 int aarch64_mode = arm_feature(env, ARM_FEATURE_AARCH64);
1190 int lpae = 0;
1191
1192 if (!aarch64_mode) {
1193 uint64_t ttbcr;
1194
1195 if (!kvm_get_one_reg(cs, ARM64_REG_TCR_EL1, &ttbcr)) {
1196 lpae = arm_feature(env, ARM_FEATURE_LPAE)
1197 && (ttbcr & TTBCR_EAE);
1198 }
1199 }
1200 /*
1201 * The verification here is based on the DFSC bits
1202 * of the ESR_EL1 reg only
1203 */
1204 return (ESR_DFSC(aarch64_mode, lpae, dfsr_val) ==
1205 ESR_DFSC_EXTABT(aarch64_mode, lpae));
1206 }
1207 return false;
1208 }
1209
kvm_arch_pre_run(CPUState * cs,struct kvm_run * run)1210 void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
1211 {
1212 ARMCPU *cpu = ARM_CPU(cs);
1213 CPUARMState *env = &cpu->env;
1214
1215 if (unlikely(env->ext_dabt_raised)) {
1216 /*
1217 * Verifying that the ext DABT has been properly injected,
1218 * otherwise risking indefinitely re-running the faulting instruction
1219 * Covering a very narrow case for kernels 5.5..5.5.4
1220 * when injected abort was misconfigured to be
1221 * an IMPLEMENTATION DEFINED exception (for 32-bit EL1)
1222 */
1223 if (!arm_feature(env, ARM_FEATURE_AARCH64) &&
1224 unlikely(!kvm_arm_verify_ext_dabt_pending(cpu))) {
1225
1226 error_report("Data abort exception with no valid ISS generated by "
1227 "guest memory access. KVM unable to emulate faulting "
1228 "instruction. Failed to inject an external data abort "
1229 "into the guest.");
1230 abort();
1231 }
1232 /* Clear the status */
1233 env->ext_dabt_raised = 0;
1234 }
1235 }
1236
kvm_arch_post_run(CPUState * cs,struct kvm_run * run)1237 MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
1238 {
1239 ARMCPU *cpu;
1240 uint32_t switched_level;
1241
1242 if (kvm_irqchip_in_kernel()) {
1243 /*
1244 * We only need to sync timer states with user-space interrupt
1245 * controllers, so return early and save cycles if we don't.
1246 */
1247 return MEMTXATTRS_UNSPECIFIED;
1248 }
1249
1250 cpu = ARM_CPU(cs);
1251
1252 /* Synchronize our shadowed in-kernel device irq lines with the kvm ones */
1253 if (run->s.regs.device_irq_level != cpu->device_irq_level) {
1254 switched_level = cpu->device_irq_level ^ run->s.regs.device_irq_level;
1255
1256 bql_lock();
1257
1258 if (switched_level & KVM_ARM_DEV_EL1_VTIMER) {
1259 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_VIRT],
1260 !!(run->s.regs.device_irq_level &
1261 KVM_ARM_DEV_EL1_VTIMER));
1262 switched_level &= ~KVM_ARM_DEV_EL1_VTIMER;
1263 }
1264
1265 if (switched_level & KVM_ARM_DEV_EL1_PTIMER) {
1266 qemu_set_irq(cpu->gt_timer_outputs[GTIMER_PHYS],
1267 !!(run->s.regs.device_irq_level &
1268 KVM_ARM_DEV_EL1_PTIMER));
1269 switched_level &= ~KVM_ARM_DEV_EL1_PTIMER;
1270 }
1271
1272 if (switched_level & KVM_ARM_DEV_PMU) {
1273 qemu_set_irq(cpu->pmu_interrupt,
1274 !!(run->s.regs.device_irq_level & KVM_ARM_DEV_PMU));
1275 switched_level &= ~KVM_ARM_DEV_PMU;
1276 }
1277
1278 if (switched_level) {
1279 qemu_log_mask(LOG_UNIMP, "%s: unhandled in-kernel device IRQ %x\n",
1280 __func__, switched_level);
1281 }
1282
1283 /* We also mark unknown levels as processed to not waste cycles */
1284 cpu->device_irq_level = run->s.regs.device_irq_level;
1285 bql_unlock();
1286 }
1287
1288 return MEMTXATTRS_UNSPECIFIED;
1289 }
1290
kvm_arm_vm_state_change(void * opaque,bool running,RunState state)1291 static void kvm_arm_vm_state_change(void *opaque, bool running, RunState state)
1292 {
1293 ARMCPU *cpu = opaque;
1294
1295 if (running) {
1296 if (cpu->kvm_adjvtime) {
1297 kvm_arm_put_virtual_time(cpu);
1298 }
1299 } else {
1300 if (cpu->kvm_adjvtime) {
1301 kvm_arm_get_virtual_time(cpu);
1302 }
1303 }
1304 }
1305
1306 /**
1307 * kvm_arm_handle_dabt_nisv:
1308 * @cpu: ARMCPU
1309 * @esr_iss: ISS encoding (limited) for the exception from Data Abort
1310 * ISV bit set to '0b0' -> no valid instruction syndrome
1311 * @fault_ipa: faulting address for the synchronous data abort
1312 *
1313 * Returns: 0 if the exception has been handled, < 0 otherwise
1314 */
kvm_arm_handle_dabt_nisv(ARMCPU * cpu,uint64_t esr_iss,uint64_t fault_ipa)1315 static int kvm_arm_handle_dabt_nisv(ARMCPU *cpu, uint64_t esr_iss,
1316 uint64_t fault_ipa)
1317 {
1318 CPUARMState *env = &cpu->env;
1319 /*
1320 * Request KVM to inject the external data abort into the guest
1321 */
1322 if (cap_has_inject_ext_dabt) {
1323 struct kvm_vcpu_events events = { };
1324 /*
1325 * The external data abort event will be handled immediately by KVM
1326 * using the address fault that triggered the exit on given VCPU.
1327 * Requesting injection of the external data abort does not rely
1328 * on any other VCPU state. Therefore, in this particular case, the VCPU
1329 * synchronization can be exceptionally skipped.
1330 */
1331 events.exception.ext_dabt_pending = 1;
1332 /* KVM_CAP_ARM_INJECT_EXT_DABT implies KVM_CAP_VCPU_EVENTS */
1333 if (!kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events)) {
1334 env->ext_dabt_raised = 1;
1335 return 0;
1336 }
1337 } else {
1338 error_report("Data abort exception triggered by guest memory access "
1339 "at physical address: 0x" TARGET_FMT_lx,
1340 (target_ulong)fault_ipa);
1341 error_printf("KVM unable to emulate faulting instruction.\n");
1342 }
1343 return -1;
1344 }
1345
1346 /**
1347 * kvm_arm_handle_debug:
1348 * @cpu: ARMCPU
1349 * @debug_exit: debug part of the KVM exit structure
1350 *
1351 * Returns: TRUE if the debug exception was handled.
1352 *
1353 * See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register
1354 *
1355 * To minimise translating between kernel and user-space the kernel
1356 * ABI just provides user-space with the full exception syndrome
1357 * register value to be decoded in QEMU.
1358 */
kvm_arm_handle_debug(ARMCPU * cpu,struct kvm_debug_exit_arch * debug_exit)1359 static bool kvm_arm_handle_debug(ARMCPU *cpu,
1360 struct kvm_debug_exit_arch *debug_exit)
1361 {
1362 int hsr_ec = syn_get_ec(debug_exit->hsr);
1363 CPUState *cs = CPU(cpu);
1364 CPUARMState *env = &cpu->env;
1365
1366 /* Ensure PC is synchronised */
1367 kvm_cpu_synchronize_state(cs);
1368
1369 switch (hsr_ec) {
1370 case EC_SOFTWARESTEP:
1371 if (cs->singlestep_enabled) {
1372 return true;
1373 } else {
1374 /*
1375 * The kernel should have suppressed the guest's ability to
1376 * single step at this point so something has gone wrong.
1377 */
1378 error_report("%s: guest single-step while debugging unsupported"
1379 " (%"PRIx64", %"PRIx32")",
1380 __func__, env->pc, debug_exit->hsr);
1381 return false;
1382 }
1383 break;
1384 case EC_AA64_BKPT:
1385 if (kvm_find_sw_breakpoint(cs, env->pc)) {
1386 return true;
1387 }
1388 break;
1389 case EC_BREAKPOINT:
1390 if (find_hw_breakpoint(cs, env->pc)) {
1391 return true;
1392 }
1393 break;
1394 case EC_WATCHPOINT:
1395 {
1396 CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far);
1397 if (wp) {
1398 cs->watchpoint_hit = wp;
1399 return true;
1400 }
1401 break;
1402 }
1403 default:
1404 error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")",
1405 __func__, debug_exit->hsr, env->pc);
1406 }
1407
1408 /* If we are not handling the debug exception it must belong to
1409 * the guest. Let's re-use the existing TCG interrupt code to set
1410 * everything up properly.
1411 */
1412 cs->exception_index = EXCP_BKPT;
1413 env->exception.syndrome = debug_exit->hsr;
1414 env->exception.vaddress = debug_exit->far;
1415 env->exception.target_el = 1;
1416 bql_lock();
1417 arm_cpu_do_interrupt(cs);
1418 bql_unlock();
1419
1420 return false;
1421 }
1422
kvm_arch_handle_exit(CPUState * cs,struct kvm_run * run)1423 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
1424 {
1425 ARMCPU *cpu = ARM_CPU(cs);
1426 int ret = 0;
1427
1428 switch (run->exit_reason) {
1429 case KVM_EXIT_DEBUG:
1430 if (kvm_arm_handle_debug(cpu, &run->debug.arch)) {
1431 ret = EXCP_DEBUG;
1432 } /* otherwise return to guest */
1433 break;
1434 case KVM_EXIT_ARM_NISV:
1435 /* External DABT with no valid iss to decode */
1436 ret = kvm_arm_handle_dabt_nisv(cpu, run->arm_nisv.esr_iss,
1437 run->arm_nisv.fault_ipa);
1438 break;
1439 default:
1440 qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
1441 __func__, run->exit_reason);
1442 break;
1443 }
1444 return ret;
1445 }
1446
kvm_arch_stop_on_emulation_error(CPUState * cs)1447 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
1448 {
1449 return true;
1450 }
1451
kvm_arch_process_async_events(CPUState * cs)1452 int kvm_arch_process_async_events(CPUState *cs)
1453 {
1454 return 0;
1455 }
1456
1457 /**
1458 * kvm_arm_hw_debug_active:
1459 * @cpu: ARMCPU
1460 *
1461 * Return: TRUE if any hardware breakpoints in use.
1462 */
kvm_arm_hw_debug_active(ARMCPU * cpu)1463 static bool kvm_arm_hw_debug_active(ARMCPU *cpu)
1464 {
1465 return ((cur_hw_wps > 0) || (cur_hw_bps > 0));
1466 }
1467
1468 /**
1469 * kvm_arm_copy_hw_debug_data:
1470 * @ptr: kvm_guest_debug_arch structure
1471 *
1472 * Copy the architecture specific debug registers into the
1473 * kvm_guest_debug ioctl structure.
1474 */
kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch * ptr)1475 static void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
1476 {
1477 int i;
1478 memset(ptr, 0, sizeof(struct kvm_guest_debug_arch));
1479
1480 for (i = 0; i < max_hw_wps; i++) {
1481 HWWatchpoint *wp = get_hw_wp(i);
1482 ptr->dbg_wcr[i] = wp->wcr;
1483 ptr->dbg_wvr[i] = wp->wvr;
1484 }
1485 for (i = 0; i < max_hw_bps; i++) {
1486 HWBreakpoint *bp = get_hw_bp(i);
1487 ptr->dbg_bcr[i] = bp->bcr;
1488 ptr->dbg_bvr[i] = bp->bvr;
1489 }
1490 }
1491
kvm_arch_update_guest_debug(CPUState * cs,struct kvm_guest_debug * dbg)1492 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
1493 {
1494 if (kvm_sw_breakpoints_active(cs)) {
1495 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1496 }
1497 if (kvm_arm_hw_debug_active(ARM_CPU(cs))) {
1498 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
1499 kvm_arm_copy_hw_debug_data(&dbg->arch);
1500 }
1501 }
1502
kvm_arch_init_irq_routing(KVMState * s)1503 void kvm_arch_init_irq_routing(KVMState *s)
1504 {
1505 }
1506
kvm_arch_irqchip_create(KVMState * s)1507 int kvm_arch_irqchip_create(KVMState *s)
1508 {
1509 if (kvm_kernel_irqchip_split()) {
1510 error_report("-machine kernel_irqchip=split is not supported on ARM.");
1511 exit(1);
1512 }
1513
1514 /* If we can create the VGIC using the newer device control API, we
1515 * let the device do this when it initializes itself, otherwise we
1516 * fall back to the old API */
1517 return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
1518 }
1519
kvm_arm_vgic_probe(void)1520 int kvm_arm_vgic_probe(void)
1521 {
1522 int val = 0;
1523
1524 if (kvm_create_device(kvm_state,
1525 KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
1526 val |= KVM_ARM_VGIC_V3;
1527 }
1528 if (kvm_create_device(kvm_state,
1529 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
1530 val |= KVM_ARM_VGIC_V2;
1531 }
1532 return val;
1533 }
1534
kvm_arm_set_irq(int cpu,int irqtype,int irq,int level)1535 int kvm_arm_set_irq(int cpu, int irqtype, int irq, int level)
1536 {
1537 int kvm_irq = (irqtype << KVM_ARM_IRQ_TYPE_SHIFT) | irq;
1538 int cpu_idx1 = cpu % 256;
1539 int cpu_idx2 = cpu / 256;
1540
1541 kvm_irq |= (cpu_idx1 << KVM_ARM_IRQ_VCPU_SHIFT) |
1542 (cpu_idx2 << KVM_ARM_IRQ_VCPU2_SHIFT);
1543
1544 return kvm_set_irq(kvm_state, kvm_irq, !!level);
1545 }
1546
kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry * route,uint64_t address,uint32_t data,PCIDevice * dev)1547 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
1548 uint64_t address, uint32_t data, PCIDevice *dev)
1549 {
1550 AddressSpace *as = pci_device_iommu_address_space(dev);
1551 hwaddr xlat, len, doorbell_gpa;
1552 MemoryRegionSection mrs;
1553 MemoryRegion *mr;
1554
1555 if (as == &address_space_memory) {
1556 return 0;
1557 }
1558
1559 /* MSI doorbell address is translated by an IOMMU */
1560
1561 RCU_READ_LOCK_GUARD();
1562
1563 mr = address_space_translate(as, address, &xlat, &len, true,
1564 MEMTXATTRS_UNSPECIFIED);
1565
1566 if (!mr) {
1567 return 1;
1568 }
1569
1570 mrs = memory_region_find(mr, xlat, 1);
1571
1572 if (!mrs.mr) {
1573 return 1;
1574 }
1575
1576 doorbell_gpa = mrs.offset_within_address_space;
1577 memory_region_unref(mrs.mr);
1578
1579 route->u.msi.address_lo = doorbell_gpa;
1580 route->u.msi.address_hi = doorbell_gpa >> 32;
1581
1582 trace_kvm_arm_fixup_msi_route(address, doorbell_gpa);
1583
1584 return 0;
1585 }
1586
kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry * route,int vector,PCIDevice * dev)1587 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
1588 int vector, PCIDevice *dev)
1589 {
1590 return 0;
1591 }
1592
kvm_arch_release_virq_post(int virq)1593 int kvm_arch_release_virq_post(int virq)
1594 {
1595 return 0;
1596 }
1597
kvm_arch_msi_data_to_gsi(uint32_t data)1598 int kvm_arch_msi_data_to_gsi(uint32_t data)
1599 {
1600 return (data - 32) & 0xffff;
1601 }
1602
kvm_arch_get_eager_split_size(Object * obj,Visitor * v,const char * name,void * opaque,Error ** errp)1603 static void kvm_arch_get_eager_split_size(Object *obj, Visitor *v,
1604 const char *name, void *opaque,
1605 Error **errp)
1606 {
1607 KVMState *s = KVM_STATE(obj);
1608 uint64_t value = s->kvm_eager_split_size;
1609
1610 visit_type_size(v, name, &value, errp);
1611 }
1612
kvm_arch_set_eager_split_size(Object * obj,Visitor * v,const char * name,void * opaque,Error ** errp)1613 static void kvm_arch_set_eager_split_size(Object *obj, Visitor *v,
1614 const char *name, void *opaque,
1615 Error **errp)
1616 {
1617 KVMState *s = KVM_STATE(obj);
1618 uint64_t value;
1619
1620 if (s->fd != -1) {
1621 error_setg(errp, "Unable to set early-split-size after KVM has been initialized");
1622 return;
1623 }
1624
1625 if (!visit_type_size(v, name, &value, errp)) {
1626 return;
1627 }
1628
1629 if (value && !is_power_of_2(value)) {
1630 error_setg(errp, "early-split-size must be a power of two");
1631 return;
1632 }
1633
1634 s->kvm_eager_split_size = value;
1635 }
1636
kvm_arch_accel_class_init(ObjectClass * oc)1637 void kvm_arch_accel_class_init(ObjectClass *oc)
1638 {
1639 object_class_property_add(oc, "eager-split-size", "size",
1640 kvm_arch_get_eager_split_size,
1641 kvm_arch_set_eager_split_size, NULL, NULL);
1642
1643 object_class_property_set_description(oc, "eager-split-size",
1644 "Eager Page Split chunk size for hugepages. (default: 0, disabled)");
1645 }
1646
kvm_arch_insert_hw_breakpoint(vaddr addr,vaddr len,int type)1647 int kvm_arch_insert_hw_breakpoint(vaddr addr, vaddr len, int type)
1648 {
1649 switch (type) {
1650 case GDB_BREAKPOINT_HW:
1651 return insert_hw_breakpoint(addr);
1652 break;
1653 case GDB_WATCHPOINT_READ:
1654 case GDB_WATCHPOINT_WRITE:
1655 case GDB_WATCHPOINT_ACCESS:
1656 return insert_hw_watchpoint(addr, len, type);
1657 default:
1658 return -ENOSYS;
1659 }
1660 }
1661
kvm_arch_remove_hw_breakpoint(vaddr addr,vaddr len,int type)1662 int kvm_arch_remove_hw_breakpoint(vaddr addr, vaddr len, int type)
1663 {
1664 switch (type) {
1665 case GDB_BREAKPOINT_HW:
1666 return delete_hw_breakpoint(addr);
1667 case GDB_WATCHPOINT_READ:
1668 case GDB_WATCHPOINT_WRITE:
1669 case GDB_WATCHPOINT_ACCESS:
1670 return delete_hw_watchpoint(addr, len, type);
1671 default:
1672 return -ENOSYS;
1673 }
1674 }
1675
kvm_arch_remove_all_hw_breakpoints(void)1676 void kvm_arch_remove_all_hw_breakpoints(void)
1677 {
1678 if (cur_hw_wps > 0) {
1679 g_array_remove_range(hw_watchpoints, 0, cur_hw_wps);
1680 }
1681 if (cur_hw_bps > 0) {
1682 g_array_remove_range(hw_breakpoints, 0, cur_hw_bps);
1683 }
1684 }
1685
kvm_arm_set_device_attr(ARMCPU * cpu,struct kvm_device_attr * attr,const char * name)1686 static bool kvm_arm_set_device_attr(ARMCPU *cpu, struct kvm_device_attr *attr,
1687 const char *name)
1688 {
1689 int err;
1690
1691 err = kvm_vcpu_ioctl(CPU(cpu), KVM_HAS_DEVICE_ATTR, attr);
1692 if (err != 0) {
1693 error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err));
1694 return false;
1695 }
1696
1697 err = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEVICE_ATTR, attr);
1698 if (err != 0) {
1699 error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err));
1700 return false;
1701 }
1702
1703 return true;
1704 }
1705
kvm_arm_pmu_init(ARMCPU * cpu)1706 void kvm_arm_pmu_init(ARMCPU *cpu)
1707 {
1708 struct kvm_device_attr attr = {
1709 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
1710 .attr = KVM_ARM_VCPU_PMU_V3_INIT,
1711 };
1712
1713 if (!cpu->has_pmu) {
1714 return;
1715 }
1716 if (!kvm_arm_set_device_attr(cpu, &attr, "PMU")) {
1717 error_report("failed to init PMU");
1718 abort();
1719 }
1720 }
1721
kvm_arm_pmu_set_irq(ARMCPU * cpu,int irq)1722 void kvm_arm_pmu_set_irq(ARMCPU *cpu, int irq)
1723 {
1724 struct kvm_device_attr attr = {
1725 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
1726 .addr = (intptr_t)&irq,
1727 .attr = KVM_ARM_VCPU_PMU_V3_IRQ,
1728 };
1729
1730 if (!cpu->has_pmu) {
1731 return;
1732 }
1733 if (!kvm_arm_set_device_attr(cpu, &attr, "PMU")) {
1734 error_report("failed to set irq for PMU");
1735 abort();
1736 }
1737 }
1738
kvm_arm_pvtime_init(ARMCPU * cpu,uint64_t ipa)1739 void kvm_arm_pvtime_init(ARMCPU *cpu, uint64_t ipa)
1740 {
1741 struct kvm_device_attr attr = {
1742 .group = KVM_ARM_VCPU_PVTIME_CTRL,
1743 .attr = KVM_ARM_VCPU_PVTIME_IPA,
1744 .addr = (uint64_t)&ipa,
1745 };
1746
1747 if (cpu->kvm_steal_time == ON_OFF_AUTO_OFF) {
1748 return;
1749 }
1750 if (!kvm_arm_set_device_attr(cpu, &attr, "PVTIME IPA")) {
1751 error_report("failed to init PVTIME IPA");
1752 abort();
1753 }
1754 }
1755
kvm_arm_steal_time_finalize(ARMCPU * cpu,Error ** errp)1756 void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp)
1757 {
1758 bool has_steal_time = kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME);
1759
1760 if (cpu->kvm_steal_time == ON_OFF_AUTO_AUTO) {
1761 if (!has_steal_time || !arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
1762 cpu->kvm_steal_time = ON_OFF_AUTO_OFF;
1763 } else {
1764 cpu->kvm_steal_time = ON_OFF_AUTO_ON;
1765 }
1766 } else if (cpu->kvm_steal_time == ON_OFF_AUTO_ON) {
1767 if (!has_steal_time) {
1768 error_setg(errp, "'kvm-steal-time' cannot be enabled "
1769 "on this host");
1770 return;
1771 } else if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
1772 /*
1773 * DEN0057A chapter 2 says "This specification only covers
1774 * systems in which the Execution state of the hypervisor
1775 * as well as EL1 of virtual machines is AArch64.". And,
1776 * to ensure that, the smc/hvc calls are only specified as
1777 * smc64/hvc64.
1778 */
1779 error_setg(errp, "'kvm-steal-time' cannot be enabled "
1780 "for AArch32 guests");
1781 return;
1782 }
1783 }
1784 }
1785
kvm_arm_aarch32_supported(void)1786 bool kvm_arm_aarch32_supported(void)
1787 {
1788 return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT);
1789 }
1790
kvm_arm_sve_supported(void)1791 bool kvm_arm_sve_supported(void)
1792 {
1793 return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE);
1794 }
1795
1796 QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1);
1797
kvm_arm_sve_get_vls(ARMCPU * cpu)1798 uint32_t kvm_arm_sve_get_vls(ARMCPU *cpu)
1799 {
1800 /* Only call this function if kvm_arm_sve_supported() returns true. */
1801 static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS];
1802 static bool probed;
1803 uint32_t vq = 0;
1804 int i;
1805
1806 /*
1807 * KVM ensures all host CPUs support the same set of vector lengths.
1808 * So we only need to create the scratch VCPUs once and then cache
1809 * the results.
1810 */
1811 if (!probed) {
1812 struct kvm_vcpu_init init = {
1813 .target = -1,
1814 .features[0] = (1 << KVM_ARM_VCPU_SVE),
1815 };
1816 struct kvm_one_reg reg = {
1817 .id = KVM_REG_ARM64_SVE_VLS,
1818 .addr = (uint64_t)&vls[0],
1819 };
1820 int fdarray[3], ret;
1821
1822 probed = true;
1823
1824 if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, &init)) {
1825 error_report("failed to create scratch VCPU with SVE enabled");
1826 abort();
1827 }
1828 ret = ioctl(fdarray[2], KVM_GET_ONE_REG, ®);
1829 kvm_arm_destroy_scratch_host_vcpu(fdarray);
1830 if (ret) {
1831 error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s",
1832 strerror(errno));
1833 abort();
1834 }
1835
1836 for (i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) {
1837 if (vls[i]) {
1838 vq = 64 - clz64(vls[i]) + i * 64;
1839 break;
1840 }
1841 }
1842 if (vq > ARM_MAX_VQ) {
1843 warn_report("KVM supports vector lengths larger than "
1844 "QEMU can enable");
1845 vls[0] &= MAKE_64BIT_MASK(0, ARM_MAX_VQ);
1846 }
1847 }
1848
1849 return vls[0];
1850 }
1851
kvm_arm_sve_set_vls(ARMCPU * cpu)1852 static int kvm_arm_sve_set_vls(ARMCPU *cpu)
1853 {
1854 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq.map };
1855
1856 assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX);
1857
1858 return kvm_set_one_reg(CPU(cpu), KVM_REG_ARM64_SVE_VLS, &vls[0]);
1859 }
1860
1861 #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5
1862
kvm_arch_init_vcpu(CPUState * cs)1863 int kvm_arch_init_vcpu(CPUState *cs)
1864 {
1865 int ret;
1866 uint64_t mpidr;
1867 ARMCPU *cpu = ARM_CPU(cs);
1868 CPUARMState *env = &cpu->env;
1869 uint64_t psciver;
1870
1871 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE ||
1872 !object_dynamic_cast(OBJECT(cpu), TYPE_AARCH64_CPU)) {
1873 error_report("KVM is not supported for this guest CPU type");
1874 return -EINVAL;
1875 }
1876
1877 qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cpu);
1878
1879 /* Determine init features for this CPU */
1880 memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features));
1881 if (cs->start_powered_off) {
1882 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF;
1883 }
1884 if (kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) {
1885 cpu->psci_version = QEMU_PSCI_VERSION_0_2;
1886 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2;
1887 }
1888 if (!arm_feature(env, ARM_FEATURE_AARCH64)) {
1889 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT;
1890 }
1891 if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PMU_V3)) {
1892 cpu->has_pmu = false;
1893 }
1894 if (cpu->has_pmu) {
1895 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
1896 } else {
1897 env->features &= ~(1ULL << ARM_FEATURE_PMU);
1898 }
1899 if (cpu_isar_feature(aa64_sve, cpu)) {
1900 assert(kvm_arm_sve_supported());
1901 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
1902 }
1903 if (cpu_isar_feature(aa64_pauth, cpu)) {
1904 cpu->kvm_init_features[0] |= (1 << KVM_ARM_VCPU_PTRAUTH_ADDRESS |
1905 1 << KVM_ARM_VCPU_PTRAUTH_GENERIC);
1906 }
1907
1908 /* Do KVM_ARM_VCPU_INIT ioctl */
1909 ret = kvm_arm_vcpu_init(cpu);
1910 if (ret) {
1911 return ret;
1912 }
1913
1914 if (cpu_isar_feature(aa64_sve, cpu)) {
1915 ret = kvm_arm_sve_set_vls(cpu);
1916 if (ret) {
1917 return ret;
1918 }
1919 ret = kvm_arm_vcpu_finalize(cpu, KVM_ARM_VCPU_SVE);
1920 if (ret) {
1921 return ret;
1922 }
1923 }
1924
1925 /*
1926 * KVM reports the exact PSCI version it is implementing via a
1927 * special sysreg. If it is present, use its contents to determine
1928 * what to report to the guest in the dtb (it is the PSCI version,
1929 * in the same 15-bits major 16-bits minor format that PSCI_VERSION
1930 * returns).
1931 */
1932 if (!kvm_get_one_reg(cs, KVM_REG_ARM_PSCI_VERSION, &psciver)) {
1933 cpu->psci_version = psciver;
1934 }
1935
1936 /*
1937 * When KVM is in use, PSCI is emulated in-kernel and not by qemu.
1938 * Currently KVM has its own idea about MPIDR assignment, so we
1939 * override our defaults with what we get from KVM.
1940 */
1941 ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr);
1942 if (ret) {
1943 return ret;
1944 }
1945 cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK;
1946
1947 return kvm_arm_init_cpreg_list(cpu);
1948 }
1949
kvm_arch_destroy_vcpu(CPUState * cs)1950 int kvm_arch_destroy_vcpu(CPUState *cs)
1951 {
1952 return 0;
1953 }
1954
1955 /* Callers must hold the iothread mutex lock */
kvm_inject_arm_sea(CPUState * c)1956 static void kvm_inject_arm_sea(CPUState *c)
1957 {
1958 ARMCPU *cpu = ARM_CPU(c);
1959 CPUARMState *env = &cpu->env;
1960 uint32_t esr;
1961 bool same_el;
1962
1963 c->exception_index = EXCP_DATA_ABORT;
1964 env->exception.target_el = 1;
1965
1966 /*
1967 * Set the DFSC to synchronous external abort and set FnV to not valid,
1968 * this will tell guest the FAR_ELx is UNKNOWN for this abort.
1969 */
1970 same_el = arm_current_el(env) == env->exception.target_el;
1971 esr = syn_data_abort_no_iss(same_el, 1, 0, 0, 0, 0, 0x10);
1972
1973 env->exception.syndrome = esr;
1974
1975 arm_cpu_do_interrupt(c);
1976 }
1977
1978 #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \
1979 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
1980
1981 #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \
1982 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
1983
1984 #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \
1985 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
1986
kvm_arch_put_fpsimd(CPUState * cs)1987 static int kvm_arch_put_fpsimd(CPUState *cs)
1988 {
1989 CPUARMState *env = &ARM_CPU(cs)->env;
1990 int i, ret;
1991
1992 for (i = 0; i < 32; i++) {
1993 uint64_t *q = aa64_vfp_qreg(env, i);
1994 #if HOST_BIG_ENDIAN
1995 uint64_t fp_val[2] = { q[1], q[0] };
1996 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]),
1997 fp_val);
1998 #else
1999 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), q);
2000 #endif
2001 if (ret) {
2002 return ret;
2003 }
2004 }
2005
2006 return 0;
2007 }
2008
2009 /*
2010 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
2011 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
2012 * code the slice index to zero for now as it's unlikely we'll need more than
2013 * one slice for quite some time.
2014 */
kvm_arch_put_sve(CPUState * cs)2015 static int kvm_arch_put_sve(CPUState *cs)
2016 {
2017 ARMCPU *cpu = ARM_CPU(cs);
2018 CPUARMState *env = &cpu->env;
2019 uint64_t tmp[ARM_MAX_VQ * 2];
2020 uint64_t *r;
2021 int n, ret;
2022
2023 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
2024 r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2);
2025 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_ZREG(n, 0), r);
2026 if (ret) {
2027 return ret;
2028 }
2029 }
2030
2031 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
2032 r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0],
2033 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
2034 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_PREG(n, 0), r);
2035 if (ret) {
2036 return ret;
2037 }
2038 }
2039
2040 r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0],
2041 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
2042 ret = kvm_set_one_reg(cs, KVM_REG_ARM64_SVE_FFR(0), r);
2043 if (ret) {
2044 return ret;
2045 }
2046
2047 return 0;
2048 }
2049
kvm_arch_put_registers(CPUState * cs,int level)2050 int kvm_arch_put_registers(CPUState *cs, int level)
2051 {
2052 uint64_t val;
2053 uint32_t fpr;
2054 int i, ret;
2055 unsigned int el;
2056
2057 ARMCPU *cpu = ARM_CPU(cs);
2058 CPUARMState *env = &cpu->env;
2059
2060 /* If we are in AArch32 mode then we need to copy the AArch32 regs to the
2061 * AArch64 registers before pushing them out to 64-bit KVM.
2062 */
2063 if (!is_a64(env)) {
2064 aarch64_sync_32_to_64(env);
2065 }
2066
2067 for (i = 0; i < 31; i++) {
2068 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]),
2069 &env->xregs[i]);
2070 if (ret) {
2071 return ret;
2072 }
2073 }
2074
2075 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
2076 * QEMU side we keep the current SP in xregs[31] as well.
2077 */
2078 aarch64_save_sp(env, 1);
2079
2080 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.sp), &env->sp_el[0]);
2081 if (ret) {
2082 return ret;
2083 }
2084
2085 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(sp_el1), &env->sp_el[1]);
2086 if (ret) {
2087 return ret;
2088 }
2089
2090 /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */
2091 if (is_a64(env)) {
2092 val = pstate_read(env);
2093 } else {
2094 val = cpsr_read(env);
2095 }
2096 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pstate), &val);
2097 if (ret) {
2098 return ret;
2099 }
2100
2101 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc);
2102 if (ret) {
2103 return ret;
2104 }
2105
2106 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(elr_el1), &env->elr_el[1]);
2107 if (ret) {
2108 return ret;
2109 }
2110
2111 /* Saved Program State Registers
2112 *
2113 * Before we restore from the banked_spsr[] array we need to
2114 * ensure that any modifications to env->spsr are correctly
2115 * reflected in the banks.
2116 */
2117 el = arm_current_el(env);
2118 if (el > 0 && !is_a64(env)) {
2119 i = bank_number(env->uncached_cpsr & CPSR_M);
2120 env->banked_spsr[i] = env->spsr;
2121 }
2122
2123 /* KVM 0-4 map to QEMU banks 1-5 */
2124 for (i = 0; i < KVM_NR_SPSR; i++) {
2125 ret = kvm_set_one_reg(cs, AARCH64_CORE_REG(spsr[i]),
2126 &env->banked_spsr[i + 1]);
2127 if (ret) {
2128 return ret;
2129 }
2130 }
2131
2132 if (cpu_isar_feature(aa64_sve, cpu)) {
2133 ret = kvm_arch_put_sve(cs);
2134 } else {
2135 ret = kvm_arch_put_fpsimd(cs);
2136 }
2137 if (ret) {
2138 return ret;
2139 }
2140
2141 fpr = vfp_get_fpsr(env);
2142 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpsr), &fpr);
2143 if (ret) {
2144 return ret;
2145 }
2146
2147 fpr = vfp_get_fpcr(env);
2148 ret = kvm_set_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpcr), &fpr);
2149 if (ret) {
2150 return ret;
2151 }
2152
2153 write_cpustate_to_list(cpu, true);
2154
2155 if (!write_list_to_kvmstate(cpu, level)) {
2156 return -EINVAL;
2157 }
2158
2159 /*
2160 * Setting VCPU events should be triggered after syncing the registers
2161 * to avoid overwriting potential changes made by KVM upon calling
2162 * KVM_SET_VCPU_EVENTS ioctl
2163 */
2164 ret = kvm_put_vcpu_events(cpu);
2165 if (ret) {
2166 return ret;
2167 }
2168
2169 return kvm_arm_sync_mpstate_to_kvm(cpu);
2170 }
2171
kvm_arch_get_fpsimd(CPUState * cs)2172 static int kvm_arch_get_fpsimd(CPUState *cs)
2173 {
2174 CPUARMState *env = &ARM_CPU(cs)->env;
2175 int i, ret;
2176
2177 for (i = 0; i < 32; i++) {
2178 uint64_t *q = aa64_vfp_qreg(env, i);
2179 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]), q);
2180 if (ret) {
2181 return ret;
2182 } else {
2183 #if HOST_BIG_ENDIAN
2184 uint64_t t;
2185 t = q[0], q[0] = q[1], q[1] = t;
2186 #endif
2187 }
2188 }
2189
2190 return 0;
2191 }
2192
2193 /*
2194 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
2195 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
2196 * code the slice index to zero for now as it's unlikely we'll need more than
2197 * one slice for quite some time.
2198 */
kvm_arch_get_sve(CPUState * cs)2199 static int kvm_arch_get_sve(CPUState *cs)
2200 {
2201 ARMCPU *cpu = ARM_CPU(cs);
2202 CPUARMState *env = &cpu->env;
2203 uint64_t *r;
2204 int n, ret;
2205
2206 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
2207 r = &env->vfp.zregs[n].d[0];
2208 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_ZREG(n, 0), r);
2209 if (ret) {
2210 return ret;
2211 }
2212 sve_bswap64(r, r, cpu->sve_max_vq * 2);
2213 }
2214
2215 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
2216 r = &env->vfp.pregs[n].p[0];
2217 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_PREG(n, 0), r);
2218 if (ret) {
2219 return ret;
2220 }
2221 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
2222 }
2223
2224 r = &env->vfp.pregs[FFR_PRED_NUM].p[0];
2225 ret = kvm_get_one_reg(cs, KVM_REG_ARM64_SVE_FFR(0), r);
2226 if (ret) {
2227 return ret;
2228 }
2229 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
2230
2231 return 0;
2232 }
2233
kvm_arch_get_registers(CPUState * cs)2234 int kvm_arch_get_registers(CPUState *cs)
2235 {
2236 uint64_t val;
2237 unsigned int el;
2238 uint32_t fpr;
2239 int i, ret;
2240
2241 ARMCPU *cpu = ARM_CPU(cs);
2242 CPUARMState *env = &cpu->env;
2243
2244 for (i = 0; i < 31; i++) {
2245 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.regs[i]),
2246 &env->xregs[i]);
2247 if (ret) {
2248 return ret;
2249 }
2250 }
2251
2252 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.sp), &env->sp_el[0]);
2253 if (ret) {
2254 return ret;
2255 }
2256
2257 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(sp_el1), &env->sp_el[1]);
2258 if (ret) {
2259 return ret;
2260 }
2261
2262 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.pstate), &val);
2263 if (ret) {
2264 return ret;
2265 }
2266
2267 env->aarch64 = ((val & PSTATE_nRW) == 0);
2268 if (is_a64(env)) {
2269 pstate_write(env, val);
2270 } else {
2271 cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
2272 }
2273
2274 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
2275 * QEMU side we keep the current SP in xregs[31] as well.
2276 */
2277 aarch64_restore_sp(env, 1);
2278
2279 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(regs.pc), &env->pc);
2280 if (ret) {
2281 return ret;
2282 }
2283
2284 /* If we are in AArch32 mode then we need to sync the AArch32 regs with the
2285 * incoming AArch64 regs received from 64-bit KVM.
2286 * We must perform this after all of the registers have been acquired from
2287 * the kernel.
2288 */
2289 if (!is_a64(env)) {
2290 aarch64_sync_64_to_32(env);
2291 }
2292
2293 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(elr_el1), &env->elr_el[1]);
2294 if (ret) {
2295 return ret;
2296 }
2297
2298 /* Fetch the SPSR registers
2299 *
2300 * KVM SPSRs 0-4 map to QEMU banks 1-5
2301 */
2302 for (i = 0; i < KVM_NR_SPSR; i++) {
2303 ret = kvm_get_one_reg(cs, AARCH64_CORE_REG(spsr[i]),
2304 &env->banked_spsr[i + 1]);
2305 if (ret) {
2306 return ret;
2307 }
2308 }
2309
2310 el = arm_current_el(env);
2311 if (el > 0 && !is_a64(env)) {
2312 i = bank_number(env->uncached_cpsr & CPSR_M);
2313 env->spsr = env->banked_spsr[i];
2314 }
2315
2316 if (cpu_isar_feature(aa64_sve, cpu)) {
2317 ret = kvm_arch_get_sve(cs);
2318 } else {
2319 ret = kvm_arch_get_fpsimd(cs);
2320 }
2321 if (ret) {
2322 return ret;
2323 }
2324
2325 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpsr), &fpr);
2326 if (ret) {
2327 return ret;
2328 }
2329 vfp_set_fpsr(env, fpr);
2330
2331 ret = kvm_get_one_reg(cs, AARCH64_SIMD_CTRL_REG(fp_regs.fpcr), &fpr);
2332 if (ret) {
2333 return ret;
2334 }
2335 vfp_set_fpcr(env, fpr);
2336
2337 ret = kvm_get_vcpu_events(cpu);
2338 if (ret) {
2339 return ret;
2340 }
2341
2342 if (!write_kvmstate_to_list(cpu)) {
2343 return -EINVAL;
2344 }
2345 /* Note that it's OK to have registers which aren't in CPUState,
2346 * so we can ignore a failure return here.
2347 */
2348 write_list_to_cpustate(cpu);
2349
2350 ret = kvm_arm_sync_mpstate_to_qemu(cpu);
2351
2352 /* TODO: other registers */
2353 return ret;
2354 }
2355
kvm_arch_on_sigbus_vcpu(CPUState * c,int code,void * addr)2356 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
2357 {
2358 ram_addr_t ram_addr;
2359 hwaddr paddr;
2360
2361 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO);
2362
2363 if (acpi_ghes_present() && addr) {
2364 ram_addr = qemu_ram_addr_from_host(addr);
2365 if (ram_addr != RAM_ADDR_INVALID &&
2366 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
2367 kvm_hwpoison_page_add(ram_addr);
2368 /*
2369 * If this is a BUS_MCEERR_AR, we know we have been called
2370 * synchronously from the vCPU thread, so we can easily
2371 * synchronize the state and inject an error.
2372 *
2373 * TODO: we currently don't tell the guest at all about
2374 * BUS_MCEERR_AO. In that case we might either be being
2375 * called synchronously from the vCPU thread, or a bit
2376 * later from the main thread, so doing the injection of
2377 * the error would be more complicated.
2378 */
2379 if (code == BUS_MCEERR_AR) {
2380 kvm_cpu_synchronize_state(c);
2381 if (!acpi_ghes_record_errors(ACPI_HEST_SRC_ID_SEA, paddr)) {
2382 kvm_inject_arm_sea(c);
2383 } else {
2384 error_report("failed to record the error");
2385 abort();
2386 }
2387 }
2388 return;
2389 }
2390 if (code == BUS_MCEERR_AO) {
2391 error_report("Hardware memory error at addr %p for memory used by "
2392 "QEMU itself instead of guest system!", addr);
2393 }
2394 }
2395
2396 if (code == BUS_MCEERR_AR) {
2397 error_report("Hardware memory error!");
2398 exit(1);
2399 }
2400 }
2401
2402 /* C6.6.29 BRK instruction */
2403 static const uint32_t brk_insn = 0xd4200000;
2404
kvm_arch_insert_sw_breakpoint(CPUState * cs,struct kvm_sw_breakpoint * bp)2405 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
2406 {
2407 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
2408 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) {
2409 return -EINVAL;
2410 }
2411 return 0;
2412 }
2413
kvm_arch_remove_sw_breakpoint(CPUState * cs,struct kvm_sw_breakpoint * bp)2414 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
2415 {
2416 static uint32_t brk;
2417
2418 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) ||
2419 brk != brk_insn ||
2420 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) {
2421 return -EINVAL;
2422 }
2423 return 0;
2424 }
2425