1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * AMD Encrypted Register State Support
4 *
5 * Author: Joerg Roedel <jroedel@suse.de>
6 *
7 * This file is not compiled stand-alone. It contains code shared
8 * between the pre-decompression boot code and the running Linux kernel
9 * and is included directly into both code-bases.
10 */
11
12 #ifndef __BOOT_COMPRESSED
13 #define error(v) pr_err(v)
14 #define has_cpuflag(f) boot_cpu_has(f)
15 #endif
16
sev_es_check_cpu_features(void)17 static bool __init sev_es_check_cpu_features(void)
18 {
19 if (!has_cpuflag(X86_FEATURE_RDRAND)) {
20 error("RDRAND instruction not supported - no trusted source of randomness available\n");
21 return false;
22 }
23
24 return true;
25 }
26
sev_es_terminate(unsigned int reason)27 static void __noreturn sev_es_terminate(unsigned int reason)
28 {
29 u64 val = GHCB_MSR_TERM_REQ;
30
31 /*
32 * Tell the hypervisor what went wrong - only reason-set 0 is
33 * currently supported.
34 */
35 val |= GHCB_SEV_TERM_REASON(0, reason);
36
37 /* Request Guest Termination from Hypvervisor */
38 sev_es_wr_ghcb_msr(val);
39 VMGEXIT();
40
41 while (true)
42 asm volatile("hlt\n" : : : "memory");
43 }
44
sev_es_negotiate_protocol(void)45 static bool sev_es_negotiate_protocol(void)
46 {
47 u64 val;
48
49 /* Do the GHCB protocol version negotiation */
50 sev_es_wr_ghcb_msr(GHCB_MSR_SEV_INFO_REQ);
51 VMGEXIT();
52 val = sev_es_rd_ghcb_msr();
53
54 if (GHCB_MSR_INFO(val) != GHCB_MSR_SEV_INFO_RESP)
55 return false;
56
57 if (GHCB_MSR_PROTO_MAX(val) < GHCB_PROTO_OUR ||
58 GHCB_MSR_PROTO_MIN(val) > GHCB_PROTO_OUR)
59 return false;
60
61 return true;
62 }
63
vc_ghcb_invalidate(struct ghcb * ghcb)64 static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
65 {
66 memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
67 }
68
vc_decoding_needed(unsigned long exit_code)69 static bool vc_decoding_needed(unsigned long exit_code)
70 {
71 /* Exceptions don't require to decode the instruction */
72 return !(exit_code >= SVM_EXIT_EXCP_BASE &&
73 exit_code <= SVM_EXIT_LAST_EXCP);
74 }
75
vc_init_em_ctxt(struct es_em_ctxt * ctxt,struct pt_regs * regs,unsigned long exit_code)76 static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
77 struct pt_regs *regs,
78 unsigned long exit_code)
79 {
80 enum es_result ret = ES_OK;
81
82 memset(ctxt, 0, sizeof(*ctxt));
83 ctxt->regs = regs;
84
85 if (vc_decoding_needed(exit_code))
86 ret = vc_decode_insn(ctxt);
87
88 return ret;
89 }
90
vc_finish_insn(struct es_em_ctxt * ctxt)91 static void vc_finish_insn(struct es_em_ctxt *ctxt)
92 {
93 ctxt->regs->ip += ctxt->insn.length;
94 }
95
sev_es_ghcb_hv_call(struct ghcb * ghcb,struct es_em_ctxt * ctxt,u64 exit_code,u64 exit_info_1,u64 exit_info_2)96 static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
97 struct es_em_ctxt *ctxt,
98 u64 exit_code, u64 exit_info_1,
99 u64 exit_info_2)
100 {
101 enum es_result ret;
102
103 /* Fill in protocol and format specifiers */
104 ghcb->protocol_version = GHCB_PROTOCOL_MAX;
105 ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
106
107 ghcb_set_sw_exit_code(ghcb, exit_code);
108 ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
109 ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
110
111 sev_es_wr_ghcb_msr(__pa(ghcb));
112 VMGEXIT();
113
114 if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
115 u64 info = ghcb->save.sw_exit_info_2;
116 unsigned long v;
117
118 info = ghcb->save.sw_exit_info_2;
119 v = info & SVM_EVTINJ_VEC_MASK;
120
121 /* Check if exception information from hypervisor is sane. */
122 if ((info & SVM_EVTINJ_VALID) &&
123 ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
124 ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
125 ctxt->fi.vector = v;
126 if (info & SVM_EVTINJ_VALID_ERR)
127 ctxt->fi.error_code = info >> 32;
128 ret = ES_EXCEPTION;
129 } else {
130 ret = ES_VMM_ERROR;
131 }
132 } else {
133 ret = ES_OK;
134 }
135
136 return ret;
137 }
138
139 /*
140 * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
141 * page yet, so it only supports the MSR based communication with the
142 * hypervisor and only the CPUID exit-code.
143 */
do_vc_no_ghcb(struct pt_regs * regs,unsigned long exit_code)144 void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
145 {
146 unsigned int fn = lower_bits(regs->ax, 32);
147 unsigned long val;
148
149 /* Only CPUID is supported via MSR protocol */
150 if (exit_code != SVM_EXIT_CPUID)
151 goto fail;
152
153 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
154 VMGEXIT();
155 val = sev_es_rd_ghcb_msr();
156 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
157 goto fail;
158 regs->ax = val >> 32;
159
160 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
161 VMGEXIT();
162 val = sev_es_rd_ghcb_msr();
163 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
164 goto fail;
165 regs->bx = val >> 32;
166
167 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
168 VMGEXIT();
169 val = sev_es_rd_ghcb_msr();
170 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
171 goto fail;
172 regs->cx = val >> 32;
173
174 sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
175 VMGEXIT();
176 val = sev_es_rd_ghcb_msr();
177 if (GHCB_RESP_CODE(val) != GHCB_MSR_CPUID_RESP)
178 goto fail;
179 regs->dx = val >> 32;
180
181 /*
182 * This is a VC handler and the #VC is only raised when SEV-ES is
183 * active, which means SEV must be active too. Do sanity checks on the
184 * CPUID results to make sure the hypervisor does not trick the kernel
185 * into the no-sev path. This could map sensitive data unencrypted and
186 * make it accessible to the hypervisor.
187 *
188 * In particular, check for:
189 * - Availability of CPUID leaf 0x8000001f
190 * - SEV CPUID bit.
191 *
192 * The hypervisor might still report the wrong C-bit position, but this
193 * can't be checked here.
194 */
195
196 if (fn == 0x80000000 && (regs->ax < 0x8000001f))
197 /* SEV leaf check */
198 goto fail;
199 else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
200 /* SEV bit */
201 goto fail;
202
203 /* Skip over the CPUID two-byte opcode */
204 regs->ip += 2;
205
206 return;
207
208 fail:
209 /* Terminate the guest */
210 sev_es_terminate(GHCB_SEV_ES_REASON_GENERAL_REQUEST);
211 }
212
vc_insn_string_read(struct es_em_ctxt * ctxt,void * src,char * buf,unsigned int data_size,unsigned int count,bool backwards)213 static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
214 void *src, char *buf,
215 unsigned int data_size,
216 unsigned int count,
217 bool backwards)
218 {
219 int i, b = backwards ? -1 : 1;
220 enum es_result ret = ES_OK;
221
222 for (i = 0; i < count; i++) {
223 void *s = src + (i * data_size * b);
224 char *d = buf + (i * data_size);
225
226 ret = vc_read_mem(ctxt, s, d, data_size);
227 if (ret != ES_OK)
228 break;
229 }
230
231 return ret;
232 }
233
vc_insn_string_write(struct es_em_ctxt * ctxt,void * dst,char * buf,unsigned int data_size,unsigned int count,bool backwards)234 static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
235 void *dst, char *buf,
236 unsigned int data_size,
237 unsigned int count,
238 bool backwards)
239 {
240 int i, s = backwards ? -1 : 1;
241 enum es_result ret = ES_OK;
242
243 for (i = 0; i < count; i++) {
244 void *d = dst + (i * data_size * s);
245 char *b = buf + (i * data_size);
246
247 ret = vc_write_mem(ctxt, d, b, data_size);
248 if (ret != ES_OK)
249 break;
250 }
251
252 return ret;
253 }
254
255 #define IOIO_TYPE_STR BIT(2)
256 #define IOIO_TYPE_IN 1
257 #define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
258 #define IOIO_TYPE_OUT 0
259 #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
260
261 #define IOIO_REP BIT(3)
262
263 #define IOIO_ADDR_64 BIT(9)
264 #define IOIO_ADDR_32 BIT(8)
265 #define IOIO_ADDR_16 BIT(7)
266
267 #define IOIO_DATA_32 BIT(6)
268 #define IOIO_DATA_16 BIT(5)
269 #define IOIO_DATA_8 BIT(4)
270
271 #define IOIO_SEG_ES (0 << 10)
272 #define IOIO_SEG_DS (3 << 10)
273
vc_ioio_exitinfo(struct es_em_ctxt * ctxt,u64 * exitinfo)274 static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
275 {
276 struct insn *insn = &ctxt->insn;
277 *exitinfo = 0;
278
279 switch (insn->opcode.bytes[0]) {
280 /* INS opcodes */
281 case 0x6c:
282 case 0x6d:
283 *exitinfo |= IOIO_TYPE_INS;
284 *exitinfo |= IOIO_SEG_ES;
285 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
286 break;
287
288 /* OUTS opcodes */
289 case 0x6e:
290 case 0x6f:
291 *exitinfo |= IOIO_TYPE_OUTS;
292 *exitinfo |= IOIO_SEG_DS;
293 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
294 break;
295
296 /* IN immediate opcodes */
297 case 0xe4:
298 case 0xe5:
299 *exitinfo |= IOIO_TYPE_IN;
300 *exitinfo |= (u8)insn->immediate.value << 16;
301 break;
302
303 /* OUT immediate opcodes */
304 case 0xe6:
305 case 0xe7:
306 *exitinfo |= IOIO_TYPE_OUT;
307 *exitinfo |= (u8)insn->immediate.value << 16;
308 break;
309
310 /* IN register opcodes */
311 case 0xec:
312 case 0xed:
313 *exitinfo |= IOIO_TYPE_IN;
314 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
315 break;
316
317 /* OUT register opcodes */
318 case 0xee:
319 case 0xef:
320 *exitinfo |= IOIO_TYPE_OUT;
321 *exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
322 break;
323
324 default:
325 return ES_DECODE_FAILED;
326 }
327
328 switch (insn->opcode.bytes[0]) {
329 case 0x6c:
330 case 0x6e:
331 case 0xe4:
332 case 0xe6:
333 case 0xec:
334 case 0xee:
335 /* Single byte opcodes */
336 *exitinfo |= IOIO_DATA_8;
337 break;
338 default:
339 /* Length determined by instruction parsing */
340 *exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
341 : IOIO_DATA_32;
342 }
343 switch (insn->addr_bytes) {
344 case 2:
345 *exitinfo |= IOIO_ADDR_16;
346 break;
347 case 4:
348 *exitinfo |= IOIO_ADDR_32;
349 break;
350 case 8:
351 *exitinfo |= IOIO_ADDR_64;
352 break;
353 }
354
355 if (insn_has_rep_prefix(insn))
356 *exitinfo |= IOIO_REP;
357
358 return ES_OK;
359 }
360
vc_handle_ioio(struct ghcb * ghcb,struct es_em_ctxt * ctxt)361 static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
362 {
363 struct pt_regs *regs = ctxt->regs;
364 u64 exit_info_1, exit_info_2;
365 enum es_result ret;
366
367 ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
368 if (ret != ES_OK)
369 return ret;
370
371 if (exit_info_1 & IOIO_TYPE_STR) {
372
373 /* (REP) INS/OUTS */
374
375 bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
376 unsigned int io_bytes, exit_bytes;
377 unsigned int ghcb_count, op_count;
378 unsigned long es_base;
379 u64 sw_scratch;
380
381 /*
382 * For the string variants with rep prefix the amount of in/out
383 * operations per #VC exception is limited so that the kernel
384 * has a chance to take interrupts and re-schedule while the
385 * instruction is emulated.
386 */
387 io_bytes = (exit_info_1 >> 4) & 0x7;
388 ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
389
390 op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
391 exit_info_2 = min(op_count, ghcb_count);
392 exit_bytes = exit_info_2 * io_bytes;
393
394 es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
395
396 /* Read bytes of OUTS into the shared buffer */
397 if (!(exit_info_1 & IOIO_TYPE_IN)) {
398 ret = vc_insn_string_read(ctxt,
399 (void *)(es_base + regs->si),
400 ghcb->shared_buffer, io_bytes,
401 exit_info_2, df);
402 if (ret)
403 return ret;
404 }
405
406 /*
407 * Issue an VMGEXIT to the HV to consume the bytes from the
408 * shared buffer or to have it write them into the shared buffer
409 * depending on the instruction: OUTS or INS.
410 */
411 sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
412 ghcb_set_sw_scratch(ghcb, sw_scratch);
413 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
414 exit_info_1, exit_info_2);
415 if (ret != ES_OK)
416 return ret;
417
418 /* Read bytes from shared buffer into the guest's destination. */
419 if (exit_info_1 & IOIO_TYPE_IN) {
420 ret = vc_insn_string_write(ctxt,
421 (void *)(es_base + regs->di),
422 ghcb->shared_buffer, io_bytes,
423 exit_info_2, df);
424 if (ret)
425 return ret;
426
427 if (df)
428 regs->di -= exit_bytes;
429 else
430 regs->di += exit_bytes;
431 } else {
432 if (df)
433 regs->si -= exit_bytes;
434 else
435 regs->si += exit_bytes;
436 }
437
438 if (exit_info_1 & IOIO_REP)
439 regs->cx -= exit_info_2;
440
441 ret = regs->cx ? ES_RETRY : ES_OK;
442
443 } else {
444
445 /* IN/OUT into/from rAX */
446
447 int bits = (exit_info_1 & 0x70) >> 1;
448 u64 rax = 0;
449
450 if (!(exit_info_1 & IOIO_TYPE_IN))
451 rax = lower_bits(regs->ax, bits);
452
453 ghcb_set_rax(ghcb, rax);
454
455 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
456 if (ret != ES_OK)
457 return ret;
458
459 if (exit_info_1 & IOIO_TYPE_IN) {
460 if (!ghcb_rax_is_valid(ghcb))
461 return ES_VMM_ERROR;
462 regs->ax = lower_bits(ghcb->save.rax, bits);
463 }
464 }
465
466 return ret;
467 }
468
vc_handle_cpuid(struct ghcb * ghcb,struct es_em_ctxt * ctxt)469 static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
470 struct es_em_ctxt *ctxt)
471 {
472 struct pt_regs *regs = ctxt->regs;
473 u32 cr4 = native_read_cr4();
474 enum es_result ret;
475
476 ghcb_set_rax(ghcb, regs->ax);
477 ghcb_set_rcx(ghcb, regs->cx);
478
479 if (cr4 & X86_CR4_OSXSAVE)
480 /* Safe to read xcr0 */
481 ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
482 else
483 /* xgetbv will cause #GP - use reset value for xcr0 */
484 ghcb_set_xcr0(ghcb, 1);
485
486 ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
487 if (ret != ES_OK)
488 return ret;
489
490 if (!(ghcb_rax_is_valid(ghcb) &&
491 ghcb_rbx_is_valid(ghcb) &&
492 ghcb_rcx_is_valid(ghcb) &&
493 ghcb_rdx_is_valid(ghcb)))
494 return ES_VMM_ERROR;
495
496 regs->ax = ghcb->save.rax;
497 regs->bx = ghcb->save.rbx;
498 regs->cx = ghcb->save.rcx;
499 regs->dx = ghcb->save.rdx;
500
501 return ES_OK;
502 }
503
vc_handle_rdtsc(struct ghcb * ghcb,struct es_em_ctxt * ctxt,unsigned long exit_code)504 static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
505 struct es_em_ctxt *ctxt,
506 unsigned long exit_code)
507 {
508 bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
509 enum es_result ret;
510
511 ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
512 if (ret != ES_OK)
513 return ret;
514
515 if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
516 (!rdtscp || ghcb_rcx_is_valid(ghcb))))
517 return ES_VMM_ERROR;
518
519 ctxt->regs->ax = ghcb->save.rax;
520 ctxt->regs->dx = ghcb->save.rdx;
521 if (rdtscp)
522 ctxt->regs->cx = ghcb->save.rcx;
523
524 return ES_OK;
525 }
526