1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
3 
4 #include <linux/jump_label.h>
5 #include <linux/uaccess.h>
6 #include <linux/export.h>
7 #include <linux/string.h>
8 #include <linux/types.h>
9 
10 #include <asm/mce.h>
11 
12 #ifdef CONFIG_X86_MCE
13 static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
14 
enable_copy_mc_fragile(void)15 void enable_copy_mc_fragile(void)
16 {
17 	static_branch_inc(&copy_mc_fragile_key);
18 }
19 #define copy_mc_fragile_enabled (static_branch_unlikely(&copy_mc_fragile_key))
20 
21 /*
22  * Similar to copy_user_handle_tail, probe for the write fault point, or
23  * source exception point.
24  */
25 __visible notrace unsigned long
copy_mc_fragile_handle_tail(char * to,char * from,unsigned len)26 copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
27 {
28 	for (; len; --len, to++, from++)
29 		if (copy_mc_fragile(to, from, 1))
30 			break;
31 	return len;
32 }
33 #else
34 /*
35  * No point in doing careful copying, or consulting a static key when
36  * there is no #MC handler in the CONFIG_X86_MCE=n case.
37  */
enable_copy_mc_fragile(void)38 void enable_copy_mc_fragile(void)
39 {
40 }
41 #define copy_mc_fragile_enabled (0)
42 #endif
43 
44 unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
45 
46 /**
47  * copy_mc_to_kernel - memory copy that handles source exceptions
48  *
49  * @dst:	destination address
50  * @src:	source address
51  * @len:	number of bytes to copy
52  *
53  * Call into the 'fragile' version on systems that benefit from avoiding
54  * corner case poison consumption scenarios, For example, accessing
55  * poison across 2 cachelines with a single instruction. Almost all
56  * other uses case can use copy_mc_enhanced_fast_string() for a fast
57  * recoverable copy, or fallback to plain memcpy.
58  *
59  * Return 0 for success, or number of bytes not copied if there was an
60  * exception.
61  */
copy_mc_to_kernel(void * dst,const void * src,unsigned len)62 unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
63 {
64 	if (copy_mc_fragile_enabled)
65 		return copy_mc_fragile(dst, src, len);
66 	if (static_cpu_has(X86_FEATURE_ERMS))
67 		return copy_mc_enhanced_fast_string(dst, src, len);
68 	memcpy(dst, src, len);
69 	return 0;
70 }
71 EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
72 
copy_mc_to_user(void * dst,const void * src,unsigned len)73 unsigned long __must_check copy_mc_to_user(void *dst, const void *src, unsigned len)
74 {
75 	unsigned long ret;
76 
77 	if (copy_mc_fragile_enabled) {
78 		__uaccess_begin();
79 		ret = copy_mc_fragile(dst, src, len);
80 		__uaccess_end();
81 		return ret;
82 	}
83 
84 	if (static_cpu_has(X86_FEATURE_ERMS)) {
85 		__uaccess_begin();
86 		ret = copy_mc_enhanced_fast_string(dst, src, len);
87 		__uaccess_end();
88 		return ret;
89 	}
90 
91 	return copy_user_generic(dst, src, len);
92 }
93