1/* SPDX-License-Identifier: GPL-2.0-only */
2/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
3
4#include <linux/linkage.h>
5#include <asm/asm.h>
6
7#ifndef CONFIG_UML
8
9#ifdef CONFIG_X86_MCE
10
11/*
12 * copy_mc_fragile - copy memory with indication if an exception / fault happened
13 *
14 * The 'fragile' version is opted into by platform quirks and takes
15 * pains to avoid unrecoverable corner cases like 'fast-string'
16 * instruction sequences, and consuming poison across a cacheline
17 * boundary. The non-fragile version is equivalent to memcpy()
18 * regardless of CPU machine-check-recovery capability.
19 */
20SYM_FUNC_START(copy_mc_fragile)
21	cmpl $8, %edx
22	/* Less than 8 bytes? Go to byte copy loop */
23	jb .L_no_whole_words
24
25	/* Check for bad alignment of source */
26	testl $7, %esi
27	/* Already aligned */
28	jz .L_8byte_aligned
29
30	/* Copy one byte at a time until source is 8-byte aligned */
31	movl %esi, %ecx
32	andl $7, %ecx
33	subl $8, %ecx
34	negl %ecx
35	subl %ecx, %edx
36.L_read_leading_bytes:
37	movb (%rsi), %al
38.L_write_leading_bytes:
39	movb %al, (%rdi)
40	incq %rsi
41	incq %rdi
42	decl %ecx
43	jnz .L_read_leading_bytes
44
45.L_8byte_aligned:
46	movl %edx, %ecx
47	andl $7, %edx
48	shrl $3, %ecx
49	jz .L_no_whole_words
50
51.L_read_words:
52	movq (%rsi), %r8
53.L_write_words:
54	movq %r8, (%rdi)
55	addq $8, %rsi
56	addq $8, %rdi
57	decl %ecx
58	jnz .L_read_words
59
60	/* Any trailing bytes? */
61.L_no_whole_words:
62	andl %edx, %edx
63	jz .L_done_memcpy_trap
64
65	/* Copy trailing bytes */
66	movl %edx, %ecx
67.L_read_trailing_bytes:
68	movb (%rsi), %al
69.L_write_trailing_bytes:
70	movb %al, (%rdi)
71	incq %rsi
72	incq %rdi
73	decl %ecx
74	jnz .L_read_trailing_bytes
75
76	/* Copy successful. Return zero */
77.L_done_memcpy_trap:
78	xorl %eax, %eax
79.L_done:
80	ret
81SYM_FUNC_END(copy_mc_fragile)
82
83	.section .fixup, "ax"
84	/*
85	 * Return number of bytes not copied for any failure. Note that
86	 * there is no "tail" handling since the source buffer is 8-byte
87	 * aligned and poison is cacheline aligned.
88	 */
89.E_read_words:
90	shll	$3, %ecx
91.E_leading_bytes:
92	addl	%edx, %ecx
93.E_trailing_bytes:
94	mov	%ecx, %eax
95	jmp	.L_done
96
97	/*
98	 * For write fault handling, given the destination is unaligned,
99	 * we handle faults on multi-byte writes with a byte-by-byte
100	 * copy up to the write-protected page.
101	 */
102.E_write_words:
103	shll	$3, %ecx
104	addl	%edx, %ecx
105	movl	%ecx, %edx
106	jmp copy_mc_fragile_handle_tail
107
108	.previous
109
110	_ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
111	_ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
112	_ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
113	_ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
114	_ASM_EXTABLE(.L_write_words, .E_write_words)
115	_ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)
116#endif /* CONFIG_X86_MCE */
117
118/*
119 * copy_mc_enhanced_fast_string - memory copy with exception handling
120 *
121 * Fast string copy + fault / exception handling. If the CPU does
122 * support machine check exception recovery, but does not support
123 * recovering from fast-string exceptions then this CPU needs to be
124 * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any
125 * machine check recovery support this version should be no slower than
126 * standard memcpy.
127 */
128SYM_FUNC_START(copy_mc_enhanced_fast_string)
129	movq %rdi, %rax
130	movq %rdx, %rcx
131.L_copy:
132	rep movsb
133	/* Copy successful. Return zero */
134	xorl %eax, %eax
135	ret
136SYM_FUNC_END(copy_mc_enhanced_fast_string)
137
138	.section .fixup, "ax"
139.E_copy:
140	/*
141	 * On fault %rcx is updated such that the copy instruction could
142	 * optionally be restarted at the fault position, i.e. it
143	 * contains 'bytes remaining'. A non-zero return indicates error
144	 * to copy_mc_generic() users, or indicate short transfers to
145	 * user-copy routines.
146	 */
147	movq %rcx, %rax
148	ret
149
150	.previous
151
152	_ASM_EXTABLE_FAULT(.L_copy, .E_copy)
153#endif /* !CONFIG_UML */
154