1 /* $NetBSD: lock.h,v 1.32 2015/02/25 13:52:42 joerg Exp $ */
2
3 /*-
4 * Copyright (c) 2000, 2001 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Machine-dependent spin lock operations.
34 *
35 * NOTE: The SWP insn used here is available only on ARM architecture
36 * version 3 and later (as well as 2a). What we are going to do is
37 * expect that the kernel will trap and emulate the insn. That will
38 * be slow, but give us the atomicity that we need.
39 */
40
41 #ifndef _ARM_LOCK_H_
42 #define _ARM_LOCK_H_
43
44 static __inline int
__SIMPLELOCK_LOCKED_P(__cpu_simple_lock_t * __ptr)45 __SIMPLELOCK_LOCKED_P(__cpu_simple_lock_t *__ptr)
46 {
47 return *__ptr == __SIMPLELOCK_LOCKED;
48 }
49
50 static __inline int
__SIMPLELOCK_UNLOCKED_P(__cpu_simple_lock_t * __ptr)51 __SIMPLELOCK_UNLOCKED_P(__cpu_simple_lock_t *__ptr)
52 {
53 return *__ptr == __SIMPLELOCK_UNLOCKED;
54 }
55
56 static __inline void
__cpu_simple_lock_clear(__cpu_simple_lock_t * __ptr)57 __cpu_simple_lock_clear(__cpu_simple_lock_t *__ptr)
58 {
59 *__ptr = __SIMPLELOCK_UNLOCKED;
60 }
61
62 static __inline void
__cpu_simple_lock_set(__cpu_simple_lock_t * __ptr)63 __cpu_simple_lock_set(__cpu_simple_lock_t *__ptr)
64 {
65 *__ptr = __SIMPLELOCK_LOCKED;
66 }
67
68 #ifdef _KERNEL
69 #include <arm/cpufunc.h>
70
71 #define mb_read drain_writebuf /* in cpufunc.h */
72 #define mb_write drain_writebuf /* in cpufunc.h */
73 #define mb_memory drain_writebuf /* in cpufunc.h */
74 #endif
75
76 #ifdef _ARM_ARCH_6
77 static __inline unsigned int
__arm_load_exclusive(__cpu_simple_lock_t * __alp)78 __arm_load_exclusive(__cpu_simple_lock_t *__alp)
79 {
80 unsigned int __rv;
81 if (/*CONSTCOND*/sizeof(*__alp) == 1) {
82 __asm __volatile("ldrexb\t%0,[%1]" : "=r"(__rv) : "r"(__alp));
83 } else {
84 __asm __volatile("ldrex\t%0,[%1]" : "=r"(__rv) : "r"(__alp));
85 }
86 return __rv;
87 }
88
89 /* returns 0 on success and 1 on failure */
90 static __inline unsigned int
__arm_store_exclusive(__cpu_simple_lock_t * __alp,unsigned int __val)91 __arm_store_exclusive(__cpu_simple_lock_t *__alp, unsigned int __val)
92 {
93 unsigned int __rv;
94 if (/*CONSTCOND*/sizeof(*__alp) == 1) {
95 __asm __volatile("strexb\t%0,%1,[%2]"
96 : "=&r"(__rv) : "r"(__val), "r"(__alp) : "cc", "memory");
97 } else {
98 __asm __volatile("strex\t%0,%1,[%2]"
99 : "=&r"(__rv) : "r"(__val), "r"(__alp) : "cc", "memory");
100 }
101 return __rv;
102 }
103 #elif defined(_KERNEL)
104 static __inline unsigned char
__swp(unsigned char __val,__cpu_simple_lock_t * __ptr)105 __swp(unsigned char __val, __cpu_simple_lock_t *__ptr)
106 {
107 uint32_t __val32;
108 __asm volatile("swpb %0, %1, [%2]"
109 : "=&r" (__val32) : "r" (__val), "r" (__ptr) : "memory");
110 return __val32;
111 }
112 #else
113 /*
114 * On MP Cortex, SWP no longer guarantees atomic results. Thus we pad
115 * out SWP so that when the cpu generates an undefined exception we can replace
116 * the SWP/MOV instructions with the right LDREX/STREX instructions.
117 *
118 * This is why we force the SWP into the template needed for LDREX/STREX
119 * including the extra instructions and extra register for testing the result.
120 */
121 static __inline int
__swp(int __val,__cpu_simple_lock_t * __ptr)122 __swp(int __val, __cpu_simple_lock_t *__ptr)
123 {
124 int __tmp, __rv;
125 __asm volatile(
126 #if 1
127 "1:\t" "swp %[__rv], %[__val], [%[__ptr]]"
128 "\n\t" "b 2f"
129 #else
130 "1:\t" "ldrex %[__rv],[%[__ptr]]"
131 "\n\t" "strex %[__tmp],%[__val],[%[__ptr]]"
132 #endif
133 "\n\t" "cmp %[__tmp],#0"
134 "\n\t" "bne 1b"
135 "\n" "2:"
136 : [__rv] "=&r" (__rv), [__tmp] "=&r" (__tmp)
137 : [__val] "r" (__val), [__ptr] "r" (__ptr) : "cc", "memory");
138 return __rv;
139 }
140 #endif /* !_ARM_ARCH_6 */
141
142 static __inline void
__arm_membar_producer(void)143 __arm_membar_producer(void)
144 {
145 #ifdef _ARM_ARCH_7
146 __asm __volatile("dsb" ::: "memory");
147 #elif defined(_ARM_ARCH_6)
148 __asm __volatile("mcr\tp15,0,%0,c7,c10,4" :: "r"(0) : "memory");
149 #endif
150 }
151
152 static __inline void
__arm_membar_consumer(void)153 __arm_membar_consumer(void)
154 {
155 #ifdef _ARM_ARCH_7
156 __asm __volatile("dmb" ::: "memory");
157 #elif defined(_ARM_ARCH_6)
158 __asm __volatile("mcr\tp15,0,%0,c7,c10,5" :: "r"(0) : "memory");
159 #endif
160 }
161
162 static __inline void __unused
__cpu_simple_lock_init(__cpu_simple_lock_t * __alp)163 __cpu_simple_lock_init(__cpu_simple_lock_t *__alp)
164 {
165
166 *__alp = __SIMPLELOCK_UNLOCKED;
167 __arm_membar_producer();
168 }
169
170 #if !defined(__thumb__) || defined(_ARM_ARCH_T2)
171 static __inline void __unused
__cpu_simple_lock(__cpu_simple_lock_t * __alp)172 __cpu_simple_lock(__cpu_simple_lock_t *__alp)
173 {
174 #ifdef _ARM_ARCH_6
175 __arm_membar_consumer();
176 do {
177 /* spin */
178 } while (__arm_load_exclusive(__alp) != __SIMPLELOCK_UNLOCKED
179 || __arm_store_exclusive(__alp, __SIMPLELOCK_LOCKED));
180 __arm_membar_producer();
181 #else
182 while (__swp(__SIMPLELOCK_LOCKED, __alp) != __SIMPLELOCK_UNLOCKED)
183 continue;
184 #endif
185 }
186 #else
187 void __cpu_simple_lock(__cpu_simple_lock_t *);
188 #endif
189
190 #if !defined(__thumb__) || defined(_ARM_ARCH_T2)
191 static __inline int __unused
__cpu_simple_lock_try(__cpu_simple_lock_t * __alp)192 __cpu_simple_lock_try(__cpu_simple_lock_t *__alp)
193 {
194 #ifdef _ARM_ARCH_6
195 __arm_membar_consumer();
196 do {
197 if (__arm_load_exclusive(__alp) != __SIMPLELOCK_UNLOCKED) {
198 return 0;
199 }
200 } while (__arm_store_exclusive(__alp, __SIMPLELOCK_LOCKED));
201 __arm_membar_producer();
202 return 1;
203 #else
204 return (__swp(__SIMPLELOCK_LOCKED, __alp) == __SIMPLELOCK_UNLOCKED);
205 #endif
206 }
207 #else
208 int __cpu_simple_lock_try(__cpu_simple_lock_t *);
209 #endif
210
211 static __inline void __unused
__cpu_simple_unlock(__cpu_simple_lock_t * __alp)212 __cpu_simple_unlock(__cpu_simple_lock_t *__alp)
213 {
214
215 #ifdef _ARM_ARCH_8
216 if (sizeof(*__alp) == 1) {
217 __asm __volatile("stlb\t%0, [%1]"
218 :: "r"(__SIMPLELOCK_UNLOCKED), "r"(__alp) : "memory");
219 } else {
220 __asm __volatile("stl\t%0, [%1]"
221 :: "r"(__SIMPLELOCK_UNLOCKED), "r"(__alp) : "memory");
222 }
223 #else
224 __arm_membar_consumer();
225 *__alp = __SIMPLELOCK_UNLOCKED;
226 __arm_membar_producer();
227 #endif
228 }
229
230 #endif /* _ARM_LOCK_H_ */
231