1 /*
2 * Copyright (c) 2006,2012-2018 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34 /*
35 * The Cache Coherency Management System (CCMS)
36 */
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/kernel.h>
41 #include <machine/limits.h>
42
43 #include <sys/spinlock2.h>
44
45 #include "hammer2_ccms.h"
46 #include "hammer2.h"
47
48 int ccms_debug = 0;
49
50 void
ccms_cst_init(ccms_cst_t * cst)51 ccms_cst_init(ccms_cst_t *cst)
52 {
53 bzero(cst, sizeof(*cst));
54 hammer2_spin_init(&cst->spin, "ccmscst");
55 }
56
57 void
ccms_cst_uninit(ccms_cst_t * cst)58 ccms_cst_uninit(ccms_cst_t *cst)
59 {
60 KKASSERT(cst->count == 0);
61 if (cst->state != CCMS_STATE_INVALID) {
62 /* XXX */
63 }
64 }
65
66 /************************************************************************
67 * CST SUPPORT FUNCTIONS *
68 ************************************************************************/
69
70 /*
71 * Acquire local cache state & lock. If the current thread already holds
72 * the lock exclusively we bump the exclusive count, even if the thread is
73 * trying to get a shared lock.
74 */
75 void
ccms_thread_lock(ccms_cst_t * cst,ccms_state_t state)76 ccms_thread_lock(ccms_cst_t *cst, ccms_state_t state)
77 {
78 /*
79 * Regardless of the type of lock requested if the current thread
80 * already holds an exclusive lock we bump the exclusive count and
81 * return. This requires no spinlock.
82 */
83 LOCKENTER;
84 if (cst->count < 0 && cst->td == curthread) {
85 --cst->count;
86 return;
87 }
88
89 /*
90 * Otherwise use the spinlock to interlock the operation and sleep
91 * as necessary.
92 */
93 hammer2_spin_ex(&cst->spin);
94 if (state == CCMS_STATE_SHARED) {
95 while (cst->count < 0 || cst->upgrade) {
96 cst->blocked = 1;
97 ssleep(cst, &cst->spin, 0, "ccmslck", hz);
98 }
99 ++cst->count;
100 KKASSERT(cst->td == NULL);
101 } else if (state == CCMS_STATE_EXCLUSIVE) {
102 while (cst->count != 0 || cst->upgrade) {
103 cst->blocked = 1;
104 ssleep(cst, &cst->spin, 0, "ccmslck", hz);
105 }
106 cst->count = -1;
107 cst->td = curthread;
108 } else {
109 hammer2_spin_unex(&cst->spin);
110 panic("ccms_thread_lock: bad state %d\n", state);
111 }
112 hammer2_spin_unex(&cst->spin);
113 }
114
115 /*
116 * Same as ccms_thread_lock() but acquires the lock non-blocking. Returns
117 * 0 on success, EBUSY on failure.
118 */
119 int
ccms_thread_lock_nonblock(ccms_cst_t * cst,ccms_state_t state)120 ccms_thread_lock_nonblock(ccms_cst_t *cst, ccms_state_t state)
121 {
122 if (cst->count < 0 && cst->td == curthread) {
123 --cst->count;
124 LOCKENTER;
125 return(0);
126 }
127
128 hammer2_spin_ex(&cst->spin);
129 if (state == CCMS_STATE_SHARED) {
130 if (cst->count < 0 || cst->upgrade) {
131 hammer2_spin_unex(&cst->spin);
132 return (EBUSY);
133 }
134 ++cst->count;
135 KKASSERT(cst->td == NULL);
136 } else if (state == CCMS_STATE_EXCLUSIVE) {
137 if (cst->count != 0 || cst->upgrade) {
138 hammer2_spin_unex(&cst->spin);
139 return (EBUSY);
140 }
141 cst->count = -1;
142 cst->td = curthread;
143 } else {
144 hammer2_spin_unex(&cst->spin);
145 panic("ccms_thread_lock_nonblock: bad state %d\n", state);
146 }
147 hammer2_spin_unex(&cst->spin);
148 LOCKENTER;
149 return(0);
150 }
151
152 ccms_state_t
ccms_thread_lock_temp_release(ccms_cst_t * cst)153 ccms_thread_lock_temp_release(ccms_cst_t *cst)
154 {
155 if (cst->count < 0) {
156 ccms_thread_unlock(cst);
157 return(CCMS_STATE_EXCLUSIVE);
158 }
159 if (cst->count > 0) {
160 ccms_thread_unlock(cst);
161 return(CCMS_STATE_SHARED);
162 }
163 return (CCMS_STATE_INVALID);
164 }
165
166 void
ccms_thread_lock_temp_restore(ccms_cst_t * cst,ccms_state_t ostate)167 ccms_thread_lock_temp_restore(ccms_cst_t *cst, ccms_state_t ostate)
168 {
169 ccms_thread_lock(cst, ostate);
170 }
171
172 /*
173 * Temporarily upgrade a thread lock for making local structural changes.
174 * No new shared or exclusive locks can be acquired by others while we are
175 * upgrading, but other upgraders are allowed.
176 */
177 ccms_state_t
ccms_thread_lock_upgrade(ccms_cst_t * cst)178 ccms_thread_lock_upgrade(ccms_cst_t *cst)
179 {
180 /*
181 * Nothing to do if already exclusive
182 */
183 if (cst->count < 0) {
184 KKASSERT(cst->td == curthread);
185 return(CCMS_STATE_EXCLUSIVE);
186 }
187
188 /*
189 * Convert a shared lock to exclusive.
190 */
191 if (cst->count > 0) {
192 hammer2_spin_ex(&cst->spin);
193 ++cst->upgrade;
194 --cst->count;
195 while (cst->count) {
196 cst->blocked = 1;
197 ssleep(cst, &cst->spin, 0, "ccmsupg", hz);
198 }
199 cst->count = -1;
200 cst->td = curthread;
201 hammer2_spin_unex(&cst->spin);
202 return(CCMS_STATE_SHARED);
203 }
204 panic("ccms_thread_lock_upgrade: not locked");
205 /* NOT REACHED */
206 return(0);
207 }
208
209 void
ccms_thread_lock_downgrade(ccms_cst_t * cst,ccms_state_t ostate)210 ccms_thread_lock_downgrade(ccms_cst_t *cst, ccms_state_t ostate)
211 {
212 if (ostate == CCMS_STATE_SHARED) {
213 KKASSERT(cst->td == curthread);
214 KKASSERT(cst->count == -1);
215 hammer2_spin_ex(&cst->spin);
216 --cst->upgrade;
217 cst->count = 1;
218 cst->td = NULL;
219 if (cst->blocked) {
220 cst->blocked = 0;
221 hammer2_spin_unex(&cst->spin);
222 wakeup(cst);
223 } else {
224 hammer2_spin_unex(&cst->spin);
225 }
226 }
227 /* else nothing to do if excl->excl */
228 }
229
230 /*
231 * Release a local thread lock
232 */
233 void
ccms_thread_unlock(ccms_cst_t * cst)234 ccms_thread_unlock(ccms_cst_t *cst)
235 {
236 LOCKEXIT;
237 if (cst->count < 0) {
238 /*
239 * Exclusive
240 */
241 KKASSERT(cst->td == curthread);
242 if (cst->count < -1) {
243 ++cst->count;
244 return;
245 }
246 hammer2_spin_ex(&cst->spin);
247 KKASSERT(cst->count == -1);
248 cst->count = 0;
249 cst->td = NULL;
250 if (cst->blocked) {
251 cst->blocked = 0;
252 hammer2_spin_unex(&cst->spin);
253 wakeup(cst);
254 return;
255 }
256 hammer2_spin_unex(&cst->spin);
257 } else if (cst->count > 0) {
258 /*
259 * Shared
260 */
261 hammer2_spin_ex(&cst->spin);
262 if (--cst->count == 0 && cst->blocked) {
263 cst->blocked = 0;
264 hammer2_spin_unex(&cst->spin);
265 wakeup(cst);
266 return;
267 }
268 hammer2_spin_unex(&cst->spin);
269 } else {
270 panic("ccms_thread_unlock: bad zero count\n");
271 }
272 }
273
274 void
ccms_thread_lock_setown(ccms_cst_t * cst)275 ccms_thread_lock_setown(ccms_cst_t *cst)
276 {
277 KKASSERT(cst->count < 0);
278 cst->td = curthread;
279 }
280
281 /*
282 * Release a previously upgraded local thread lock
283 */
284 void
ccms_thread_unlock_upgraded(ccms_cst_t * cst,ccms_state_t ostate)285 ccms_thread_unlock_upgraded(ccms_cst_t *cst, ccms_state_t ostate)
286 {
287 if (ostate == CCMS_STATE_SHARED) {
288 LOCKEXIT;
289 KKASSERT(cst->td == curthread);
290 KKASSERT(cst->count == -1);
291 hammer2_spin_ex(&cst->spin);
292 --cst->upgrade;
293 cst->count = 0;
294 cst->td = NULL;
295 if (cst->blocked) {
296 cst->blocked = 0;
297 hammer2_spin_unex(&cst->spin);
298 wakeup(cst);
299 } else {
300 hammer2_spin_unex(&cst->spin);
301 }
302 } else {
303 ccms_thread_unlock(cst);
304 }
305 }
306
307 int
ccms_thread_lock_owned(ccms_cst_t * cst)308 ccms_thread_lock_owned(ccms_cst_t *cst)
309 {
310 return(cst->count < 0 && cst->td == curthread);
311 }
312