1 /*-
2 * This file is a part of the MDCacheD server
3 * (c) 2007.-2012. Ivan Voras <ivoras@gmail.com>
4 * Released under the 2-clause BSDL
5 * $Id: LockPool.h 475 2012-04-08 10:27:00Z ivoras $
6 */
7
8 #ifndef _LOCKPOOL_H_
9 #define _LOCKPOOL_H_
10
11 #include <exception>
12 #include <pthread.h>
13 #include <errno.h>
14 #include <stdio.h>
15 #include <string.h>
16 #include <assert.h>
17 #include "misc.h"
18 #include "logger.h"
19
20 enum LOCK_RESULT { LR_OK, LR_TRYAGAIN, LR_DISASTER };
21
22 enum LOCK_OP { LOP_RDLOCK, LOP_WRLOCK, LOP_UNLOCK, LOP_TRY_RDLOCK, LOP_TRY_WRLOCK };
23
24 //#define LOCK_TRACE_ENABLE
25
26 #ifdef LOCK_TRACE_ENABLE
27 #define LOCK_TRACE(op,n) mc_log(LOG_DEBUG, "%%%slock on %d of %p[%d]", op, n, this, N);
28 #else
29 #define LOCK_TRACE(op,n)
30 #endif
31
32 class LockException : public std::exception {
33 private:
34 char *msg;
35 public:
LockException(char * msg)36 LockException(char *msg) : exception() {
37 this->msg = msg;
38 }
39 };
40
41
42 class LockNotifier {
43 public:
44 virtual enum LOCK_RESULT rdlock(unsigned *nlist, unsigned list_size) = 0;
45 virtual enum LOCK_RESULT wrlock(unsigned *nlist, unsigned list_size) = 0;
46 virtual enum LOCK_RESULT unlock(unsigned *nlist, unsigned list_size) = 0;
47 virtual enum LOCK_RESULT try_rdlock(unsigned *nlist, unsigned list_size) = 0;
48 virtual enum LOCK_RESULT try_wrlock(unsigned *nlist, unsigned list_size) = 0;
~LockNotifier()49 virtual ~LockNotifier() {}
50 };
51
52
53 /**
54 * LockPool is a giant pool of locks which are used practically in all locking
55 * operations, with the specific purpose of making them available over the
56 * network. It basically implements an array of rwlocks.
57 */
58 template <unsigned N>
59 class LockPool {
60 private:
61 pthread_rwlock_t locks[N];
62 /** The generation is incremented every time a lock is wrlocked */
63 unsigned generations[N];
64 LockNotifier *lnf;
65 public:
66 LockPool() throw(LockException);
67 ~LockPool();
getSize()68 unsigned getSize() { return N; }
getGeneration(unsigned n)69 unsigned getGeneration(unsigned n) { return generations[n]; }
70
set_lock_notifier(LockNotifier * lnf)71 void set_lock_notifier(LockNotifier *lnf) { this->lnf = lnf; }
unset_lock_notifier()72 void unset_lock_notifier() { this->lnf = NULL; }
73
74 /* Success-or-death, atomic */
75 enum LOCK_RESULT rdlock(unsigned n);
76 enum LOCK_RESULT rdlock(unsigned *nlist, unsigned list_size);
77
78 enum LOCK_RESULT wrlock(unsigned n);
79 enum LOCK_RESULT wrlock(unsigned *nlist, unsigned list_size);
80
81 enum LOCK_RESULT unlock(unsigned n);
82 enum LOCK_RESULT unlock(unsigned *nlist, unsigned list_size);
83
84 /* EWOULDBLOCK style; will return true only if both local and notifier
85 * return true. */
86 enum LOCK_RESULT try_rdlock(unsigned n);
87 enum LOCK_RESULT try_rdlock(unsigned *nlist, unsigned list_size);
88
89 enum LOCK_RESULT try_wrlock(unsigned n);
90 enum LOCK_RESULT try_wrlock(unsigned *nlist, unsigned list_size);
91
92 /* Backdoor, not going through the notifier */
93 enum LOCK_RESULT _internalLockOp(enum LOCK_OP lop, unsigned *nlist, unsigned list_size);
94
95 /* Check if all the locks are free to be locked. Expensive and for debugging only */
96 bool allClear();
97 };
98
99
100 template <unsigned N>
LockPool()101 LockPool<N>::LockPool() throw (LockException) {
102 for (unsigned i = 0; i < N; i++) {
103 if (pthread_rwlock_init(&locks[i], NULL) != 0)
104 throw new LockException((char*)"pthread_rwlock_init failed at " __FILE__);
105 }
106 lnf = NULL;
107 memset(&generations, 0, sizeof(generations));
108 }
109
110
111 template <unsigned N>
~LockPool()112 LockPool<N>::~LockPool() {
113 for (unsigned i = 0; i < N; i++)
114 pthread_rwlock_destroy(&locks[i]);
115 }
116
117 static inline int
pthread_rwlock_rdlock_l(pthread_rwlock_t * lock)118 pthread_rwlock_rdlock_l(pthread_rwlock_t *lock) {
119 int r;
120
121 for (;;) {
122 r = pthread_rwlock_rdlock(lock);
123 if (r == 0 || r != EAGAIN)
124 break;
125 }
126 return r;
127 }
128
129 template <unsigned N>
rdlock(unsigned n)130 enum LOCK_RESULT LockPool<N>::rdlock(unsigned n) {
131 assert(n < N);
132 LOCK_TRACE("rd", n);
133 CRIT_ASSERT(pthread_rwlock_rdlock_l(&locks[n]) == 0);
134 if (unlikely(lnf != NULL)) {
135 enum LOCK_RESULT lres = lnf->rdlock(&n, 1);
136
137 if (unlikely(lres != LR_OK))
138 CRIT_ASSERT(pthread_rwlock_unlock(&locks[n])); // Rollback
139 return lres;
140 }
141 return LR_OK;
142 }
143
144
145 template <unsigned N>
rdlock(unsigned * nlist,unsigned list_size)146 enum LOCK_RESULT LockPool<N>::rdlock(unsigned *nlist, unsigned list_size) {
147 for (unsigned i = 0; i < list_size; i++) {
148 assert(nlist[i] < N);
149 LOCK_TRACE("*rd", nlist[i]);
150 CRIT_ASSERT(pthread_rwlock_rdlock_l(&locks[nlist[i]]) == 0);
151 }
152 if (unlikely(lnf != NULL)) {
153 enum LOCK_RESULT lres = lnf->rdlock(nlist, list_size);
154
155 if (unlikely(lres != LR_OK))
156 for (unsigned i = 0; i < list_size; i++) // Rollback
157 CRIT_ASSERT(pthread_rwlock_unlock(&locks[nlist[i]]) == 0);
158 return lres;
159 }
160 return LR_OK;
161 }
162
163
164 template <unsigned N>
wrlock(unsigned n)165 enum LOCK_RESULT LockPool<N>::wrlock(unsigned n) {
166 assert(n < N);
167 LOCK_TRACE("wr", n);
168 CRIT_ASSERT(pthread_rwlock_wrlock(&locks[n]) == 0);
169 if (unlikely(lnf != NULL)) {
170 enum LOCK_RESULT lres = lnf->wrlock(&n, 1);
171
172 if (unlikely(lres != LR_OK))
173 CRIT_ASSERT(pthread_rwlock_unlock(&locks[n]) == 0); // Rollback
174 return lres;
175 }
176 generations[n]++;
177 return LR_OK;
178 }
179
180
181 template <unsigned N>
wrlock(unsigned * nlist,unsigned list_size)182 enum LOCK_RESULT LockPool<N>::wrlock(unsigned *nlist, unsigned list_size) {
183 for (unsigned i = 0; i < list_size; i++) {
184 assert(nlist[i] < N);
185 LOCK_TRACE("*wr", nlist[i]);
186 CRIT_ASSERT(pthread_rwlock_wrlock(&locks[nlist[i]]) == 0);
187 }
188 if (unlikely(lnf != NULL)) {
189 enum LOCK_RESULT lres = lnf->wrlock(nlist, list_size);
190
191 if (unlikely(lres != LR_OK))
192 for (unsigned i = 0; i < list_size; i++) // Rollback
193 CRIT_ASSERT(pthread_rwlock_unlock(&locks[nlist[i]]) == 0);
194 return lres;
195 }
196 for (unsigned i = 0; i < list_size; i++)
197 generations[nlist[i]]++;
198 return LR_OK;
199 }
200
201
202 /* TODO: What does a remote failure at UNLOCK mean? Will it leave inconsistent
203 * lock states? Will it matter?*/
204 template <unsigned N>
unlock(unsigned n)205 enum LOCK_RESULT LockPool<N>::unlock(unsigned n) {
206 LOCK_TRACE("un", n);
207 int result;
208 if ((result = pthread_rwlock_unlock(&locks[n])) != 0)
209 mc_log(LOG_CRIT, "pthread_rwlock_unlock() failed on lock %u: errno %d (%s)", n, result, strerror(result));
210 if (unlikely(lnf != NULL))
211 lnf->unlock(&n, 1);
212 return LR_OK;
213 }
214
215
216 template <unsigned N>
unlock(unsigned * nlist,unsigned list_size)217 enum LOCK_RESULT LockPool<N>::unlock(unsigned *nlist, unsigned list_size) {
218 for (unsigned i = 0; i < list_size; i++) {
219 LOCK_TRACE("*un", nlist[i]);
220 CRIT_ASSERT(pthread_rwlock_unlock(&locks[nlist[i]]) == 0);
221 }
222 if (unlikely(lnf != NULL))
223 lnf->unlock(nlist, list_size);
224 return LR_OK;
225 }
226
227
228 template <unsigned N>
try_rdlock(unsigned n)229 enum LOCK_RESULT LockPool<N>::try_rdlock(unsigned n) {
230 assert(n < N);
231 LOCK_TRACE("tryrd", n);
232 int result = pthread_rwlock_tryrdlock(&locks[n]);
233 if (likely(result == 0)) {
234 if (unlikely(lnf != NULL)) {
235 if (lnf->try_rdlock(&n, 1) == LR_OK)
236 return LR_OK;
237 else {
238 /* Does it matter if the remote locker failed with
239 * LR_TRYAGAIN or LR_DISASTER? */
240 pthread_rwlock_unlock(&locks[n]); // Rollback
241 return LR_TRYAGAIN;
242 }
243 } else
244 return LR_OK;
245 } else {
246 if (result == EBUSY)
247 return LR_TRYAGAIN;
248 mc_log(LOG_CRIT, "Error in pthread_rwlock_tryrdlock: errno %d", result);
249 return LR_DISASTER;
250 }
251 }
252
253
254 template <unsigned N>
try_rdlock(unsigned * nlist,unsigned list_size)255 enum LOCK_RESULT LockPool<N>::try_rdlock(unsigned *nlist, unsigned list_size) {
256 for (unsigned i = 0; i < list_size; i++) {
257 LOCK_TRACE("*tryrd", nlist[i]);
258 int result = pthread_rwlock_tryrdlock(&locks[nlist[i]]);
259 if (unlikely(result != 0)) {
260 for (unsigned j = 0; j < i; j++)
261 pthread_rwlock_unlock(&locks[nlist[j]]);
262 if (result != EBUSY)
263 mc_log(LOG_CRIT, "Error in pthread_rwlock_tryrdlock: errno %d", result);
264 return LR_TRYAGAIN;
265 }
266 }
267 if (unlikely(lnf != NULL)) {
268 if (lnf->try_rdlock(nlist, list_size) == LR_OK)
269 return LR_OK;
270 else {
271 for (unsigned i = 0; i < list_size; i++) // Rollback
272 pthread_rwlock_unlock(&locks[nlist[i]]);
273 return LR_TRYAGAIN;
274 }
275 } else
276 return LR_OK;
277 }
278
279
280 template <unsigned N>
try_wrlock(unsigned n)281 enum LOCK_RESULT LockPool<N>::try_wrlock(unsigned n) {
282 assert(n < N);
283 LOCK_TRACE("trywr", n);
284 int result = pthread_rwlock_trywrlock(&locks[n]);
285 if (likely(result == 0)) {
286 if (unlikely(lnf != NULL)) {
287 if (lnf->try_wrlock(&n, 1) == LR_OK)
288 return LR_OK;
289 else {
290 pthread_rwlock_unlock(&locks[n]); // Rollback
291 return LR_TRYAGAIN;
292 }
293 }
294 return LR_OK;
295 } else {
296 if (result == EBUSY)
297 return LR_TRYAGAIN;
298 mc_log(LOG_CRIT, "Error in pthread_rwlock_trywrlock: errno %d", result);
299 return LR_DISASTER;
300 }
301 }
302
303
304 template <unsigned N>
try_wrlock(unsigned * nlist,unsigned list_size)305 enum LOCK_RESULT LockPool<N>::try_wrlock(unsigned *nlist, unsigned list_size) {
306 for (unsigned i = 0; i < list_size; i++) {
307 LOCK_TRACE("*trywr", nlist[i]);
308 int result = pthread_rwlock_trywrlock(&locks[nlist[i]]);
309 if (unlikely(result != 0)) {
310 for (unsigned j = 0; j < i; j++)
311 pthread_rwlock_unlock(&locks[nlist[j]]);
312 if (result != EBUSY)
313 mc_log(LOG_CRIT, "Error in pthread_rwlock_trywrlock: errno %d", result);
314 return LR_TRYAGAIN;
315 }
316 }
317 if (unlikely(lnf != NULL)) {
318 if (lnf->try_wrlock(nlist, list_size) == LR_OK)
319 return LR_OK;
320 else {
321 for (unsigned i = 0; i < list_size; i++) // Rollback
322 pthread_rwlock_unlock(&locks[nlist[i]]);
323 return LR_TRYAGAIN;
324 }
325 } else
326 return LR_OK;
327 }
328
329
330 template <unsigned N>
_internalLockOp(enum LOCK_OP lop,unsigned * nlist,unsigned list_size)331 enum LOCK_RESULT LockPool<N>::_internalLockOp(enum LOCK_OP lop, unsigned *nlist, unsigned list_size) {
332 unsigned i;
333 int result;
334
335 switch (lop) {
336 case LOP_RDLOCK:
337 for (i = 0; i < list_size; i++)
338 CRIT_ASSERT(pthread_rwlock_rdlock_l(&locks[nlist[i]]) == 0);
339 return LR_OK;
340 case LOP_WRLOCK:
341 for (i = 0; i < list_size; i++)
342 CRIT_ASSERT(pthread_rwlock_wrlock(&locks[nlist[i]]) == 0);
343 return LR_OK;
344 case LOP_UNLOCK:
345 for (i = 0; i < list_size; i++)
346 CRIT_ASSERT(pthread_rwlock_unlock(&locks[nlist[i]]) == 0);
347 return LR_OK;
348 case LOP_TRY_RDLOCK:
349 for (i = 0; i < list_size; i++)
350 if ((result = pthread_rwlock_tryrdlock(&locks[nlist[i]])) != 0) {
351 for (unsigned j = 0; j < i; j++) // Rollback
352 CRIT_ASSERT(pthread_rwlock_unlock(&locks[nlist[j]]) == 0);
353 return result == EBUSY ? LR_TRYAGAIN : LR_DISASTER;
354 }
355 return LR_OK;
356 case LOP_TRY_WRLOCK:
357 for (i = 0; i < list_size; i++)
358 if ((result = pthread_rwlock_trywrlock(&locks[nlist[i]])) != 0) {
359 for (unsigned j = 0; j < i; j++) // Rollback
360 CRIT_ASSERT(pthread_rwlock_unlock(&locks[nlist[j]]) == 0);
361 return result == EBUSY ? LR_TRYAGAIN : LR_DISASTER;
362 }
363 return LR_OK;
364 default:
365 mc_log(LOG_ERR, "Unknown lock op: %d", lop);
366 }
367 return LR_DISASTER;
368 }
369
370
371 /**
372 * Test if all locks in the pool can be acquired. This is pointless to do except
373 * for extreme debugging :)
374 */
375 template <unsigned N>
allClear()376 bool LockPool<N>::allClear() {
377 int r;
378 for (unsigned i = 0; i < N; i++) {
379 if ((r = pthread_rwlock_trywrlock(&locks[i])) != 0) {
380 for (unsigned j = 0; j < i; j++)
381 pthread_rwlock_unlock(&locks[j]);
382 printf("allClear<%d>: taken: %d? err %d: \n", N, i, r);
383 return false;
384 }
385 }
386 for (unsigned i = 0; i < N; i++)
387 pthread_rwlock_unlock(&locks[i]);
388 return true;
389 }
390
391 #undef LOCK_TRACE
392
393 #endif
394