1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5 */
6
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9 #include <linux/sched.h>
10 #include <linux/slab.h>
11 #include <linux/spinlock.h>
12 #include <linux/buffer_head.h>
13 #include <linux/delay.h>
14 #include <linux/sort.h>
15 #include <linux/hash.h>
16 #include <linux/jhash.h>
17 #include <linux/kallsyms.h>
18 #include <linux/gfs2_ondisk.h>
19 #include <linux/list.h>
20 #include <linux/wait.h>
21 #include <linux/module.h>
22 #include <linux/uaccess.h>
23 #include <linux/seq_file.h>
24 #include <linux/debugfs.h>
25 #include <linux/kthread.h>
26 #include <linux/freezer.h>
27 #include <linux/workqueue.h>
28 #include <linux/jiffies.h>
29 #include <linux/rcupdate.h>
30 #include <linux/rculist_bl.h>
31 #include <linux/bit_spinlock.h>
32 #include <linux/percpu.h>
33 #include <linux/list_sort.h>
34 #include <linux/lockref.h>
35 #include <linux/rhashtable.h>
36 #include <linux/pid_namespace.h>
37 #include <linux/fdtable.h>
38 #include <linux/file.h>
39
40 #include "gfs2.h"
41 #include "incore.h"
42 #include "glock.h"
43 #include "glops.h"
44 #include "inode.h"
45 #include "lops.h"
46 #include "meta_io.h"
47 #include "quota.h"
48 #include "super.h"
49 #include "util.h"
50 #include "bmap.h"
51 #define CREATE_TRACE_POINTS
52 #include "trace_gfs2.h"
53
54 struct gfs2_glock_iter {
55 struct gfs2_sbd *sdp; /* incore superblock */
56 struct rhashtable_iter hti; /* rhashtable iterator */
57 struct gfs2_glock *gl; /* current glock struct */
58 loff_t last_pos; /* last position */
59 };
60
61 typedef void (*glock_examiner) (struct gfs2_glock * gl);
62
63 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
64 static void __gfs2_glock_dq(struct gfs2_holder *gh);
65 static void handle_callback(struct gfs2_glock *gl, unsigned int state,
66 unsigned long delay, bool remote);
67
68 static struct dentry *gfs2_root;
69 static struct workqueue_struct *glock_workqueue;
70 static LIST_HEAD(lru_list);
71 static atomic_t lru_count = ATOMIC_INIT(0);
72 static DEFINE_SPINLOCK(lru_lock);
73
74 #define GFS2_GL_HASH_SHIFT 15
75 #define GFS2_GL_HASH_SIZE BIT(GFS2_GL_HASH_SHIFT)
76
77 static const struct rhashtable_params ht_parms = {
78 .nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
79 .key_len = offsetofend(struct lm_lockname, ln_type),
80 .key_offset = offsetof(struct gfs2_glock, gl_name),
81 .head_offset = offsetof(struct gfs2_glock, gl_node),
82 };
83
84 static struct rhashtable gl_hash_table;
85
86 #define GLOCK_WAIT_TABLE_BITS 12
87 #define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
88 static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
89
90 struct wait_glock_queue {
91 struct lm_lockname *name;
92 wait_queue_entry_t wait;
93 };
94
glock_wake_function(wait_queue_entry_t * wait,unsigned int mode,int sync,void * key)95 static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
96 int sync, void *key)
97 {
98 struct wait_glock_queue *wait_glock =
99 container_of(wait, struct wait_glock_queue, wait);
100 struct lm_lockname *wait_name = wait_glock->name;
101 struct lm_lockname *wake_name = key;
102
103 if (wake_name->ln_sbd != wait_name->ln_sbd ||
104 wake_name->ln_number != wait_name->ln_number ||
105 wake_name->ln_type != wait_name->ln_type)
106 return 0;
107 return autoremove_wake_function(wait, mode, sync, key);
108 }
109
glock_waitqueue(struct lm_lockname * name)110 static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
111 {
112 u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0);
113
114 return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS);
115 }
116
117 /**
118 * wake_up_glock - Wake up waiters on a glock
119 * @gl: the glock
120 */
wake_up_glock(struct gfs2_glock * gl)121 static void wake_up_glock(struct gfs2_glock *gl)
122 {
123 wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name);
124
125 if (waitqueue_active(wq))
126 __wake_up(wq, TASK_NORMAL, 1, &gl->gl_name);
127 }
128
gfs2_glock_dealloc(struct rcu_head * rcu)129 static void gfs2_glock_dealloc(struct rcu_head *rcu)
130 {
131 struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
132
133 kfree(gl->gl_lksb.sb_lvbptr);
134 if (gl->gl_ops->go_flags & GLOF_ASPACE) {
135 struct gfs2_glock_aspace *gla =
136 container_of(gl, struct gfs2_glock_aspace, glock);
137 kmem_cache_free(gfs2_glock_aspace_cachep, gla);
138 } else
139 kmem_cache_free(gfs2_glock_cachep, gl);
140 }
141
142 /**
143 * glock_blocked_by_withdraw - determine if we can still use a glock
144 * @gl: the glock
145 *
146 * We need to allow some glocks to be enqueued, dequeued, promoted, and demoted
147 * when we're withdrawn. For example, to maintain metadata integrity, we should
148 * disallow the use of inode and rgrp glocks when withdrawn. Other glocks like
149 * the iopen or freeze glock may be safely used because none of their
150 * metadata goes through the journal. So in general, we should disallow all
151 * glocks that are journaled, and allow all the others. One exception is:
152 * we need to allow our active journal to be promoted and demoted so others
153 * may recover it and we can reacquire it when they're done.
154 */
glock_blocked_by_withdraw(struct gfs2_glock * gl)155 static bool glock_blocked_by_withdraw(struct gfs2_glock *gl)
156 {
157 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
158
159 if (!gfs2_withdrawing_or_withdrawn(sdp))
160 return false;
161 if (gl->gl_ops->go_flags & GLOF_NONDISK)
162 return false;
163 if (!sdp->sd_jdesc ||
164 gl->gl_name.ln_number == sdp->sd_jdesc->jd_no_addr)
165 return false;
166 return true;
167 }
168
__gfs2_glock_free(struct gfs2_glock * gl)169 static void __gfs2_glock_free(struct gfs2_glock *gl)
170 {
171 rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
172 smp_mb();
173 wake_up_glock(gl);
174 call_rcu(&gl->gl_rcu, gfs2_glock_dealloc);
175 }
176
gfs2_glock_free(struct gfs2_glock * gl)177 void gfs2_glock_free(struct gfs2_glock *gl) {
178 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
179
180 __gfs2_glock_free(gl);
181 if (atomic_dec_and_test(&sdp->sd_glock_disposal))
182 wake_up(&sdp->sd_kill_wait);
183 }
184
gfs2_glock_free_later(struct gfs2_glock * gl)185 void gfs2_glock_free_later(struct gfs2_glock *gl) {
186 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
187
188 spin_lock(&lru_lock);
189 list_add(&gl->gl_lru, &sdp->sd_dead_glocks);
190 spin_unlock(&lru_lock);
191 if (atomic_dec_and_test(&sdp->sd_glock_disposal))
192 wake_up(&sdp->sd_kill_wait);
193 }
194
gfs2_free_dead_glocks(struct gfs2_sbd * sdp)195 static void gfs2_free_dead_glocks(struct gfs2_sbd *sdp)
196 {
197 struct list_head *list = &sdp->sd_dead_glocks;
198
199 while(!list_empty(list)) {
200 struct gfs2_glock *gl;
201
202 gl = list_first_entry(list, struct gfs2_glock, gl_lru);
203 list_del_init(&gl->gl_lru);
204 __gfs2_glock_free(gl);
205 }
206 }
207
208 /**
209 * gfs2_glock_hold() - increment reference count on glock
210 * @gl: The glock to hold
211 *
212 */
213
gfs2_glock_hold(struct gfs2_glock * gl)214 struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl)
215 {
216 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
217 lockref_get(&gl->gl_lockref);
218 return gl;
219 }
220
221 /**
222 * demote_ok - Check to see if it's ok to unlock a glock
223 * @gl: the glock
224 *
225 * Returns: 1 if it's ok
226 */
227
demote_ok(const struct gfs2_glock * gl)228 static int demote_ok(const struct gfs2_glock *gl)
229 {
230 const struct gfs2_glock_operations *glops = gl->gl_ops;
231
232 if (gl->gl_state == LM_ST_UNLOCKED)
233 return 0;
234 if (!list_empty(&gl->gl_holders))
235 return 0;
236 if (glops->go_demote_ok)
237 return glops->go_demote_ok(gl);
238 return 1;
239 }
240
241
gfs2_glock_add_to_lru(struct gfs2_glock * gl)242 void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
243 {
244 if (!(gl->gl_ops->go_flags & GLOF_LRU))
245 return;
246
247 spin_lock(&lru_lock);
248
249 list_move_tail(&gl->gl_lru, &lru_list);
250
251 if (!test_bit(GLF_LRU, &gl->gl_flags)) {
252 set_bit(GLF_LRU, &gl->gl_flags);
253 atomic_inc(&lru_count);
254 }
255
256 spin_unlock(&lru_lock);
257 }
258
gfs2_glock_remove_from_lru(struct gfs2_glock * gl)259 static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
260 {
261 if (!(gl->gl_ops->go_flags & GLOF_LRU))
262 return;
263
264 spin_lock(&lru_lock);
265 if (test_bit(GLF_LRU, &gl->gl_flags)) {
266 list_del_init(&gl->gl_lru);
267 atomic_dec(&lru_count);
268 clear_bit(GLF_LRU, &gl->gl_flags);
269 }
270 spin_unlock(&lru_lock);
271 }
272
273 /*
274 * Enqueue the glock on the work queue. Passes one glock reference on to the
275 * work queue.
276 */
gfs2_glock_queue_work(struct gfs2_glock * gl,unsigned long delay)277 static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
278 if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) {
279 /*
280 * We are holding the lockref spinlock, and the work was still
281 * queued above. The queued work (glock_work_func) takes that
282 * spinlock before dropping its glock reference(s), so it
283 * cannot have dropped them in the meantime.
284 */
285 GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
286 gl->gl_lockref.count--;
287 }
288 }
289
__gfs2_glock_put(struct gfs2_glock * gl)290 static void __gfs2_glock_put(struct gfs2_glock *gl)
291 {
292 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
293 struct address_space *mapping = gfs2_glock2aspace(gl);
294
295 lockref_mark_dead(&gl->gl_lockref);
296 spin_unlock(&gl->gl_lockref.lock);
297 gfs2_glock_remove_from_lru(gl);
298 GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
299 if (mapping) {
300 truncate_inode_pages_final(mapping);
301 if (!gfs2_withdrawing_or_withdrawn(sdp))
302 GLOCK_BUG_ON(gl, !mapping_empty(mapping));
303 }
304 trace_gfs2_glock_put(gl);
305 sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
306 }
307
308 /**
309 * gfs2_glock_put() - Decrement reference count on glock
310 * @gl: The glock to put
311 *
312 */
313
gfs2_glock_put(struct gfs2_glock * gl)314 void gfs2_glock_put(struct gfs2_glock *gl)
315 {
316 if (lockref_put_or_lock(&gl->gl_lockref))
317 return;
318
319 __gfs2_glock_put(gl);
320 }
321
322 /*
323 * gfs2_glock_put_async - Decrement reference count without sleeping
324 * @gl: The glock to put
325 *
326 * Decrement the reference count on glock immediately unless it is the last
327 * reference. Defer putting the last reference to work queue context.
328 */
gfs2_glock_put_async(struct gfs2_glock * gl)329 void gfs2_glock_put_async(struct gfs2_glock *gl)
330 {
331 if (lockref_put_or_lock(&gl->gl_lockref))
332 return;
333
334 GLOCK_BUG_ON(gl, gl->gl_lockref.count != 1);
335 gfs2_glock_queue_work(gl, 0);
336 spin_unlock(&gl->gl_lockref.lock);
337 }
338
339 /**
340 * may_grant - check if it's ok to grant a new lock
341 * @gl: The glock
342 * @current_gh: One of the current holders of @gl
343 * @gh: The lock request which we wish to grant
344 *
345 * With our current compatibility rules, if a glock has one or more active
346 * holders (HIF_HOLDER flag set), any of those holders can be passed in as
347 * @current_gh; they are all the same as far as compatibility with the new @gh
348 * goes.
349 *
350 * Returns true if it's ok to grant the lock.
351 */
352
may_grant(struct gfs2_glock * gl,struct gfs2_holder * current_gh,struct gfs2_holder * gh)353 static inline bool may_grant(struct gfs2_glock *gl,
354 struct gfs2_holder *current_gh,
355 struct gfs2_holder *gh)
356 {
357 if (current_gh) {
358 GLOCK_BUG_ON(gl, !test_bit(HIF_HOLDER, ¤t_gh->gh_iflags));
359
360 switch(current_gh->gh_state) {
361 case LM_ST_EXCLUSIVE:
362 /*
363 * Here we make a special exception to grant holders
364 * who agree to share the EX lock with other holders
365 * who also have the bit set. If the original holder
366 * has the LM_FLAG_NODE_SCOPE bit set, we grant more
367 * holders with the bit set.
368 */
369 return gh->gh_state == LM_ST_EXCLUSIVE &&
370 (current_gh->gh_flags & LM_FLAG_NODE_SCOPE) &&
371 (gh->gh_flags & LM_FLAG_NODE_SCOPE);
372
373 case LM_ST_SHARED:
374 case LM_ST_DEFERRED:
375 return gh->gh_state == current_gh->gh_state;
376
377 default:
378 return false;
379 }
380 }
381
382 if (gl->gl_state == gh->gh_state)
383 return true;
384 if (gh->gh_flags & GL_EXACT)
385 return false;
386 if (gl->gl_state == LM_ST_EXCLUSIVE) {
387 return gh->gh_state == LM_ST_SHARED ||
388 gh->gh_state == LM_ST_DEFERRED;
389 }
390 if (gh->gh_flags & LM_FLAG_ANY)
391 return gl->gl_state != LM_ST_UNLOCKED;
392 return false;
393 }
394
gfs2_holder_wake(struct gfs2_holder * gh)395 static void gfs2_holder_wake(struct gfs2_holder *gh)
396 {
397 clear_bit(HIF_WAIT, &gh->gh_iflags);
398 smp_mb__after_atomic();
399 wake_up_bit(&gh->gh_iflags, HIF_WAIT);
400 if (gh->gh_flags & GL_ASYNC) {
401 struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
402
403 wake_up(&sdp->sd_async_glock_wait);
404 }
405 }
406
407 /**
408 * do_error - Something unexpected has happened during a lock request
409 * @gl: The glock
410 * @ret: The status from the DLM
411 */
412
do_error(struct gfs2_glock * gl,const int ret)413 static void do_error(struct gfs2_glock *gl, const int ret)
414 {
415 struct gfs2_holder *gh, *tmp;
416
417 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
418 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
419 continue;
420 if (ret & LM_OUT_ERROR)
421 gh->gh_error = -EIO;
422 else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
423 gh->gh_error = GLR_TRYFAILED;
424 else
425 continue;
426 list_del_init(&gh->gh_list);
427 trace_gfs2_glock_queue(gh, 0);
428 gfs2_holder_wake(gh);
429 }
430 }
431
432 /**
433 * find_first_holder - find the first "holder" gh
434 * @gl: the glock
435 */
436
find_first_holder(const struct gfs2_glock * gl)437 static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
438 {
439 struct gfs2_holder *gh;
440
441 if (!list_empty(&gl->gl_holders)) {
442 gh = list_first_entry(&gl->gl_holders, struct gfs2_holder,
443 gh_list);
444 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
445 return gh;
446 }
447 return NULL;
448 }
449
450 /*
451 * gfs2_instantiate - Call the glops instantiate function
452 * @gh: The glock holder
453 *
454 * Returns: 0 if instantiate was successful, or error.
455 */
gfs2_instantiate(struct gfs2_holder * gh)456 int gfs2_instantiate(struct gfs2_holder *gh)
457 {
458 struct gfs2_glock *gl = gh->gh_gl;
459 const struct gfs2_glock_operations *glops = gl->gl_ops;
460 int ret;
461
462 again:
463 if (!test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags))
464 goto done;
465
466 /*
467 * Since we unlock the lockref lock, we set a flag to indicate
468 * instantiate is in progress.
469 */
470 if (test_and_set_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags)) {
471 wait_on_bit(&gl->gl_flags, GLF_INSTANTIATE_IN_PROG,
472 TASK_UNINTERRUPTIBLE);
473 /*
474 * Here we just waited for a different instantiate to finish.
475 * But that may not have been successful, as when a process
476 * locks an inode glock _before_ it has an actual inode to
477 * instantiate into. So we check again. This process might
478 * have an inode to instantiate, so might be successful.
479 */
480 goto again;
481 }
482
483 ret = glops->go_instantiate(gl);
484 if (!ret)
485 clear_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags);
486 clear_and_wake_up_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags);
487 if (ret)
488 return ret;
489
490 done:
491 if (glops->go_held)
492 return glops->go_held(gh);
493 return 0;
494 }
495
496 /**
497 * do_promote - promote as many requests as possible on the current queue
498 * @gl: The glock
499 *
500 * Returns true on success (i.e., progress was made or there are no waiters).
501 */
502
do_promote(struct gfs2_glock * gl)503 static bool do_promote(struct gfs2_glock *gl)
504 {
505 struct gfs2_holder *gh, *current_gh;
506
507 current_gh = find_first_holder(gl);
508 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
509 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
510 continue;
511 if (!may_grant(gl, current_gh, gh)) {
512 /*
513 * If we get here, it means we may not grant this
514 * holder for some reason. If this holder is at the
515 * head of the list, it means we have a blocked holder
516 * at the head, so return false.
517 */
518 if (list_is_first(&gh->gh_list, &gl->gl_holders))
519 return false;
520 do_error(gl, 0);
521 break;
522 }
523 set_bit(HIF_HOLDER, &gh->gh_iflags);
524 trace_gfs2_promote(gh);
525 gfs2_holder_wake(gh);
526 if (!current_gh)
527 current_gh = gh;
528 }
529 return true;
530 }
531
532 /**
533 * find_first_waiter - find the first gh that's waiting for the glock
534 * @gl: the glock
535 */
536
find_first_waiter(const struct gfs2_glock * gl)537 static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
538 {
539 struct gfs2_holder *gh;
540
541 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
542 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
543 return gh;
544 }
545 return NULL;
546 }
547
548 /**
549 * find_last_waiter - find the last gh that's waiting for the glock
550 * @gl: the glock
551 *
552 * This also is a fast way of finding out if there are any waiters.
553 */
554
find_last_waiter(const struct gfs2_glock * gl)555 static inline struct gfs2_holder *find_last_waiter(const struct gfs2_glock *gl)
556 {
557 struct gfs2_holder *gh;
558
559 if (list_empty(&gl->gl_holders))
560 return NULL;
561 gh = list_last_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
562 return test_bit(HIF_HOLDER, &gh->gh_iflags) ? NULL : gh;
563 }
564
565 /**
566 * state_change - record that the glock is now in a different state
567 * @gl: the glock
568 * @new_state: the new state
569 */
570
state_change(struct gfs2_glock * gl,unsigned int new_state)571 static void state_change(struct gfs2_glock *gl, unsigned int new_state)
572 {
573 int held1, held2;
574
575 held1 = (gl->gl_state != LM_ST_UNLOCKED);
576 held2 = (new_state != LM_ST_UNLOCKED);
577
578 if (held1 != held2) {
579 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
580 if (held2)
581 gl->gl_lockref.count++;
582 else
583 gl->gl_lockref.count--;
584 }
585 if (new_state != gl->gl_target)
586 /* shorten our minimum hold time */
587 gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
588 GL_GLOCK_MIN_HOLD);
589 gl->gl_state = new_state;
590 gl->gl_tchange = jiffies;
591 }
592
gfs2_set_demote(struct gfs2_glock * gl)593 static void gfs2_set_demote(struct gfs2_glock *gl)
594 {
595 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
596
597 set_bit(GLF_DEMOTE, &gl->gl_flags);
598 smp_mb();
599 wake_up(&sdp->sd_async_glock_wait);
600 }
601
gfs2_demote_wake(struct gfs2_glock * gl)602 static void gfs2_demote_wake(struct gfs2_glock *gl)
603 {
604 gl->gl_demote_state = LM_ST_EXCLUSIVE;
605 clear_bit(GLF_DEMOTE, &gl->gl_flags);
606 smp_mb__after_atomic();
607 wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
608 }
609
610 /**
611 * finish_xmote - The DLM has replied to one of our lock requests
612 * @gl: The glock
613 * @ret: The status from the DLM
614 *
615 */
616
finish_xmote(struct gfs2_glock * gl,unsigned int ret)617 static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
618 {
619 const struct gfs2_glock_operations *glops = gl->gl_ops;
620 struct gfs2_holder *gh;
621 unsigned state = ret & LM_OUT_ST_MASK;
622
623 trace_gfs2_glock_state_change(gl, state);
624 state_change(gl, state);
625 gh = find_first_waiter(gl);
626
627 /* Demote to UN request arrived during demote to SH or DF */
628 if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
629 state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED)
630 gl->gl_target = LM_ST_UNLOCKED;
631
632 /* Check for state != intended state */
633 if (unlikely(state != gl->gl_target)) {
634 if (gh && (ret & LM_OUT_CANCELED))
635 gfs2_holder_wake(gh);
636 if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
637 /* move to back of queue and try next entry */
638 if (ret & LM_OUT_CANCELED) {
639 list_move_tail(&gh->gh_list, &gl->gl_holders);
640 gh = find_first_waiter(gl);
641 gl->gl_target = gh->gh_state;
642 if (do_promote(gl))
643 goto out;
644 goto retry;
645 }
646 /* Some error or failed "try lock" - report it */
647 if ((ret & LM_OUT_ERROR) ||
648 (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
649 gl->gl_target = gl->gl_state;
650 do_error(gl, ret);
651 goto out;
652 }
653 }
654 switch(state) {
655 /* Unlocked due to conversion deadlock, try again */
656 case LM_ST_UNLOCKED:
657 retry:
658 do_xmote(gl, gh, gl->gl_target);
659 break;
660 /* Conversion fails, unlock and try again */
661 case LM_ST_SHARED:
662 case LM_ST_DEFERRED:
663 do_xmote(gl, gh, LM_ST_UNLOCKED);
664 break;
665 default: /* Everything else */
666 fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n",
667 gl->gl_target, state);
668 GLOCK_BUG_ON(gl, 1);
669 }
670 return;
671 }
672
673 /* Fast path - we got what we asked for */
674 if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags))
675 gfs2_demote_wake(gl);
676 if (state != LM_ST_UNLOCKED) {
677 if (glops->go_xmote_bh) {
678 int rv;
679
680 spin_unlock(&gl->gl_lockref.lock);
681 rv = glops->go_xmote_bh(gl);
682 spin_lock(&gl->gl_lockref.lock);
683 if (rv) {
684 do_error(gl, rv);
685 goto out;
686 }
687 }
688 do_promote(gl);
689 }
690 out:
691 clear_bit(GLF_LOCK, &gl->gl_flags);
692 }
693
is_system_glock(struct gfs2_glock * gl)694 static bool is_system_glock(struct gfs2_glock *gl)
695 {
696 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
697 struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
698
699 if (gl == m_ip->i_gl)
700 return true;
701 return false;
702 }
703
704 /**
705 * do_xmote - Calls the DLM to change the state of a lock
706 * @gl: The lock state
707 * @gh: The holder (only for promotes)
708 * @target: The target lock state
709 *
710 */
711
do_xmote(struct gfs2_glock * gl,struct gfs2_holder * gh,unsigned int target)712 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh,
713 unsigned int target)
714 __releases(&gl->gl_lockref.lock)
715 __acquires(&gl->gl_lockref.lock)
716 {
717 const struct gfs2_glock_operations *glops = gl->gl_ops;
718 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
719 struct lm_lockstruct *ls = &sdp->sd_lockstruct;
720 unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
721 int ret;
722
723 if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) &&
724 gh && !(gh->gh_flags & LM_FLAG_NOEXP))
725 goto skip_inval;
726
727 lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP);
728 GLOCK_BUG_ON(gl, gl->gl_state == target);
729 GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
730 if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
731 glops->go_inval) {
732 /*
733 * If another process is already doing the invalidate, let that
734 * finish first. The glock state machine will get back to this
735 * holder again later.
736 */
737 if (test_and_set_bit(GLF_INVALIDATE_IN_PROGRESS,
738 &gl->gl_flags))
739 return;
740 do_error(gl, 0); /* Fail queued try locks */
741 }
742 gl->gl_req = target;
743 set_bit(GLF_BLOCKING, &gl->gl_flags);
744 if ((gl->gl_req == LM_ST_UNLOCKED) ||
745 (gl->gl_state == LM_ST_EXCLUSIVE) ||
746 (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
747 clear_bit(GLF_BLOCKING, &gl->gl_flags);
748 if (!glops->go_inval && !glops->go_sync)
749 goto skip_inval;
750
751 spin_unlock(&gl->gl_lockref.lock);
752 if (glops->go_sync) {
753 ret = glops->go_sync(gl);
754 /* If we had a problem syncing (due to io errors or whatever,
755 * we should not invalidate the metadata or tell dlm to
756 * release the glock to other nodes.
757 */
758 if (ret) {
759 if (cmpxchg(&sdp->sd_log_error, 0, ret)) {
760 fs_err(sdp, "Error %d syncing glock \n", ret);
761 gfs2_dump_glock(NULL, gl, true);
762 }
763 spin_lock(&gl->gl_lockref.lock);
764 goto skip_inval;
765 }
766 }
767 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) {
768 /*
769 * The call to go_sync should have cleared out the ail list.
770 * If there are still items, we have a problem. We ought to
771 * withdraw, but we can't because the withdraw code also uses
772 * glocks. Warn about the error, dump the glock, then fall
773 * through and wait for logd to do the withdraw for us.
774 */
775 if ((atomic_read(&gl->gl_ail_count) != 0) &&
776 (!cmpxchg(&sdp->sd_log_error, 0, -EIO))) {
777 gfs2_glock_assert_warn(gl,
778 !atomic_read(&gl->gl_ail_count));
779 gfs2_dump_glock(NULL, gl, true);
780 }
781 glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
782 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
783 }
784 spin_lock(&gl->gl_lockref.lock);
785
786 skip_inval:
787 gl->gl_lockref.count++;
788 /*
789 * Check for an error encountered since we called go_sync and go_inval.
790 * If so, we can't withdraw from the glock code because the withdraw
791 * code itself uses glocks (see function signal_our_withdraw) to
792 * change the mount to read-only. Most importantly, we must not call
793 * dlm to unlock the glock until the journal is in a known good state
794 * (after journal replay) otherwise other nodes may use the object
795 * (rgrp or dinode) and then later, journal replay will corrupt the
796 * file system. The best we can do here is wait for the logd daemon
797 * to see sd_log_error and withdraw, and in the meantime, requeue the
798 * work for later.
799 *
800 * We make a special exception for some system glocks, such as the
801 * system statfs inode glock, which needs to be granted before the
802 * gfs2_quotad daemon can exit, and that exit needs to finish before
803 * we can unmount the withdrawn file system.
804 *
805 * However, if we're just unlocking the lock (say, for unmount, when
806 * gfs2_gl_hash_clear calls clear_glock) and recovery is complete
807 * then it's okay to tell dlm to unlock it.
808 */
809 if (unlikely(sdp->sd_log_error) && !gfs2_withdrawing_or_withdrawn(sdp))
810 gfs2_withdraw_delayed(sdp);
811 if (glock_blocked_by_withdraw(gl) &&
812 (target != LM_ST_UNLOCKED ||
813 test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags))) {
814 if (!is_system_glock(gl)) {
815 handle_callback(gl, LM_ST_UNLOCKED, 0, false); /* sets demote */
816 /*
817 * Ordinarily, we would call dlm and its callback would call
818 * finish_xmote, which would call state_change() to the new state.
819 * Since we withdrew, we won't call dlm, so call state_change
820 * manually, but to the UNLOCKED state we desire.
821 */
822 state_change(gl, LM_ST_UNLOCKED);
823 /*
824 * We skip telling dlm to do the locking, so we won't get a
825 * reply that would otherwise clear GLF_LOCK. So we clear it here.
826 */
827 clear_bit(GLF_LOCK, &gl->gl_flags);
828 clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
829 gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD);
830 return;
831 } else {
832 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
833 }
834 }
835
836 if (ls->ls_ops->lm_lock) {
837 spin_unlock(&gl->gl_lockref.lock);
838 ret = ls->ls_ops->lm_lock(gl, target, lck_flags);
839 spin_lock(&gl->gl_lockref.lock);
840
841 if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
842 target == LM_ST_UNLOCKED &&
843 test_bit(DFL_UNMOUNT, &ls->ls_recover_flags)) {
844 /*
845 * The lockspace has been released and the lock has
846 * been unlocked implicitly.
847 */
848 } else if (ret) {
849 fs_err(sdp, "lm_lock ret %d\n", ret);
850 target = gl->gl_state | LM_OUT_ERROR;
851 } else {
852 /* The operation will be completed asynchronously. */
853 return;
854 }
855 }
856
857 /* Complete the operation now. */
858 finish_xmote(gl, target);
859 gfs2_glock_queue_work(gl, 0);
860 }
861
862 /**
863 * run_queue - do all outstanding tasks related to a glock
864 * @gl: The glock in question
865 * @nonblock: True if we must not block in run_queue
866 *
867 */
868
run_queue(struct gfs2_glock * gl,const int nonblock)869 static void run_queue(struct gfs2_glock *gl, const int nonblock)
870 __releases(&gl->gl_lockref.lock)
871 __acquires(&gl->gl_lockref.lock)
872 {
873 struct gfs2_holder *gh = NULL;
874
875 if (test_bit(GLF_LOCK, &gl->gl_flags))
876 return;
877 set_bit(GLF_LOCK, &gl->gl_flags);
878
879 GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
880
881 if (test_bit(GLF_DEMOTE, &gl->gl_flags) &&
882 gl->gl_demote_state != gl->gl_state) {
883 if (find_first_holder(gl))
884 goto out_unlock;
885 if (nonblock)
886 goto out_sched;
887 set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
888 GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
889 gl->gl_target = gl->gl_demote_state;
890 } else {
891 if (test_bit(GLF_DEMOTE, &gl->gl_flags))
892 gfs2_demote_wake(gl);
893 if (do_promote(gl))
894 goto out_unlock;
895 gh = find_first_waiter(gl);
896 gl->gl_target = gh->gh_state;
897 if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
898 do_error(gl, 0); /* Fail queued try locks */
899 }
900 do_xmote(gl, gh, gl->gl_target);
901 return;
902
903 out_sched:
904 clear_bit(GLF_LOCK, &gl->gl_flags);
905 smp_mb__after_atomic();
906 gl->gl_lockref.count++;
907 gfs2_glock_queue_work(gl, 0);
908 return;
909
910 out_unlock:
911 clear_bit(GLF_LOCK, &gl->gl_flags);
912 smp_mb__after_atomic();
913 return;
914 }
915
916 /**
917 * glock_set_object - set the gl_object field of a glock
918 * @gl: the glock
919 * @object: the object
920 */
glock_set_object(struct gfs2_glock * gl,void * object)921 void glock_set_object(struct gfs2_glock *gl, void *object)
922 {
923 void *prev_object;
924
925 spin_lock(&gl->gl_lockref.lock);
926 prev_object = gl->gl_object;
927 gl->gl_object = object;
928 spin_unlock(&gl->gl_lockref.lock);
929 if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == NULL)) {
930 pr_warn("glock=%u/%llx\n",
931 gl->gl_name.ln_type,
932 (unsigned long long)gl->gl_name.ln_number);
933 gfs2_dump_glock(NULL, gl, true);
934 }
935 }
936
937 /**
938 * glock_clear_object - clear the gl_object field of a glock
939 * @gl: the glock
940 * @object: object the glock currently points at
941 */
glock_clear_object(struct gfs2_glock * gl,void * object)942 void glock_clear_object(struct gfs2_glock *gl, void *object)
943 {
944 void *prev_object;
945
946 spin_lock(&gl->gl_lockref.lock);
947 prev_object = gl->gl_object;
948 gl->gl_object = NULL;
949 spin_unlock(&gl->gl_lockref.lock);
950 if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == object)) {
951 pr_warn("glock=%u/%llx\n",
952 gl->gl_name.ln_type,
953 (unsigned long long)gl->gl_name.ln_number);
954 gfs2_dump_glock(NULL, gl, true);
955 }
956 }
957
gfs2_inode_remember_delete(struct gfs2_glock * gl,u64 generation)958 void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation)
959 {
960 struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
961
962 if (ri->ri_magic == 0)
963 ri->ri_magic = cpu_to_be32(GFS2_MAGIC);
964 if (ri->ri_magic == cpu_to_be32(GFS2_MAGIC))
965 ri->ri_generation_deleted = cpu_to_be64(generation);
966 }
967
gfs2_inode_already_deleted(struct gfs2_glock * gl,u64 generation)968 bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation)
969 {
970 struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
971
972 if (ri->ri_magic != cpu_to_be32(GFS2_MAGIC))
973 return false;
974 return generation <= be64_to_cpu(ri->ri_generation_deleted);
975 }
976
gfs2_glock_poke(struct gfs2_glock * gl)977 static void gfs2_glock_poke(struct gfs2_glock *gl)
978 {
979 int flags = LM_FLAG_TRY_1CB | LM_FLAG_ANY | GL_SKIP;
980 struct gfs2_holder gh;
981 int error;
982
983 __gfs2_holder_init(gl, LM_ST_SHARED, flags, &gh, _RET_IP_);
984 error = gfs2_glock_nq(&gh);
985 if (!error)
986 gfs2_glock_dq(&gh);
987 gfs2_holder_uninit(&gh);
988 }
989
gfs2_try_evict(struct gfs2_glock * gl)990 static bool gfs2_try_evict(struct gfs2_glock *gl)
991 {
992 struct gfs2_inode *ip;
993 bool evicted = false;
994
995 /*
996 * If there is contention on the iopen glock and we have an inode, try
997 * to grab and release the inode so that it can be evicted. This will
998 * allow the remote node to go ahead and delete the inode without us
999 * having to do it, which will avoid rgrp glock thrashing.
1000 *
1001 * The remote node is likely still holding the corresponding inode
1002 * glock, so it will run before we get to verify that the delete has
1003 * happened below.
1004 */
1005 spin_lock(&gl->gl_lockref.lock);
1006 ip = gl->gl_object;
1007 if (ip && !igrab(&ip->i_inode))
1008 ip = NULL;
1009 spin_unlock(&gl->gl_lockref.lock);
1010 if (ip) {
1011 gl->gl_no_formal_ino = ip->i_no_formal_ino;
1012 set_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
1013 d_prune_aliases(&ip->i_inode);
1014 iput(&ip->i_inode);
1015
1016 /* If the inode was evicted, gl->gl_object will now be NULL. */
1017 spin_lock(&gl->gl_lockref.lock);
1018 ip = gl->gl_object;
1019 if (ip) {
1020 clear_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
1021 if (!igrab(&ip->i_inode))
1022 ip = NULL;
1023 }
1024 spin_unlock(&gl->gl_lockref.lock);
1025 if (ip) {
1026 gfs2_glock_poke(ip->i_gl);
1027 iput(&ip->i_inode);
1028 }
1029 evicted = !ip;
1030 }
1031 return evicted;
1032 }
1033
gfs2_queue_try_to_evict(struct gfs2_glock * gl)1034 bool gfs2_queue_try_to_evict(struct gfs2_glock *gl)
1035 {
1036 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1037
1038 if (test_and_set_bit(GLF_TRY_TO_EVICT, &gl->gl_flags))
1039 return false;
1040 return queue_delayed_work(sdp->sd_delete_wq,
1041 &gl->gl_delete, 0);
1042 }
1043
gfs2_queue_verify_evict(struct gfs2_glock * gl)1044 static bool gfs2_queue_verify_evict(struct gfs2_glock *gl)
1045 {
1046 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1047
1048 if (test_and_set_bit(GLF_VERIFY_EVICT, &gl->gl_flags))
1049 return false;
1050 return queue_delayed_work(sdp->sd_delete_wq,
1051 &gl->gl_delete, 5 * HZ);
1052 }
1053
delete_work_func(struct work_struct * work)1054 static void delete_work_func(struct work_struct *work)
1055 {
1056 struct delayed_work *dwork = to_delayed_work(work);
1057 struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete);
1058 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1059 struct inode *inode;
1060 u64 no_addr = gl->gl_name.ln_number;
1061
1062 if (test_and_clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags)) {
1063 /*
1064 * If we can evict the inode, give the remote node trying to
1065 * delete the inode some time before verifying that the delete
1066 * has happened. Otherwise, if we cause contention on the inode glock
1067 * immediately, the remote node will think that we still have
1068 * the inode in use, and so it will give up waiting.
1069 *
1070 * If we can't evict the inode, signal to the remote node that
1071 * the inode is still in use. We'll later try to delete the
1072 * inode locally in gfs2_evict_inode.
1073 *
1074 * FIXME: We only need to verify that the remote node has
1075 * deleted the inode because nodes before this remote delete
1076 * rework won't cooperate. At a later time, when we no longer
1077 * care about compatibility with such nodes, we can skip this
1078 * step entirely.
1079 */
1080 if (gfs2_try_evict(gl)) {
1081 if (test_bit(SDF_KILL, &sdp->sd_flags))
1082 goto out;
1083 if (gfs2_queue_verify_evict(gl))
1084 return;
1085 }
1086 goto out;
1087 }
1088
1089 if (test_and_clear_bit(GLF_VERIFY_EVICT, &gl->gl_flags)) {
1090 inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino,
1091 GFS2_BLKST_UNLINKED);
1092 if (IS_ERR(inode)) {
1093 if (PTR_ERR(inode) == -EAGAIN &&
1094 !test_bit(SDF_KILL, &sdp->sd_flags) &&
1095 gfs2_queue_verify_evict(gl))
1096 return;
1097 } else {
1098 d_prune_aliases(inode);
1099 iput(inode);
1100 }
1101 }
1102
1103 out:
1104 gfs2_glock_put(gl);
1105 }
1106
glock_work_func(struct work_struct * work)1107 static void glock_work_func(struct work_struct *work)
1108 {
1109 unsigned long delay = 0;
1110 struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
1111 unsigned int drop_refs = 1;
1112
1113 spin_lock(&gl->gl_lockref.lock);
1114 if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags)) {
1115 clear_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1116 finish_xmote(gl, gl->gl_reply);
1117 drop_refs++;
1118 }
1119 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1120 gl->gl_state != LM_ST_UNLOCKED &&
1121 gl->gl_demote_state != LM_ST_EXCLUSIVE) {
1122 unsigned long holdtime, now = jiffies;
1123
1124 holdtime = gl->gl_tchange + gl->gl_hold_time;
1125 if (time_before(now, holdtime))
1126 delay = holdtime - now;
1127
1128 if (!delay) {
1129 clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1130 gfs2_set_demote(gl);
1131 }
1132 }
1133 run_queue(gl, 0);
1134 if (delay) {
1135 /* Keep one glock reference for the work we requeue. */
1136 drop_refs--;
1137 if (gl->gl_name.ln_type != LM_TYPE_INODE)
1138 delay = 0;
1139 gfs2_glock_queue_work(gl, delay);
1140 }
1141
1142 /*
1143 * Drop the remaining glock references manually here. (Mind that
1144 * gfs2_glock_queue_work depends on the lockref spinlock begin held
1145 * here as well.)
1146 */
1147 gl->gl_lockref.count -= drop_refs;
1148 if (!gl->gl_lockref.count) {
1149 __gfs2_glock_put(gl);
1150 return;
1151 }
1152 spin_unlock(&gl->gl_lockref.lock);
1153 }
1154
find_insert_glock(struct lm_lockname * name,struct gfs2_glock * new)1155 static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
1156 struct gfs2_glock *new)
1157 {
1158 struct wait_glock_queue wait;
1159 wait_queue_head_t *wq = glock_waitqueue(name);
1160 struct gfs2_glock *gl;
1161
1162 wait.name = name;
1163 init_wait(&wait.wait);
1164 wait.wait.func = glock_wake_function;
1165
1166 again:
1167 prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
1168 rcu_read_lock();
1169 if (new) {
1170 gl = rhashtable_lookup_get_insert_fast(&gl_hash_table,
1171 &new->gl_node, ht_parms);
1172 if (IS_ERR(gl))
1173 goto out;
1174 } else {
1175 gl = rhashtable_lookup_fast(&gl_hash_table,
1176 name, ht_parms);
1177 }
1178 if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
1179 rcu_read_unlock();
1180 schedule();
1181 goto again;
1182 }
1183 out:
1184 rcu_read_unlock();
1185 finish_wait(wq, &wait.wait);
1186 return gl;
1187 }
1188
1189 /**
1190 * gfs2_glock_get() - Get a glock, or create one if one doesn't exist
1191 * @sdp: The GFS2 superblock
1192 * @number: the lock number
1193 * @glops: The glock_operations to use
1194 * @create: If 0, don't create the glock if it doesn't exist
1195 * @glp: the glock is returned here
1196 *
1197 * This does not lock a glock, just finds/creates structures for one.
1198 *
1199 * Returns: errno
1200 */
1201
gfs2_glock_get(struct gfs2_sbd * sdp,u64 number,const struct gfs2_glock_operations * glops,int create,struct gfs2_glock ** glp)1202 int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
1203 const struct gfs2_glock_operations *glops, int create,
1204 struct gfs2_glock **glp)
1205 {
1206 struct super_block *s = sdp->sd_vfs;
1207 struct lm_lockname name = { .ln_number = number,
1208 .ln_type = glops->go_type,
1209 .ln_sbd = sdp };
1210 struct gfs2_glock *gl, *tmp;
1211 struct address_space *mapping;
1212 int ret = 0;
1213
1214 gl = find_insert_glock(&name, NULL);
1215 if (gl) {
1216 *glp = gl;
1217 return 0;
1218 }
1219 if (!create)
1220 return -ENOENT;
1221
1222 if (glops->go_flags & GLOF_ASPACE) {
1223 struct gfs2_glock_aspace *gla =
1224 kmem_cache_alloc(gfs2_glock_aspace_cachep, GFP_NOFS);
1225 if (!gla)
1226 return -ENOMEM;
1227 gl = &gla->glock;
1228 } else {
1229 gl = kmem_cache_alloc(gfs2_glock_cachep, GFP_NOFS);
1230 if (!gl)
1231 return -ENOMEM;
1232 }
1233 memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
1234 gl->gl_ops = glops;
1235
1236 if (glops->go_flags & GLOF_LVB) {
1237 gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS);
1238 if (!gl->gl_lksb.sb_lvbptr) {
1239 gfs2_glock_dealloc(&gl->gl_rcu);
1240 return -ENOMEM;
1241 }
1242 }
1243
1244 atomic_inc(&sdp->sd_glock_disposal);
1245 gl->gl_node.next = NULL;
1246 gl->gl_flags = glops->go_instantiate ? BIT(GLF_INSTANTIATE_NEEDED) : 0;
1247 gl->gl_name = name;
1248 lockdep_set_subclass(&gl->gl_lockref.lock, glops->go_subclass);
1249 gl->gl_lockref.count = 1;
1250 gl->gl_state = LM_ST_UNLOCKED;
1251 gl->gl_target = LM_ST_UNLOCKED;
1252 gl->gl_demote_state = LM_ST_EXCLUSIVE;
1253 gl->gl_dstamp = 0;
1254 preempt_disable();
1255 /* We use the global stats to estimate the initial per-glock stats */
1256 gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
1257 preempt_enable();
1258 gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
1259 gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
1260 gl->gl_tchange = jiffies;
1261 gl->gl_object = NULL;
1262 gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
1263 INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
1264 if (gl->gl_name.ln_type == LM_TYPE_IOPEN)
1265 INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func);
1266
1267 mapping = gfs2_glock2aspace(gl);
1268 if (mapping) {
1269 mapping->a_ops = &gfs2_meta_aops;
1270 mapping->host = s->s_bdev->bd_mapping->host;
1271 mapping->flags = 0;
1272 mapping_set_gfp_mask(mapping, GFP_NOFS);
1273 mapping->i_private_data = NULL;
1274 mapping->writeback_index = 0;
1275 }
1276
1277 tmp = find_insert_glock(&name, gl);
1278 if (!tmp) {
1279 *glp = gl;
1280 goto out;
1281 }
1282 if (IS_ERR(tmp)) {
1283 ret = PTR_ERR(tmp);
1284 goto out_free;
1285 }
1286 *glp = tmp;
1287
1288 out_free:
1289 gfs2_glock_dealloc(&gl->gl_rcu);
1290 if (atomic_dec_and_test(&sdp->sd_glock_disposal))
1291 wake_up(&sdp->sd_kill_wait);
1292
1293 out:
1294 return ret;
1295 }
1296
1297 /**
1298 * __gfs2_holder_init - initialize a struct gfs2_holder in the default way
1299 * @gl: the glock
1300 * @state: the state we're requesting
1301 * @flags: the modifier flags
1302 * @gh: the holder structure
1303 *
1304 */
1305
__gfs2_holder_init(struct gfs2_glock * gl,unsigned int state,u16 flags,struct gfs2_holder * gh,unsigned long ip)1306 void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
1307 struct gfs2_holder *gh, unsigned long ip)
1308 {
1309 INIT_LIST_HEAD(&gh->gh_list);
1310 gh->gh_gl = gfs2_glock_hold(gl);
1311 gh->gh_ip = ip;
1312 gh->gh_owner_pid = get_pid(task_pid(current));
1313 gh->gh_state = state;
1314 gh->gh_flags = flags;
1315 gh->gh_iflags = 0;
1316 }
1317
1318 /**
1319 * gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it
1320 * @state: the state we're requesting
1321 * @flags: the modifier flags
1322 * @gh: the holder structure
1323 *
1324 * Don't mess with the glock.
1325 *
1326 */
1327
gfs2_holder_reinit(unsigned int state,u16 flags,struct gfs2_holder * gh)1328 void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
1329 {
1330 gh->gh_state = state;
1331 gh->gh_flags = flags;
1332 gh->gh_iflags = 0;
1333 gh->gh_ip = _RET_IP_;
1334 put_pid(gh->gh_owner_pid);
1335 gh->gh_owner_pid = get_pid(task_pid(current));
1336 }
1337
1338 /**
1339 * gfs2_holder_uninit - uninitialize a holder structure (drop glock reference)
1340 * @gh: the holder structure
1341 *
1342 */
1343
gfs2_holder_uninit(struct gfs2_holder * gh)1344 void gfs2_holder_uninit(struct gfs2_holder *gh)
1345 {
1346 put_pid(gh->gh_owner_pid);
1347 gfs2_glock_put(gh->gh_gl);
1348 gfs2_holder_mark_uninitialized(gh);
1349 gh->gh_ip = 0;
1350 }
1351
gfs2_glock_update_hold_time(struct gfs2_glock * gl,unsigned long start_time)1352 static void gfs2_glock_update_hold_time(struct gfs2_glock *gl,
1353 unsigned long start_time)
1354 {
1355 /* Have we waited longer that a second? */
1356 if (time_after(jiffies, start_time + HZ)) {
1357 /* Lengthen the minimum hold time. */
1358 gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR,
1359 GL_GLOCK_MAX_HOLD);
1360 }
1361 }
1362
1363 /**
1364 * gfs2_glock_holder_ready - holder is ready and its error code can be collected
1365 * @gh: the glock holder
1366 *
1367 * Called when a glock holder no longer needs to be waited for because it is
1368 * now either held (HIF_HOLDER set; gh_error == 0), or acquiring the lock has
1369 * failed (gh_error != 0).
1370 */
1371
gfs2_glock_holder_ready(struct gfs2_holder * gh)1372 int gfs2_glock_holder_ready(struct gfs2_holder *gh)
1373 {
1374 if (gh->gh_error || (gh->gh_flags & GL_SKIP))
1375 return gh->gh_error;
1376 gh->gh_error = gfs2_instantiate(gh);
1377 if (gh->gh_error)
1378 gfs2_glock_dq(gh);
1379 return gh->gh_error;
1380 }
1381
1382 /**
1383 * gfs2_glock_wait - wait on a glock acquisition
1384 * @gh: the glock holder
1385 *
1386 * Returns: 0 on success
1387 */
1388
gfs2_glock_wait(struct gfs2_holder * gh)1389 int gfs2_glock_wait(struct gfs2_holder *gh)
1390 {
1391 unsigned long start_time = jiffies;
1392
1393 might_sleep();
1394 wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1395 gfs2_glock_update_hold_time(gh->gh_gl, start_time);
1396 return gfs2_glock_holder_ready(gh);
1397 }
1398
glocks_pending(unsigned int num_gh,struct gfs2_holder * ghs)1399 static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
1400 {
1401 int i;
1402
1403 for (i = 0; i < num_gh; i++)
1404 if (test_bit(HIF_WAIT, &ghs[i].gh_iflags))
1405 return 1;
1406 return 0;
1407 }
1408
1409 /**
1410 * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions
1411 * @num_gh: the number of holders in the array
1412 * @ghs: the glock holder array
1413 *
1414 * Returns: 0 on success, meaning all glocks have been granted and are held.
1415 * -ESTALE if the request timed out, meaning all glocks were released,
1416 * and the caller should retry the operation.
1417 */
1418
gfs2_glock_async_wait(unsigned int num_gh,struct gfs2_holder * ghs)1419 int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs)
1420 {
1421 struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
1422 int i, ret = 0, timeout = 0;
1423 unsigned long start_time = jiffies;
1424
1425 might_sleep();
1426 /*
1427 * Total up the (minimum hold time * 2) of all glocks and use that to
1428 * determine the max amount of time we should wait.
1429 */
1430 for (i = 0; i < num_gh; i++)
1431 timeout += ghs[i].gh_gl->gl_hold_time << 1;
1432
1433 if (!wait_event_timeout(sdp->sd_async_glock_wait,
1434 !glocks_pending(num_gh, ghs), timeout)) {
1435 ret = -ESTALE; /* request timed out. */
1436 goto out;
1437 }
1438
1439 for (i = 0; i < num_gh; i++) {
1440 struct gfs2_holder *gh = &ghs[i];
1441 int ret2;
1442
1443 if (test_bit(HIF_HOLDER, &gh->gh_iflags)) {
1444 gfs2_glock_update_hold_time(gh->gh_gl,
1445 start_time);
1446 }
1447 ret2 = gfs2_glock_holder_ready(gh);
1448 if (!ret)
1449 ret = ret2;
1450 }
1451
1452 out:
1453 if (ret) {
1454 for (i = 0; i < num_gh; i++) {
1455 struct gfs2_holder *gh = &ghs[i];
1456
1457 gfs2_glock_dq(gh);
1458 }
1459 }
1460 return ret;
1461 }
1462
1463 /**
1464 * handle_callback - process a demote request
1465 * @gl: the glock
1466 * @state: the state the caller wants us to change to
1467 * @delay: zero to demote immediately; otherwise pending demote
1468 * @remote: true if this came from a different cluster node
1469 *
1470 * There are only two requests that we are going to see in actual
1471 * practise: LM_ST_SHARED and LM_ST_UNLOCKED
1472 */
1473
handle_callback(struct gfs2_glock * gl,unsigned int state,unsigned long delay,bool remote)1474 static void handle_callback(struct gfs2_glock *gl, unsigned int state,
1475 unsigned long delay, bool remote)
1476 {
1477 if (delay)
1478 set_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1479 else
1480 gfs2_set_demote(gl);
1481 if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
1482 gl->gl_demote_state = state;
1483 gl->gl_demote_time = jiffies;
1484 } else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
1485 gl->gl_demote_state != state) {
1486 gl->gl_demote_state = LM_ST_UNLOCKED;
1487 }
1488 if (gl->gl_ops->go_callback)
1489 gl->gl_ops->go_callback(gl, remote);
1490 trace_gfs2_demote_rq(gl, remote);
1491 }
1492
gfs2_print_dbg(struct seq_file * seq,const char * fmt,...)1493 void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
1494 {
1495 struct va_format vaf;
1496 va_list args;
1497
1498 va_start(args, fmt);
1499
1500 if (seq) {
1501 seq_vprintf(seq, fmt, args);
1502 } else {
1503 vaf.fmt = fmt;
1504 vaf.va = &args;
1505
1506 pr_err("%pV", &vaf);
1507 }
1508
1509 va_end(args);
1510 }
1511
pid_is_meaningful(const struct gfs2_holder * gh)1512 static inline bool pid_is_meaningful(const struct gfs2_holder *gh)
1513 {
1514 if (!(gh->gh_flags & GL_NOPID))
1515 return true;
1516 if (gh->gh_state == LM_ST_UNLOCKED)
1517 return true;
1518 return false;
1519 }
1520
1521 /**
1522 * add_to_queue - Add a holder to the wait queue (but look for recursion)
1523 * @gh: the holder structure to add
1524 *
1525 * Eventually we should move the recursive locking trap to a
1526 * debugging option or something like that. This is the fast
1527 * path and needs to have the minimum number of distractions.
1528 *
1529 */
1530
add_to_queue(struct gfs2_holder * gh)1531 static inline void add_to_queue(struct gfs2_holder *gh)
1532 __releases(&gl->gl_lockref.lock)
1533 __acquires(&gl->gl_lockref.lock)
1534 {
1535 struct gfs2_glock *gl = gh->gh_gl;
1536 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1537 struct list_head *insert_pt = NULL;
1538 struct gfs2_holder *gh2;
1539 int try_futile = 0;
1540
1541 GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
1542 if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))
1543 GLOCK_BUG_ON(gl, true);
1544
1545 if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
1546 if (test_bit(GLF_LOCK, &gl->gl_flags)) {
1547 struct gfs2_holder *current_gh;
1548
1549 current_gh = find_first_holder(gl);
1550 try_futile = !may_grant(gl, current_gh, gh);
1551 }
1552 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
1553 goto fail;
1554 }
1555
1556 list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1557 if (likely(gh2->gh_owner_pid != gh->gh_owner_pid))
1558 continue;
1559 if (gh->gh_gl->gl_ops->go_type == LM_TYPE_FLOCK)
1560 continue;
1561 if (!pid_is_meaningful(gh2))
1562 continue;
1563 goto trap_recursive;
1564 }
1565 list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1566 if (try_futile &&
1567 !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
1568 fail:
1569 gh->gh_error = GLR_TRYFAILED;
1570 gfs2_holder_wake(gh);
1571 return;
1572 }
1573 if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1574 continue;
1575 }
1576 trace_gfs2_glock_queue(gh, 1);
1577 gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT);
1578 gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT);
1579 if (likely(insert_pt == NULL)) {
1580 list_add_tail(&gh->gh_list, &gl->gl_holders);
1581 return;
1582 }
1583 list_add_tail(&gh->gh_list, insert_pt);
1584 spin_unlock(&gl->gl_lockref.lock);
1585 if (sdp->sd_lockstruct.ls_ops->lm_cancel)
1586 sdp->sd_lockstruct.ls_ops->lm_cancel(gl);
1587 spin_lock(&gl->gl_lockref.lock);
1588 return;
1589
1590 trap_recursive:
1591 fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1592 fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1593 fs_err(sdp, "lock type: %d req lock state : %d\n",
1594 gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1595 fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1596 fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1597 fs_err(sdp, "lock type: %d req lock state : %d\n",
1598 gh->gh_gl->gl_name.ln_type, gh->gh_state);
1599 gfs2_dump_glock(NULL, gl, true);
1600 BUG();
1601 }
1602
1603 /**
1604 * gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock)
1605 * @gh: the holder structure
1606 *
1607 * if (gh->gh_flags & GL_ASYNC), this never returns an error
1608 *
1609 * Returns: 0, GLR_TRYFAILED, or errno on failure
1610 */
1611
gfs2_glock_nq(struct gfs2_holder * gh)1612 int gfs2_glock_nq(struct gfs2_holder *gh)
1613 {
1614 struct gfs2_glock *gl = gh->gh_gl;
1615 int error;
1616
1617 if (glock_blocked_by_withdraw(gl) && !(gh->gh_flags & LM_FLAG_NOEXP))
1618 return -EIO;
1619
1620 if (gh->gh_flags & GL_NOBLOCK) {
1621 struct gfs2_holder *current_gh;
1622
1623 error = -ECHILD;
1624 spin_lock(&gl->gl_lockref.lock);
1625 if (find_last_waiter(gl))
1626 goto unlock;
1627 current_gh = find_first_holder(gl);
1628 if (!may_grant(gl, current_gh, gh))
1629 goto unlock;
1630 set_bit(HIF_HOLDER, &gh->gh_iflags);
1631 list_add_tail(&gh->gh_list, &gl->gl_holders);
1632 trace_gfs2_promote(gh);
1633 error = 0;
1634 unlock:
1635 spin_unlock(&gl->gl_lockref.lock);
1636 return error;
1637 }
1638
1639 if (test_bit(GLF_LRU, &gl->gl_flags))
1640 gfs2_glock_remove_from_lru(gl);
1641
1642 gh->gh_error = 0;
1643 spin_lock(&gl->gl_lockref.lock);
1644 add_to_queue(gh);
1645 if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&
1646 test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) {
1647 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1648 gl->gl_lockref.count++;
1649 gfs2_glock_queue_work(gl, 0);
1650 }
1651 run_queue(gl, 1);
1652 spin_unlock(&gl->gl_lockref.lock);
1653
1654 error = 0;
1655 if (!(gh->gh_flags & GL_ASYNC))
1656 error = gfs2_glock_wait(gh);
1657
1658 return error;
1659 }
1660
1661 /**
1662 * gfs2_glock_poll - poll to see if an async request has been completed
1663 * @gh: the holder
1664 *
1665 * Returns: 1 if the request is ready to be gfs2_glock_wait()ed on
1666 */
1667
gfs2_glock_poll(struct gfs2_holder * gh)1668 int gfs2_glock_poll(struct gfs2_holder *gh)
1669 {
1670 return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1671 }
1672
needs_demote(struct gfs2_glock * gl)1673 static inline bool needs_demote(struct gfs2_glock *gl)
1674 {
1675 return (test_bit(GLF_DEMOTE, &gl->gl_flags) ||
1676 test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags));
1677 }
1678
__gfs2_glock_dq(struct gfs2_holder * gh)1679 static void __gfs2_glock_dq(struct gfs2_holder *gh)
1680 {
1681 struct gfs2_glock *gl = gh->gh_gl;
1682 unsigned delay = 0;
1683 int fast_path = 0;
1684
1685 /*
1686 * This holder should not be cached, so mark it for demote.
1687 * Note: this should be done before the check for needs_demote
1688 * below.
1689 */
1690 if (gh->gh_flags & GL_NOCACHE)
1691 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1692
1693 list_del_init(&gh->gh_list);
1694 clear_bit(HIF_HOLDER, &gh->gh_iflags);
1695 trace_gfs2_glock_queue(gh, 0);
1696
1697 /*
1698 * If there hasn't been a demote request we are done.
1699 * (Let the remaining holders, if any, keep holding it.)
1700 */
1701 if (!needs_demote(gl)) {
1702 if (list_empty(&gl->gl_holders))
1703 fast_path = 1;
1704 }
1705
1706 if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
1707 gfs2_glock_add_to_lru(gl);
1708
1709 if (unlikely(!fast_path)) {
1710 gl->gl_lockref.count++;
1711 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1712 !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1713 gl->gl_name.ln_type == LM_TYPE_INODE)
1714 delay = gl->gl_hold_time;
1715 gfs2_glock_queue_work(gl, delay);
1716 }
1717 }
1718
1719 /**
1720 * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
1721 * @gh: the glock holder
1722 *
1723 */
gfs2_glock_dq(struct gfs2_holder * gh)1724 void gfs2_glock_dq(struct gfs2_holder *gh)
1725 {
1726 struct gfs2_glock *gl = gh->gh_gl;
1727 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1728
1729 spin_lock(&gl->gl_lockref.lock);
1730 if (!gfs2_holder_queued(gh)) {
1731 /*
1732 * May have already been dequeued because the locking request
1733 * was GL_ASYNC and it has failed in the meantime.
1734 */
1735 goto out;
1736 }
1737
1738 if (list_is_first(&gh->gh_list, &gl->gl_holders) &&
1739 !test_bit(HIF_HOLDER, &gh->gh_iflags)) {
1740 spin_unlock(&gl->gl_lockref.lock);
1741 gl->gl_name.ln_sbd->sd_lockstruct.ls_ops->lm_cancel(gl);
1742 wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1743 spin_lock(&gl->gl_lockref.lock);
1744 }
1745
1746 /*
1747 * If we're in the process of file system withdraw, we cannot just
1748 * dequeue any glocks until our journal is recovered, lest we introduce
1749 * file system corruption. We need two exceptions to this rule: We need
1750 * to allow unlocking of nondisk glocks and the glock for our own
1751 * journal that needs recovery.
1752 */
1753 if (test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags) &&
1754 glock_blocked_by_withdraw(gl) &&
1755 gh->gh_gl != sdp->sd_jinode_gl) {
1756 sdp->sd_glock_dqs_held++;
1757 spin_unlock(&gl->gl_lockref.lock);
1758 might_sleep();
1759 wait_on_bit(&sdp->sd_flags, SDF_WITHDRAW_RECOVERY,
1760 TASK_UNINTERRUPTIBLE);
1761 spin_lock(&gl->gl_lockref.lock);
1762 }
1763
1764 __gfs2_glock_dq(gh);
1765 out:
1766 spin_unlock(&gl->gl_lockref.lock);
1767 }
1768
gfs2_glock_dq_wait(struct gfs2_holder * gh)1769 void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1770 {
1771 struct gfs2_glock *gl = gh->gh_gl;
1772 gfs2_glock_dq(gh);
1773 might_sleep();
1774 wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1775 }
1776
1777 /**
1778 * gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it
1779 * @gh: the holder structure
1780 *
1781 */
1782
gfs2_glock_dq_uninit(struct gfs2_holder * gh)1783 void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1784 {
1785 gfs2_glock_dq(gh);
1786 gfs2_holder_uninit(gh);
1787 }
1788
1789 /**
1790 * gfs2_glock_nq_num - acquire a glock based on lock number
1791 * @sdp: the filesystem
1792 * @number: the lock number
1793 * @glops: the glock operations for the type of glock
1794 * @state: the state to acquire the glock in
1795 * @flags: modifier flags for the acquisition
1796 * @gh: the struct gfs2_holder
1797 *
1798 * Returns: errno
1799 */
1800
gfs2_glock_nq_num(struct gfs2_sbd * sdp,u64 number,const struct gfs2_glock_operations * glops,unsigned int state,u16 flags,struct gfs2_holder * gh)1801 int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1802 const struct gfs2_glock_operations *glops,
1803 unsigned int state, u16 flags, struct gfs2_holder *gh)
1804 {
1805 struct gfs2_glock *gl;
1806 int error;
1807
1808 error = gfs2_glock_get(sdp, number, glops, CREATE, &gl);
1809 if (!error) {
1810 error = gfs2_glock_nq_init(gl, state, flags, gh);
1811 gfs2_glock_put(gl);
1812 }
1813
1814 return error;
1815 }
1816
1817 /**
1818 * glock_compare - Compare two struct gfs2_glock structures for sorting
1819 * @arg_a: the first structure
1820 * @arg_b: the second structure
1821 *
1822 */
1823
glock_compare(const void * arg_a,const void * arg_b)1824 static int glock_compare(const void *arg_a, const void *arg_b)
1825 {
1826 const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1827 const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1828 const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1829 const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1830
1831 if (a->ln_number > b->ln_number)
1832 return 1;
1833 if (a->ln_number < b->ln_number)
1834 return -1;
1835 BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1836 return 0;
1837 }
1838
1839 /**
1840 * nq_m_sync - synchronously acquire more than one glock in deadlock free order
1841 * @num_gh: the number of structures
1842 * @ghs: an array of struct gfs2_holder structures
1843 * @p: placeholder for the holder structure to pass back
1844 *
1845 * Returns: 0 on success (all glocks acquired),
1846 * errno on failure (no glocks acquired)
1847 */
1848
nq_m_sync(unsigned int num_gh,struct gfs2_holder * ghs,struct gfs2_holder ** p)1849 static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1850 struct gfs2_holder **p)
1851 {
1852 unsigned int x;
1853 int error = 0;
1854
1855 for (x = 0; x < num_gh; x++)
1856 p[x] = &ghs[x];
1857
1858 sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL);
1859
1860 for (x = 0; x < num_gh; x++) {
1861 error = gfs2_glock_nq(p[x]);
1862 if (error) {
1863 while (x--)
1864 gfs2_glock_dq(p[x]);
1865 break;
1866 }
1867 }
1868
1869 return error;
1870 }
1871
1872 /**
1873 * gfs2_glock_nq_m - acquire multiple glocks
1874 * @num_gh: the number of structures
1875 * @ghs: an array of struct gfs2_holder structures
1876 *
1877 * Returns: 0 on success (all glocks acquired),
1878 * errno on failure (no glocks acquired)
1879 */
1880
gfs2_glock_nq_m(unsigned int num_gh,struct gfs2_holder * ghs)1881 int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1882 {
1883 struct gfs2_holder *tmp[4];
1884 struct gfs2_holder **pph = tmp;
1885 int error = 0;
1886
1887 switch(num_gh) {
1888 case 0:
1889 return 0;
1890 case 1:
1891 return gfs2_glock_nq(ghs);
1892 default:
1893 if (num_gh <= 4)
1894 break;
1895 pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *),
1896 GFP_NOFS);
1897 if (!pph)
1898 return -ENOMEM;
1899 }
1900
1901 error = nq_m_sync(num_gh, ghs, pph);
1902
1903 if (pph != tmp)
1904 kfree(pph);
1905
1906 return error;
1907 }
1908
1909 /**
1910 * gfs2_glock_dq_m - release multiple glocks
1911 * @num_gh: the number of structures
1912 * @ghs: an array of struct gfs2_holder structures
1913 *
1914 */
1915
gfs2_glock_dq_m(unsigned int num_gh,struct gfs2_holder * ghs)1916 void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1917 {
1918 while (num_gh--)
1919 gfs2_glock_dq(&ghs[num_gh]);
1920 }
1921
gfs2_glock_cb(struct gfs2_glock * gl,unsigned int state)1922 void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1923 {
1924 unsigned long delay = 0;
1925 unsigned long holdtime;
1926 unsigned long now = jiffies;
1927
1928 gfs2_glock_hold(gl);
1929 spin_lock(&gl->gl_lockref.lock);
1930 holdtime = gl->gl_tchange + gl->gl_hold_time;
1931 if (!list_empty(&gl->gl_holders) &&
1932 gl->gl_name.ln_type == LM_TYPE_INODE) {
1933 if (time_before(now, holdtime))
1934 delay = holdtime - now;
1935 if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
1936 delay = gl->gl_hold_time;
1937 }
1938 handle_callback(gl, state, delay, true);
1939 gfs2_glock_queue_work(gl, delay);
1940 spin_unlock(&gl->gl_lockref.lock);
1941 }
1942
1943 /**
1944 * gfs2_should_freeze - Figure out if glock should be frozen
1945 * @gl: The glock in question
1946 *
1947 * Glocks are not frozen if (a) the result of the dlm operation is
1948 * an error, (b) the locking operation was an unlock operation or
1949 * (c) if there is a "noexp" flagged request anywhere in the queue
1950 *
1951 * Returns: 1 if freezing should occur, 0 otherwise
1952 */
1953
gfs2_should_freeze(const struct gfs2_glock * gl)1954 static int gfs2_should_freeze(const struct gfs2_glock *gl)
1955 {
1956 const struct gfs2_holder *gh;
1957
1958 if (gl->gl_reply & ~LM_OUT_ST_MASK)
1959 return 0;
1960 if (gl->gl_target == LM_ST_UNLOCKED)
1961 return 0;
1962
1963 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1964 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1965 continue;
1966 if (LM_FLAG_NOEXP & gh->gh_flags)
1967 return 0;
1968 }
1969
1970 return 1;
1971 }
1972
1973 /**
1974 * gfs2_glock_complete - Callback used by locking
1975 * @gl: Pointer to the glock
1976 * @ret: The return value from the dlm
1977 *
1978 * The gl_reply field is under the gl_lockref.lock lock so that it is ok
1979 * to use a bitfield shared with other glock state fields.
1980 */
1981
gfs2_glock_complete(struct gfs2_glock * gl,int ret)1982 void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1983 {
1984 struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1985
1986 spin_lock(&gl->gl_lockref.lock);
1987 gl->gl_reply = ret;
1988
1989 if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1990 if (gfs2_should_freeze(gl)) {
1991 set_bit(GLF_FROZEN, &gl->gl_flags);
1992 spin_unlock(&gl->gl_lockref.lock);
1993 return;
1994 }
1995 }
1996
1997 gl->gl_lockref.count++;
1998 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1999 gfs2_glock_queue_work(gl, 0);
2000 spin_unlock(&gl->gl_lockref.lock);
2001 }
2002
glock_cmp(void * priv,const struct list_head * a,const struct list_head * b)2003 static int glock_cmp(void *priv, const struct list_head *a,
2004 const struct list_head *b)
2005 {
2006 struct gfs2_glock *gla, *glb;
2007
2008 gla = list_entry(a, struct gfs2_glock, gl_lru);
2009 glb = list_entry(b, struct gfs2_glock, gl_lru);
2010
2011 if (gla->gl_name.ln_number > glb->gl_name.ln_number)
2012 return 1;
2013 if (gla->gl_name.ln_number < glb->gl_name.ln_number)
2014 return -1;
2015
2016 return 0;
2017 }
2018
can_free_glock(struct gfs2_glock * gl)2019 static bool can_free_glock(struct gfs2_glock *gl)
2020 {
2021 bool held = gl->gl_state != LM_ST_UNLOCKED;
2022
2023 return !test_bit(GLF_LOCK, &gl->gl_flags) &&
2024 gl->gl_lockref.count == held;
2025 }
2026
2027 /**
2028 * gfs2_dispose_glock_lru - Demote a list of glocks
2029 * @list: The list to dispose of
2030 *
2031 * Disposing of glocks may involve disk accesses, so that here we sort
2032 * the glocks by number (i.e. disk location of the inodes) so that if
2033 * there are any such accesses, they'll be sent in order (mostly).
2034 *
2035 * Must be called under the lru_lock, but may drop and retake this
2036 * lock. While the lru_lock is dropped, entries may vanish from the
2037 * list, but no new entries will appear on the list (since it is
2038 * private)
2039 */
2040
gfs2_dispose_glock_lru(struct list_head * list)2041 static unsigned long gfs2_dispose_glock_lru(struct list_head *list)
2042 __releases(&lru_lock)
2043 __acquires(&lru_lock)
2044 {
2045 struct gfs2_glock *gl;
2046 unsigned long freed = 0;
2047
2048 list_sort(NULL, list, glock_cmp);
2049
2050 while(!list_empty(list)) {
2051 gl = list_first_entry(list, struct gfs2_glock, gl_lru);
2052 if (!spin_trylock(&gl->gl_lockref.lock)) {
2053 add_back_to_lru:
2054 list_move(&gl->gl_lru, &lru_list);
2055 continue;
2056 }
2057 if (!can_free_glock(gl)) {
2058 spin_unlock(&gl->gl_lockref.lock);
2059 goto add_back_to_lru;
2060 }
2061 list_del_init(&gl->gl_lru);
2062 atomic_dec(&lru_count);
2063 clear_bit(GLF_LRU, &gl->gl_flags);
2064 freed++;
2065 gl->gl_lockref.count++;
2066 if (demote_ok(gl))
2067 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
2068 gfs2_glock_queue_work(gl, 0);
2069 spin_unlock(&gl->gl_lockref.lock);
2070 cond_resched_lock(&lru_lock);
2071 }
2072 return freed;
2073 }
2074
2075 /**
2076 * gfs2_scan_glock_lru - Scan the LRU looking for locks to demote
2077 * @nr: The number of entries to scan
2078 *
2079 * This function selects the entries on the LRU which are able to
2080 * be demoted, and then kicks off the process by calling
2081 * gfs2_dispose_glock_lru() above.
2082 */
2083
gfs2_scan_glock_lru(unsigned long nr)2084 static unsigned long gfs2_scan_glock_lru(unsigned long nr)
2085 {
2086 struct gfs2_glock *gl, *next;
2087 LIST_HEAD(dispose);
2088 unsigned long freed = 0;
2089
2090 spin_lock(&lru_lock);
2091 list_for_each_entry_safe(gl, next, &lru_list, gl_lru) {
2092 if (!nr--)
2093 break;
2094 if (can_free_glock(gl))
2095 list_move(&gl->gl_lru, &dispose);
2096 }
2097 if (!list_empty(&dispose))
2098 freed = gfs2_dispose_glock_lru(&dispose);
2099 spin_unlock(&lru_lock);
2100
2101 return freed;
2102 }
2103
gfs2_glock_shrink_scan(struct shrinker * shrink,struct shrink_control * sc)2104 static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
2105 struct shrink_control *sc)
2106 {
2107 if (!(sc->gfp_mask & __GFP_FS))
2108 return SHRINK_STOP;
2109 return gfs2_scan_glock_lru(sc->nr_to_scan);
2110 }
2111
gfs2_glock_shrink_count(struct shrinker * shrink,struct shrink_control * sc)2112 static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
2113 struct shrink_control *sc)
2114 {
2115 return vfs_pressure_ratio(atomic_read(&lru_count));
2116 }
2117
2118 static struct shrinker *glock_shrinker;
2119
2120 /**
2121 * glock_hash_walk - Call a function for glock in a hash bucket
2122 * @examiner: the function
2123 * @sdp: the filesystem
2124 *
2125 * Note that the function can be called multiple times on the same
2126 * object. So the user must ensure that the function can cope with
2127 * that.
2128 */
2129
glock_hash_walk(glock_examiner examiner,const struct gfs2_sbd * sdp)2130 static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
2131 {
2132 struct gfs2_glock *gl;
2133 struct rhashtable_iter iter;
2134
2135 rhashtable_walk_enter(&gl_hash_table, &iter);
2136
2137 do {
2138 rhashtable_walk_start(&iter);
2139
2140 while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) {
2141 if (gl->gl_name.ln_sbd == sdp)
2142 examiner(gl);
2143 }
2144
2145 rhashtable_walk_stop(&iter);
2146 } while (cond_resched(), gl == ERR_PTR(-EAGAIN));
2147
2148 rhashtable_walk_exit(&iter);
2149 }
2150
gfs2_cancel_delete_work(struct gfs2_glock * gl)2151 void gfs2_cancel_delete_work(struct gfs2_glock *gl)
2152 {
2153 clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags);
2154 clear_bit(GLF_VERIFY_EVICT, &gl->gl_flags);
2155 if (cancel_delayed_work(&gl->gl_delete))
2156 gfs2_glock_put(gl);
2157 }
2158
flush_delete_work(struct gfs2_glock * gl)2159 static void flush_delete_work(struct gfs2_glock *gl)
2160 {
2161 if (gl->gl_name.ln_type == LM_TYPE_IOPEN) {
2162 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2163
2164 if (cancel_delayed_work(&gl->gl_delete)) {
2165 queue_delayed_work(sdp->sd_delete_wq,
2166 &gl->gl_delete, 0);
2167 }
2168 }
2169 }
2170
gfs2_flush_delete_work(struct gfs2_sbd * sdp)2171 void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
2172 {
2173 glock_hash_walk(flush_delete_work, sdp);
2174 flush_workqueue(sdp->sd_delete_wq);
2175 }
2176
2177 /**
2178 * thaw_glock - thaw out a glock which has an unprocessed reply waiting
2179 * @gl: The glock to thaw
2180 *
2181 */
2182
thaw_glock(struct gfs2_glock * gl)2183 static void thaw_glock(struct gfs2_glock *gl)
2184 {
2185 if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))
2186 return;
2187 if (!lockref_get_not_dead(&gl->gl_lockref))
2188 return;
2189
2190 spin_lock(&gl->gl_lockref.lock);
2191 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
2192 gfs2_glock_queue_work(gl, 0);
2193 spin_unlock(&gl->gl_lockref.lock);
2194 }
2195
2196 /**
2197 * clear_glock - look at a glock and see if we can free it from glock cache
2198 * @gl: the glock to look at
2199 *
2200 */
2201
clear_glock(struct gfs2_glock * gl)2202 static void clear_glock(struct gfs2_glock *gl)
2203 {
2204 gfs2_glock_remove_from_lru(gl);
2205
2206 spin_lock(&gl->gl_lockref.lock);
2207 if (!__lockref_is_dead(&gl->gl_lockref)) {
2208 gl->gl_lockref.count++;
2209 if (gl->gl_state != LM_ST_UNLOCKED)
2210 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
2211 gfs2_glock_queue_work(gl, 0);
2212 }
2213 spin_unlock(&gl->gl_lockref.lock);
2214 }
2215
2216 /**
2217 * gfs2_glock_thaw - Thaw any frozen glocks
2218 * @sdp: The super block
2219 *
2220 */
2221
gfs2_glock_thaw(struct gfs2_sbd * sdp)2222 void gfs2_glock_thaw(struct gfs2_sbd *sdp)
2223 {
2224 glock_hash_walk(thaw_glock, sdp);
2225 }
2226
dump_glock(struct seq_file * seq,struct gfs2_glock * gl,bool fsid)2227 static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2228 {
2229 spin_lock(&gl->gl_lockref.lock);
2230 gfs2_dump_glock(seq, gl, fsid);
2231 spin_unlock(&gl->gl_lockref.lock);
2232 }
2233
dump_glock_func(struct gfs2_glock * gl)2234 static void dump_glock_func(struct gfs2_glock *gl)
2235 {
2236 dump_glock(NULL, gl, true);
2237 }
2238
withdraw_dq(struct gfs2_glock * gl)2239 static void withdraw_dq(struct gfs2_glock *gl)
2240 {
2241 spin_lock(&gl->gl_lockref.lock);
2242 if (!__lockref_is_dead(&gl->gl_lockref) &&
2243 glock_blocked_by_withdraw(gl))
2244 do_error(gl, LM_OUT_ERROR); /* remove pending waiters */
2245 spin_unlock(&gl->gl_lockref.lock);
2246 }
2247
gfs2_gl_dq_holders(struct gfs2_sbd * sdp)2248 void gfs2_gl_dq_holders(struct gfs2_sbd *sdp)
2249 {
2250 glock_hash_walk(withdraw_dq, sdp);
2251 }
2252
2253 /**
2254 * gfs2_gl_hash_clear - Empty out the glock hash table
2255 * @sdp: the filesystem
2256 *
2257 * Called when unmounting the filesystem.
2258 */
2259
gfs2_gl_hash_clear(struct gfs2_sbd * sdp)2260 void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
2261 {
2262 set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
2263 flush_workqueue(glock_workqueue);
2264 glock_hash_walk(clear_glock, sdp);
2265 flush_workqueue(glock_workqueue);
2266 wait_event_timeout(sdp->sd_kill_wait,
2267 atomic_read(&sdp->sd_glock_disposal) == 0,
2268 HZ * 600);
2269 gfs2_lm_unmount(sdp);
2270 gfs2_free_dead_glocks(sdp);
2271 glock_hash_walk(dump_glock_func, sdp);
2272 }
2273
state2str(unsigned state)2274 static const char *state2str(unsigned state)
2275 {
2276 switch(state) {
2277 case LM_ST_UNLOCKED:
2278 return "UN";
2279 case LM_ST_SHARED:
2280 return "SH";
2281 case LM_ST_DEFERRED:
2282 return "DF";
2283 case LM_ST_EXCLUSIVE:
2284 return "EX";
2285 }
2286 return "??";
2287 }
2288
hflags2str(char * buf,u16 flags,unsigned long iflags)2289 static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
2290 {
2291 char *p = buf;
2292 if (flags & LM_FLAG_TRY)
2293 *p++ = 't';
2294 if (flags & LM_FLAG_TRY_1CB)
2295 *p++ = 'T';
2296 if (flags & LM_FLAG_NOEXP)
2297 *p++ = 'e';
2298 if (flags & LM_FLAG_ANY)
2299 *p++ = 'A';
2300 if (flags & LM_FLAG_NODE_SCOPE)
2301 *p++ = 'n';
2302 if (flags & GL_ASYNC)
2303 *p++ = 'a';
2304 if (flags & GL_EXACT)
2305 *p++ = 'E';
2306 if (flags & GL_NOCACHE)
2307 *p++ = 'c';
2308 if (test_bit(HIF_HOLDER, &iflags))
2309 *p++ = 'H';
2310 if (test_bit(HIF_WAIT, &iflags))
2311 *p++ = 'W';
2312 if (flags & GL_SKIP)
2313 *p++ = 's';
2314 *p = 0;
2315 return buf;
2316 }
2317
2318 /**
2319 * dump_holder - print information about a glock holder
2320 * @seq: the seq_file struct
2321 * @gh: the glock holder
2322 * @fs_id_buf: pointer to file system id (if requested)
2323 *
2324 */
2325
dump_holder(struct seq_file * seq,const struct gfs2_holder * gh,const char * fs_id_buf)2326 static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
2327 const char *fs_id_buf)
2328 {
2329 const char *comm = "(none)";
2330 pid_t owner_pid = 0;
2331 char flags_buf[32];
2332
2333 rcu_read_lock();
2334 if (pid_is_meaningful(gh)) {
2335 struct task_struct *gh_owner;
2336
2337 comm = "(ended)";
2338 owner_pid = pid_nr(gh->gh_owner_pid);
2339 gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
2340 if (gh_owner)
2341 comm = gh_owner->comm;
2342 }
2343 gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
2344 fs_id_buf, state2str(gh->gh_state),
2345 hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
2346 gh->gh_error, (long)owner_pid, comm, (void *)gh->gh_ip);
2347 rcu_read_unlock();
2348 }
2349
gflags2str(char * buf,const struct gfs2_glock * gl)2350 static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
2351 {
2352 const unsigned long *gflags = &gl->gl_flags;
2353 char *p = buf;
2354
2355 if (test_bit(GLF_LOCK, gflags))
2356 *p++ = 'l';
2357 if (test_bit(GLF_DEMOTE, gflags))
2358 *p++ = 'D';
2359 if (test_bit(GLF_PENDING_DEMOTE, gflags))
2360 *p++ = 'd';
2361 if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
2362 *p++ = 'p';
2363 if (test_bit(GLF_DIRTY, gflags))
2364 *p++ = 'y';
2365 if (test_bit(GLF_LFLUSH, gflags))
2366 *p++ = 'f';
2367 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags))
2368 *p++ = 'i';
2369 if (test_bit(GLF_REPLY_PENDING, gflags))
2370 *p++ = 'r';
2371 if (test_bit(GLF_INITIAL, gflags))
2372 *p++ = 'I';
2373 if (test_bit(GLF_FROZEN, gflags))
2374 *p++ = 'F';
2375 if (!list_empty(&gl->gl_holders))
2376 *p++ = 'q';
2377 if (test_bit(GLF_LRU, gflags))
2378 *p++ = 'L';
2379 if (gl->gl_object)
2380 *p++ = 'o';
2381 if (test_bit(GLF_BLOCKING, gflags))
2382 *p++ = 'b';
2383 if (test_bit(GLF_FREEING, gflags))
2384 *p++ = 'x';
2385 if (test_bit(GLF_INSTANTIATE_NEEDED, gflags))
2386 *p++ = 'n';
2387 if (test_bit(GLF_INSTANTIATE_IN_PROG, gflags))
2388 *p++ = 'N';
2389 if (test_bit(GLF_TRY_TO_EVICT, gflags))
2390 *p++ = 'e';
2391 if (test_bit(GLF_VERIFY_EVICT, gflags))
2392 *p++ = 'E';
2393 *p = 0;
2394 return buf;
2395 }
2396
2397 /**
2398 * gfs2_dump_glock - print information about a glock
2399 * @seq: The seq_file struct
2400 * @gl: the glock
2401 * @fsid: If true, also dump the file system id
2402 *
2403 * The file format is as follows:
2404 * One line per object, capital letters are used to indicate objects
2405 * G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented,
2406 * other objects are indented by a single space and follow the glock to
2407 * which they are related. Fields are indicated by lower case letters
2408 * followed by a colon and the field value, except for strings which are in
2409 * [] so that its possible to see if they are composed of spaces for
2410 * example. The field's are n = number (id of the object), f = flags,
2411 * t = type, s = state, r = refcount, e = error, p = pid.
2412 *
2413 */
2414
gfs2_dump_glock(struct seq_file * seq,struct gfs2_glock * gl,bool fsid)2415 void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2416 {
2417 const struct gfs2_glock_operations *glops = gl->gl_ops;
2418 unsigned long long dtime;
2419 const struct gfs2_holder *gh;
2420 char gflags_buf[32];
2421 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2422 char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
2423 unsigned long nrpages = 0;
2424
2425 if (gl->gl_ops->go_flags & GLOF_ASPACE) {
2426 struct address_space *mapping = gfs2_glock2aspace(gl);
2427
2428 nrpages = mapping->nrpages;
2429 }
2430 memset(fs_id_buf, 0, sizeof(fs_id_buf));
2431 if (fsid && sdp) /* safety precaution */
2432 sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname);
2433 dtime = jiffies - gl->gl_demote_time;
2434 dtime *= 1000000/HZ; /* demote time in uSec */
2435 if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
2436 dtime = 0;
2437 gfs2_print_dbg(seq, "%sG: s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
2438 "v:%d r:%d m:%ld p:%lu\n",
2439 fs_id_buf, state2str(gl->gl_state),
2440 gl->gl_name.ln_type,
2441 (unsigned long long)gl->gl_name.ln_number,
2442 gflags2str(gflags_buf, gl),
2443 state2str(gl->gl_target),
2444 state2str(gl->gl_demote_state), dtime,
2445 atomic_read(&gl->gl_ail_count),
2446 atomic_read(&gl->gl_revokes),
2447 (int)gl->gl_lockref.count, gl->gl_hold_time, nrpages);
2448
2449 list_for_each_entry(gh, &gl->gl_holders, gh_list)
2450 dump_holder(seq, gh, fs_id_buf);
2451
2452 if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
2453 glops->go_dump(seq, gl, fs_id_buf);
2454 }
2455
gfs2_glstats_seq_show(struct seq_file * seq,void * iter_ptr)2456 static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
2457 {
2458 struct gfs2_glock *gl = iter_ptr;
2459
2460 seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
2461 gl->gl_name.ln_type,
2462 (unsigned long long)gl->gl_name.ln_number,
2463 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
2464 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
2465 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
2466 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
2467 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
2468 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
2469 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
2470 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
2471 return 0;
2472 }
2473
2474 static const char *gfs2_gltype[] = {
2475 "type",
2476 "reserved",
2477 "nondisk",
2478 "inode",
2479 "rgrp",
2480 "meta",
2481 "iopen",
2482 "flock",
2483 "plock",
2484 "quota",
2485 "journal",
2486 };
2487
2488 static const char *gfs2_stype[] = {
2489 [GFS2_LKS_SRTT] = "srtt",
2490 [GFS2_LKS_SRTTVAR] = "srttvar",
2491 [GFS2_LKS_SRTTB] = "srttb",
2492 [GFS2_LKS_SRTTVARB] = "srttvarb",
2493 [GFS2_LKS_SIRT] = "sirt",
2494 [GFS2_LKS_SIRTVAR] = "sirtvar",
2495 [GFS2_LKS_DCOUNT] = "dlm",
2496 [GFS2_LKS_QCOUNT] = "queue",
2497 };
2498
2499 #define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
2500
gfs2_sbstats_seq_show(struct seq_file * seq,void * iter_ptr)2501 static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
2502 {
2503 struct gfs2_sbd *sdp = seq->private;
2504 loff_t pos = *(loff_t *)iter_ptr;
2505 unsigned index = pos >> 3;
2506 unsigned subindex = pos & 0x07;
2507 int i;
2508
2509 if (index == 0 && subindex != 0)
2510 return 0;
2511
2512 seq_printf(seq, "%-10s %8s:", gfs2_gltype[index],
2513 (index == 0) ? "cpu": gfs2_stype[subindex]);
2514
2515 for_each_possible_cpu(i) {
2516 const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
2517
2518 if (index == 0)
2519 seq_printf(seq, " %15u", i);
2520 else
2521 seq_printf(seq, " %15llu", (unsigned long long)lkstats->
2522 lkstats[index - 1].stats[subindex]);
2523 }
2524 seq_putc(seq, '\n');
2525 return 0;
2526 }
2527
gfs2_glock_init(void)2528 int __init gfs2_glock_init(void)
2529 {
2530 int i, ret;
2531
2532 ret = rhashtable_init(&gl_hash_table, &ht_parms);
2533 if (ret < 0)
2534 return ret;
2535
2536 glock_workqueue = alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM |
2537 WQ_HIGHPRI | WQ_FREEZABLE, 0);
2538 if (!glock_workqueue) {
2539 rhashtable_destroy(&gl_hash_table);
2540 return -ENOMEM;
2541 }
2542
2543 glock_shrinker = shrinker_alloc(0, "gfs2-glock");
2544 if (!glock_shrinker) {
2545 destroy_workqueue(glock_workqueue);
2546 rhashtable_destroy(&gl_hash_table);
2547 return -ENOMEM;
2548 }
2549
2550 glock_shrinker->count_objects = gfs2_glock_shrink_count;
2551 glock_shrinker->scan_objects = gfs2_glock_shrink_scan;
2552
2553 shrinker_register(glock_shrinker);
2554
2555 for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
2556 init_waitqueue_head(glock_wait_table + i);
2557
2558 return 0;
2559 }
2560
gfs2_glock_exit(void)2561 void gfs2_glock_exit(void)
2562 {
2563 shrinker_free(glock_shrinker);
2564 rhashtable_destroy(&gl_hash_table);
2565 destroy_workqueue(glock_workqueue);
2566 }
2567
gfs2_glock_iter_next(struct gfs2_glock_iter * gi,loff_t n)2568 static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
2569 {
2570 struct gfs2_glock *gl = gi->gl;
2571
2572 if (gl) {
2573 if (n == 0)
2574 return;
2575 gfs2_glock_put_async(gl);
2576 }
2577 for (;;) {
2578 gl = rhashtable_walk_next(&gi->hti);
2579 if (IS_ERR_OR_NULL(gl)) {
2580 if (gl == ERR_PTR(-EAGAIN)) {
2581 n = 1;
2582 continue;
2583 }
2584 gl = NULL;
2585 break;
2586 }
2587 if (gl->gl_name.ln_sbd != gi->sdp)
2588 continue;
2589 if (n <= 1) {
2590 if (!lockref_get_not_dead(&gl->gl_lockref))
2591 continue;
2592 break;
2593 } else {
2594 if (__lockref_is_dead(&gl->gl_lockref))
2595 continue;
2596 n--;
2597 }
2598 }
2599 gi->gl = gl;
2600 }
2601
gfs2_glock_seq_start(struct seq_file * seq,loff_t * pos)2602 static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
2603 __acquires(RCU)
2604 {
2605 struct gfs2_glock_iter *gi = seq->private;
2606 loff_t n;
2607
2608 /*
2609 * We can either stay where we are, skip to the next hash table
2610 * entry, or start from the beginning.
2611 */
2612 if (*pos < gi->last_pos) {
2613 rhashtable_walk_exit(&gi->hti);
2614 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2615 n = *pos + 1;
2616 } else {
2617 n = *pos - gi->last_pos;
2618 }
2619
2620 rhashtable_walk_start(&gi->hti);
2621
2622 gfs2_glock_iter_next(gi, n);
2623 gi->last_pos = *pos;
2624 return gi->gl;
2625 }
2626
gfs2_glock_seq_next(struct seq_file * seq,void * iter_ptr,loff_t * pos)2627 static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
2628 loff_t *pos)
2629 {
2630 struct gfs2_glock_iter *gi = seq->private;
2631
2632 (*pos)++;
2633 gi->last_pos = *pos;
2634 gfs2_glock_iter_next(gi, 1);
2635 return gi->gl;
2636 }
2637
gfs2_glock_seq_stop(struct seq_file * seq,void * iter_ptr)2638 static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2639 __releases(RCU)
2640 {
2641 struct gfs2_glock_iter *gi = seq->private;
2642
2643 rhashtable_walk_stop(&gi->hti);
2644 }
2645
gfs2_glock_seq_show(struct seq_file * seq,void * iter_ptr)2646 static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2647 {
2648 dump_glock(seq, iter_ptr, false);
2649 return 0;
2650 }
2651
gfs2_sbstats_seq_start(struct seq_file * seq,loff_t * pos)2652 static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2653 {
2654 preempt_disable();
2655 if (*pos >= GFS2_NR_SBSTATS)
2656 return NULL;
2657 return pos;
2658 }
2659
gfs2_sbstats_seq_next(struct seq_file * seq,void * iter_ptr,loff_t * pos)2660 static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2661 loff_t *pos)
2662 {
2663 (*pos)++;
2664 if (*pos >= GFS2_NR_SBSTATS)
2665 return NULL;
2666 return pos;
2667 }
2668
gfs2_sbstats_seq_stop(struct seq_file * seq,void * iter_ptr)2669 static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2670 {
2671 preempt_enable();
2672 }
2673
2674 static const struct seq_operations gfs2_glock_seq_ops = {
2675 .start = gfs2_glock_seq_start,
2676 .next = gfs2_glock_seq_next,
2677 .stop = gfs2_glock_seq_stop,
2678 .show = gfs2_glock_seq_show,
2679 };
2680
2681 static const struct seq_operations gfs2_glstats_seq_ops = {
2682 .start = gfs2_glock_seq_start,
2683 .next = gfs2_glock_seq_next,
2684 .stop = gfs2_glock_seq_stop,
2685 .show = gfs2_glstats_seq_show,
2686 };
2687
2688 static const struct seq_operations gfs2_sbstats_sops = {
2689 .start = gfs2_sbstats_seq_start,
2690 .next = gfs2_sbstats_seq_next,
2691 .stop = gfs2_sbstats_seq_stop,
2692 .show = gfs2_sbstats_seq_show,
2693 };
2694
2695 #define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2696
__gfs2_glocks_open(struct inode * inode,struct file * file,const struct seq_operations * ops)2697 static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2698 const struct seq_operations *ops)
2699 {
2700 int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2701 if (ret == 0) {
2702 struct seq_file *seq = file->private_data;
2703 struct gfs2_glock_iter *gi = seq->private;
2704
2705 gi->sdp = inode->i_private;
2706 seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2707 if (seq->buf)
2708 seq->size = GFS2_SEQ_GOODSIZE;
2709 /*
2710 * Initially, we are "before" the first hash table entry; the
2711 * first call to rhashtable_walk_next gets us the first entry.
2712 */
2713 gi->last_pos = -1;
2714 gi->gl = NULL;
2715 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2716 }
2717 return ret;
2718 }
2719
gfs2_glocks_open(struct inode * inode,struct file * file)2720 static int gfs2_glocks_open(struct inode *inode, struct file *file)
2721 {
2722 return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops);
2723 }
2724
gfs2_glocks_release(struct inode * inode,struct file * file)2725 static int gfs2_glocks_release(struct inode *inode, struct file *file)
2726 {
2727 struct seq_file *seq = file->private_data;
2728 struct gfs2_glock_iter *gi = seq->private;
2729
2730 if (gi->gl)
2731 gfs2_glock_put(gi->gl);
2732 rhashtable_walk_exit(&gi->hti);
2733 return seq_release_private(inode, file);
2734 }
2735
gfs2_glstats_open(struct inode * inode,struct file * file)2736 static int gfs2_glstats_open(struct inode *inode, struct file *file)
2737 {
2738 return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops);
2739 }
2740
2741 static const struct file_operations gfs2_glocks_fops = {
2742 .owner = THIS_MODULE,
2743 .open = gfs2_glocks_open,
2744 .read = seq_read,
2745 .llseek = seq_lseek,
2746 .release = gfs2_glocks_release,
2747 };
2748
2749 static const struct file_operations gfs2_glstats_fops = {
2750 .owner = THIS_MODULE,
2751 .open = gfs2_glstats_open,
2752 .read = seq_read,
2753 .llseek = seq_lseek,
2754 .release = gfs2_glocks_release,
2755 };
2756
2757 struct gfs2_glockfd_iter {
2758 struct super_block *sb;
2759 unsigned int tgid;
2760 struct task_struct *task;
2761 unsigned int fd;
2762 struct file *file;
2763 };
2764
gfs2_glockfd_next_task(struct gfs2_glockfd_iter * i)2765 static struct task_struct *gfs2_glockfd_next_task(struct gfs2_glockfd_iter *i)
2766 {
2767 struct pid_namespace *ns = task_active_pid_ns(current);
2768 struct pid *pid;
2769
2770 if (i->task)
2771 put_task_struct(i->task);
2772
2773 rcu_read_lock();
2774 retry:
2775 i->task = NULL;
2776 pid = find_ge_pid(i->tgid, ns);
2777 if (pid) {
2778 i->tgid = pid_nr_ns(pid, ns);
2779 i->task = pid_task(pid, PIDTYPE_TGID);
2780 if (!i->task) {
2781 i->tgid++;
2782 goto retry;
2783 }
2784 get_task_struct(i->task);
2785 }
2786 rcu_read_unlock();
2787 return i->task;
2788 }
2789
gfs2_glockfd_next_file(struct gfs2_glockfd_iter * i)2790 static struct file *gfs2_glockfd_next_file(struct gfs2_glockfd_iter *i)
2791 {
2792 if (i->file) {
2793 fput(i->file);
2794 i->file = NULL;
2795 }
2796
2797 rcu_read_lock();
2798 for(;; i->fd++) {
2799 struct inode *inode;
2800
2801 i->file = task_lookup_next_fdget_rcu(i->task, &i->fd);
2802 if (!i->file) {
2803 i->fd = 0;
2804 break;
2805 }
2806
2807 inode = file_inode(i->file);
2808 if (inode->i_sb == i->sb)
2809 break;
2810
2811 rcu_read_unlock();
2812 fput(i->file);
2813 rcu_read_lock();
2814 }
2815 rcu_read_unlock();
2816 return i->file;
2817 }
2818
gfs2_glockfd_seq_start(struct seq_file * seq,loff_t * pos)2819 static void *gfs2_glockfd_seq_start(struct seq_file *seq, loff_t *pos)
2820 {
2821 struct gfs2_glockfd_iter *i = seq->private;
2822
2823 if (*pos)
2824 return NULL;
2825 while (gfs2_glockfd_next_task(i)) {
2826 if (gfs2_glockfd_next_file(i))
2827 return i;
2828 i->tgid++;
2829 }
2830 return NULL;
2831 }
2832
gfs2_glockfd_seq_next(struct seq_file * seq,void * iter_ptr,loff_t * pos)2833 static void *gfs2_glockfd_seq_next(struct seq_file *seq, void *iter_ptr,
2834 loff_t *pos)
2835 {
2836 struct gfs2_glockfd_iter *i = seq->private;
2837
2838 (*pos)++;
2839 i->fd++;
2840 do {
2841 if (gfs2_glockfd_next_file(i))
2842 return i;
2843 i->tgid++;
2844 } while (gfs2_glockfd_next_task(i));
2845 return NULL;
2846 }
2847
gfs2_glockfd_seq_stop(struct seq_file * seq,void * iter_ptr)2848 static void gfs2_glockfd_seq_stop(struct seq_file *seq, void *iter_ptr)
2849 {
2850 struct gfs2_glockfd_iter *i = seq->private;
2851
2852 if (i->file)
2853 fput(i->file);
2854 if (i->task)
2855 put_task_struct(i->task);
2856 }
2857
gfs2_glockfd_seq_show_flock(struct seq_file * seq,struct gfs2_glockfd_iter * i)2858 static void gfs2_glockfd_seq_show_flock(struct seq_file *seq,
2859 struct gfs2_glockfd_iter *i)
2860 {
2861 struct gfs2_file *fp = i->file->private_data;
2862 struct gfs2_holder *fl_gh = &fp->f_fl_gh;
2863 struct lm_lockname gl_name = { .ln_type = LM_TYPE_RESERVED };
2864
2865 if (!READ_ONCE(fl_gh->gh_gl))
2866 return;
2867
2868 spin_lock(&i->file->f_lock);
2869 if (gfs2_holder_initialized(fl_gh))
2870 gl_name = fl_gh->gh_gl->gl_name;
2871 spin_unlock(&i->file->f_lock);
2872
2873 if (gl_name.ln_type != LM_TYPE_RESERVED) {
2874 seq_printf(seq, "%d %u %u/%llx\n",
2875 i->tgid, i->fd, gl_name.ln_type,
2876 (unsigned long long)gl_name.ln_number);
2877 }
2878 }
2879
gfs2_glockfd_seq_show(struct seq_file * seq,void * iter_ptr)2880 static int gfs2_glockfd_seq_show(struct seq_file *seq, void *iter_ptr)
2881 {
2882 struct gfs2_glockfd_iter *i = seq->private;
2883 struct inode *inode = file_inode(i->file);
2884 struct gfs2_glock *gl;
2885
2886 inode_lock_shared(inode);
2887 gl = GFS2_I(inode)->i_iopen_gh.gh_gl;
2888 if (gl) {
2889 seq_printf(seq, "%d %u %u/%llx\n",
2890 i->tgid, i->fd, gl->gl_name.ln_type,
2891 (unsigned long long)gl->gl_name.ln_number);
2892 }
2893 gfs2_glockfd_seq_show_flock(seq, i);
2894 inode_unlock_shared(inode);
2895 return 0;
2896 }
2897
2898 static const struct seq_operations gfs2_glockfd_seq_ops = {
2899 .start = gfs2_glockfd_seq_start,
2900 .next = gfs2_glockfd_seq_next,
2901 .stop = gfs2_glockfd_seq_stop,
2902 .show = gfs2_glockfd_seq_show,
2903 };
2904
gfs2_glockfd_open(struct inode * inode,struct file * file)2905 static int gfs2_glockfd_open(struct inode *inode, struct file *file)
2906 {
2907 struct gfs2_glockfd_iter *i;
2908 struct gfs2_sbd *sdp = inode->i_private;
2909
2910 i = __seq_open_private(file, &gfs2_glockfd_seq_ops,
2911 sizeof(struct gfs2_glockfd_iter));
2912 if (!i)
2913 return -ENOMEM;
2914 i->sb = sdp->sd_vfs;
2915 return 0;
2916 }
2917
2918 static const struct file_operations gfs2_glockfd_fops = {
2919 .owner = THIS_MODULE,
2920 .open = gfs2_glockfd_open,
2921 .read = seq_read,
2922 .llseek = seq_lseek,
2923 .release = seq_release_private,
2924 };
2925
2926 DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats);
2927
gfs2_create_debugfs_file(struct gfs2_sbd * sdp)2928 void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2929 {
2930 sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root);
2931
2932 debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2933 &gfs2_glocks_fops);
2934
2935 debugfs_create_file("glockfd", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2936 &gfs2_glockfd_fops);
2937
2938 debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2939 &gfs2_glstats_fops);
2940
2941 debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2942 &gfs2_sbstats_fops);
2943 }
2944
gfs2_delete_debugfs_file(struct gfs2_sbd * sdp)2945 void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2946 {
2947 debugfs_remove_recursive(sdp->debugfs_dir);
2948 sdp->debugfs_dir = NULL;
2949 }
2950
gfs2_register_debugfs(void)2951 void gfs2_register_debugfs(void)
2952 {
2953 gfs2_root = debugfs_create_dir("gfs2", NULL);
2954 }
2955
gfs2_unregister_debugfs(void)2956 void gfs2_unregister_debugfs(void)
2957 {
2958 debugfs_remove(gfs2_root);
2959 gfs2_root = NULL;
2960 }
2961