832af9a03b151b458a012486cd7748719a9142ae
[linux-2.6-microblaze.git] / fs / gfs2 / glock.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) Sistina Software, Inc.  1997-2003 All rights reserved.
4  * Copyright (C) 2004-2008 Red Hat, Inc.  All rights reserved.
5  */
6
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9 #include <linux/sched.h>
10 #include <linux/slab.h>
11 #include <linux/spinlock.h>
12 #include <linux/buffer_head.h>
13 #include <linux/delay.h>
14 #include <linux/sort.h>
15 #include <linux/hash.h>
16 #include <linux/jhash.h>
17 #include <linux/kallsyms.h>
18 #include <linux/gfs2_ondisk.h>
19 #include <linux/list.h>
20 #include <linux/wait.h>
21 #include <linux/module.h>
22 #include <linux/uaccess.h>
23 #include <linux/seq_file.h>
24 #include <linux/debugfs.h>
25 #include <linux/kthread.h>
26 #include <linux/freezer.h>
27 #include <linux/workqueue.h>
28 #include <linux/jiffies.h>
29 #include <linux/rcupdate.h>
30 #include <linux/rculist_bl.h>
31 #include <linux/bit_spinlock.h>
32 #include <linux/percpu.h>
33 #include <linux/list_sort.h>
34 #include <linux/lockref.h>
35 #include <linux/rhashtable.h>
36
37 #include "gfs2.h"
38 #include "incore.h"
39 #include "glock.h"
40 #include "glops.h"
41 #include "inode.h"
42 #include "lops.h"
43 #include "meta_io.h"
44 #include "quota.h"
45 #include "super.h"
46 #include "util.h"
47 #include "bmap.h"
48 #define CREATE_TRACE_POINTS
49 #include "trace_gfs2.h"
50
51 struct gfs2_glock_iter {
52         struct gfs2_sbd *sdp;           /* incore superblock           */
53         struct rhashtable_iter hti;     /* rhashtable iterator         */
54         struct gfs2_glock *gl;          /* current glock struct        */
55         loff_t last_pos;                /* last position               */
56 };
57
58 typedef void (*glock_examiner) (struct gfs2_glock * gl);
59
60 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
61 static void __gfs2_glock_dq(struct gfs2_holder *gh);
62
63 static struct dentry *gfs2_root;
64 static struct workqueue_struct *glock_workqueue;
65 struct workqueue_struct *gfs2_delete_workqueue;
66 static LIST_HEAD(lru_list);
67 static atomic_t lru_count = ATOMIC_INIT(0);
68 static DEFINE_SPINLOCK(lru_lock);
69
70 #define GFS2_GL_HASH_SHIFT      15
71 #define GFS2_GL_HASH_SIZE       BIT(GFS2_GL_HASH_SHIFT)
72
73 static const struct rhashtable_params ht_parms = {
74         .nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
75         .key_len = offsetofend(struct lm_lockname, ln_type),
76         .key_offset = offsetof(struct gfs2_glock, gl_name),
77         .head_offset = offsetof(struct gfs2_glock, gl_node),
78 };
79
80 static struct rhashtable gl_hash_table;
81
82 #define GLOCK_WAIT_TABLE_BITS 12
83 #define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
84 static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
85
86 struct wait_glock_queue {
87         struct lm_lockname *name;
88         wait_queue_entry_t wait;
89 };
90
91 static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
92                                int sync, void *key)
93 {
94         struct wait_glock_queue *wait_glock =
95                 container_of(wait, struct wait_glock_queue, wait);
96         struct lm_lockname *wait_name = wait_glock->name;
97         struct lm_lockname *wake_name = key;
98
99         if (wake_name->ln_sbd != wait_name->ln_sbd ||
100             wake_name->ln_number != wait_name->ln_number ||
101             wake_name->ln_type != wait_name->ln_type)
102                 return 0;
103         return autoremove_wake_function(wait, mode, sync, key);
104 }
105
106 static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
107 {
108         u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0);
109
110         return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS);
111 }
112
113 /**
114  * wake_up_glock  -  Wake up waiters on a glock
115  * @gl: the glock
116  */
117 static void wake_up_glock(struct gfs2_glock *gl)
118 {
119         wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name);
120
121         if (waitqueue_active(wq))
122                 __wake_up(wq, TASK_NORMAL, 1, &gl->gl_name);
123 }
124
125 static void gfs2_glock_dealloc(struct rcu_head *rcu)
126 {
127         struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
128
129         kfree(gl->gl_lksb.sb_lvbptr);
130         if (gl->gl_ops->go_flags & GLOF_ASPACE) {
131                 struct gfs2_glock_aspace *gla =
132                         container_of(gl, struct gfs2_glock_aspace, glock);
133                 kmem_cache_free(gfs2_glock_aspace_cachep, gla);
134         } else
135                 kmem_cache_free(gfs2_glock_cachep, gl);
136 }
137
138 /**
139  * glock_blocked_by_withdraw - determine if we can still use a glock
140  * @gl: the glock
141  *
142  * We need to allow some glocks to be enqueued, dequeued, promoted, and demoted
143  * when we're withdrawn. For example, to maintain metadata integrity, we should
144  * disallow the use of inode and rgrp glocks when withdrawn. Other glocks, like
145  * iopen or the transaction glocks may be safely used because none of their
146  * metadata goes through the journal. So in general, we should disallow all
147  * glocks that are journaled, and allow all the others. One exception is:
148  * we need to allow our active journal to be promoted and demoted so others
149  * may recover it and we can reacquire it when they're done.
150  */
151 static bool glock_blocked_by_withdraw(struct gfs2_glock *gl)
152 {
153         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
154
155         if (likely(!gfs2_withdrawn(sdp)))
156                 return false;
157         if (gl->gl_ops->go_flags & GLOF_NONDISK)
158                 return false;
159         if (!sdp->sd_jdesc ||
160             gl->gl_name.ln_number == sdp->sd_jdesc->jd_no_addr)
161                 return false;
162         return true;
163 }
164
165 void gfs2_glock_free(struct gfs2_glock *gl)
166 {
167         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
168
169         gfs2_glock_assert_withdraw(gl, atomic_read(&gl->gl_revokes) == 0);
170         rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
171         smp_mb();
172         wake_up_glock(gl);
173         call_rcu(&gl->gl_rcu, gfs2_glock_dealloc);
174         if (atomic_dec_and_test(&sdp->sd_glock_disposal))
175                 wake_up(&sdp->sd_glock_wait);
176 }
177
178 /**
179  * gfs2_glock_hold() - increment reference count on glock
180  * @gl: The glock to hold
181  *
182  */
183
184 void gfs2_glock_hold(struct gfs2_glock *gl)
185 {
186         GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
187         lockref_get(&gl->gl_lockref);
188 }
189
190 /**
191  * demote_ok - Check to see if it's ok to unlock a glock
192  * @gl: the glock
193  *
194  * Returns: 1 if it's ok
195  */
196
197 static int demote_ok(const struct gfs2_glock *gl)
198 {
199         const struct gfs2_glock_operations *glops = gl->gl_ops;
200
201         if (gl->gl_state == LM_ST_UNLOCKED)
202                 return 0;
203         /*
204          * Note that demote_ok is used for the lru process of disposing of
205          * glocks. For this purpose, we don't care if the glock's holders
206          * have the HIF_MAY_DEMOTE flag set or not. If someone is using
207          * them, don't demote.
208          */
209         if (!list_empty(&gl->gl_holders))
210                 return 0;
211         if (glops->go_demote_ok)
212                 return glops->go_demote_ok(gl);
213         return 1;
214 }
215
216
217 void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
218 {
219         if (!(gl->gl_ops->go_flags & GLOF_LRU))
220                 return;
221
222         spin_lock(&lru_lock);
223
224         list_move_tail(&gl->gl_lru, &lru_list);
225
226         if (!test_bit(GLF_LRU, &gl->gl_flags)) {
227                 set_bit(GLF_LRU, &gl->gl_flags);
228                 atomic_inc(&lru_count);
229         }
230
231         spin_unlock(&lru_lock);
232 }
233
234 static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
235 {
236         if (!(gl->gl_ops->go_flags & GLOF_LRU))
237                 return;
238
239         spin_lock(&lru_lock);
240         if (test_bit(GLF_LRU, &gl->gl_flags)) {
241                 list_del_init(&gl->gl_lru);
242                 atomic_dec(&lru_count);
243                 clear_bit(GLF_LRU, &gl->gl_flags);
244         }
245         spin_unlock(&lru_lock);
246 }
247
248 /*
249  * Enqueue the glock on the work queue.  Passes one glock reference on to the
250  * work queue.
251  */
252 static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
253         if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) {
254                 /*
255                  * We are holding the lockref spinlock, and the work was still
256                  * queued above.  The queued work (glock_work_func) takes that
257                  * spinlock before dropping its glock reference(s), so it
258                  * cannot have dropped them in the meantime.
259                  */
260                 GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
261                 gl->gl_lockref.count--;
262         }
263 }
264
265 static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
266         spin_lock(&gl->gl_lockref.lock);
267         __gfs2_glock_queue_work(gl, delay);
268         spin_unlock(&gl->gl_lockref.lock);
269 }
270
271 static void __gfs2_glock_put(struct gfs2_glock *gl)
272 {
273         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
274         struct address_space *mapping = gfs2_glock2aspace(gl);
275
276         lockref_mark_dead(&gl->gl_lockref);
277
278         gfs2_glock_remove_from_lru(gl);
279         spin_unlock(&gl->gl_lockref.lock);
280         GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
281         if (mapping) {
282                 truncate_inode_pages_final(mapping);
283                 if (!gfs2_withdrawn(sdp))
284                         GLOCK_BUG_ON(gl, !mapping_empty(mapping));
285         }
286         trace_gfs2_glock_put(gl);
287         sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
288 }
289
290 /*
291  * Cause the glock to be put in work queue context.
292  */
293 void gfs2_glock_queue_put(struct gfs2_glock *gl)
294 {
295         gfs2_glock_queue_work(gl, 0);
296 }
297
298 /**
299  * gfs2_glock_put() - Decrement reference count on glock
300  * @gl: The glock to put
301  *
302  */
303
304 void gfs2_glock_put(struct gfs2_glock *gl)
305 {
306         if (lockref_put_or_lock(&gl->gl_lockref))
307                 return;
308
309         __gfs2_glock_put(gl);
310 }
311
312 /**
313  * may_grant - check if it's ok to grant a new lock
314  * @gl: The glock
315  * @current_gh: One of the current holders of @gl
316  * @gh: The lock request which we wish to grant
317  *
318  * With our current compatibility rules, if a glock has one or more active
319  * holders (HIF_HOLDER flag set), any of those holders can be passed in as
320  * @current_gh; they are all the same as far as compatibility with the new @gh
321  * goes.
322  *
323  * Returns true if it's ok to grant the lock.
324  */
325
326 static inline bool may_grant(struct gfs2_glock *gl,
327                              struct gfs2_holder *current_gh,
328                              struct gfs2_holder *gh)
329 {
330         if (current_gh) {
331                 GLOCK_BUG_ON(gl, !test_bit(HIF_HOLDER, &current_gh->gh_iflags));
332
333                 switch(current_gh->gh_state) {
334                 case LM_ST_EXCLUSIVE:
335                         /*
336                          * Here we make a special exception to grant holders
337                          * who agree to share the EX lock with other holders
338                          * who also have the bit set. If the original holder
339                          * has the LM_FLAG_NODE_SCOPE bit set, we grant more
340                          * holders with the bit set.
341                          */
342                         return gh->gh_state == LM_ST_EXCLUSIVE &&
343                                (current_gh->gh_flags & LM_FLAG_NODE_SCOPE) &&
344                                (gh->gh_flags & LM_FLAG_NODE_SCOPE);
345
346                 case LM_ST_SHARED:
347                 case LM_ST_DEFERRED:
348                         return gh->gh_state == current_gh->gh_state;
349
350                 default:
351                         return false;
352                 }
353         }
354
355         if (gl->gl_state == gh->gh_state)
356                 return true;
357         if (gh->gh_flags & GL_EXACT)
358                 return false;
359         if (gl->gl_state == LM_ST_EXCLUSIVE) {
360                 return gh->gh_state == LM_ST_SHARED ||
361                        gh->gh_state == LM_ST_DEFERRED;
362         }
363         if (gh->gh_flags & LM_FLAG_ANY)
364                 return gl->gl_state != LM_ST_UNLOCKED;
365         return false;
366 }
367
368 static void gfs2_holder_wake(struct gfs2_holder *gh)
369 {
370         clear_bit(HIF_WAIT, &gh->gh_iflags);
371         smp_mb__after_atomic();
372         wake_up_bit(&gh->gh_iflags, HIF_WAIT);
373         if (gh->gh_flags & GL_ASYNC) {
374                 struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
375
376                 wake_up(&sdp->sd_async_glock_wait);
377         }
378 }
379
380 /**
381  * do_error - Something unexpected has happened during a lock request
382  * @gl: The glock
383  * @ret: The status from the DLM
384  */
385
386 static void do_error(struct gfs2_glock *gl, const int ret)
387 {
388         struct gfs2_holder *gh, *tmp;
389
390         list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
391                 if (!test_bit(HIF_WAIT, &gh->gh_iflags))
392                         continue;
393                 if (ret & LM_OUT_ERROR)
394                         gh->gh_error = -EIO;
395                 else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
396                         gh->gh_error = GLR_TRYFAILED;
397                 else
398                         continue;
399                 list_del_init(&gh->gh_list);
400                 trace_gfs2_glock_queue(gh, 0);
401                 gfs2_holder_wake(gh);
402         }
403 }
404
405 /**
406  * demote_incompat_holders - demote incompatible demoteable holders
407  * @gl: the glock we want to promote
408  * @new_gh: the new holder to be promoted
409  */
410 static void demote_incompat_holders(struct gfs2_glock *gl,
411                                     struct gfs2_holder *new_gh)
412 {
413         struct gfs2_holder *gh, *tmp;
414
415         /*
416          * Demote incompatible holders before we make ourselves eligible.
417          * (This holder may or may not allow auto-demoting, but we don't want
418          * to demote the new holder before it's even granted.)
419          */
420         list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
421                 /*
422                  * Since holders are at the front of the list, we stop when we
423                  * find the first non-holder.
424                  */
425                 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
426                         return;
427                 if (test_bit(HIF_MAY_DEMOTE, &gh->gh_iflags) &&
428                     !may_grant(gl, new_gh, gh)) {
429                         /*
430                          * We should not recurse into do_promote because
431                          * __gfs2_glock_dq only calls handle_callback,
432                          * gfs2_glock_add_to_lru and __gfs2_glock_queue_work.
433                          */
434                         __gfs2_glock_dq(gh);
435                 }
436         }
437 }
438
439 /**
440  * find_first_holder - find the first "holder" gh
441  * @gl: the glock
442  */
443
444 static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
445 {
446         struct gfs2_holder *gh;
447
448         if (!list_empty(&gl->gl_holders)) {
449                 gh = list_first_entry(&gl->gl_holders, struct gfs2_holder,
450                                       gh_list);
451                 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
452                         return gh;
453         }
454         return NULL;
455 }
456
457 /**
458  * find_first_strong_holder - find the first non-demoteable holder
459  * @gl: the glock
460  *
461  * Find the first holder that doesn't have the HIF_MAY_DEMOTE flag set.
462  */
463 static inline struct gfs2_holder *
464 find_first_strong_holder(struct gfs2_glock *gl)
465 {
466         struct gfs2_holder *gh;
467
468         list_for_each_entry(gh, &gl->gl_holders, gh_list) {
469                 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
470                         return NULL;
471                 if (!test_bit(HIF_MAY_DEMOTE, &gh->gh_iflags))
472                         return gh;
473         }
474         return NULL;
475 }
476
477 /*
478  * gfs2_instantiate - Call the glops instantiate function
479  * @gh: The glock holder
480  *
481  * Returns: 0 if instantiate was successful, or error.
482  */
483 int gfs2_instantiate(struct gfs2_holder *gh)
484 {
485         struct gfs2_glock *gl = gh->gh_gl;
486         const struct gfs2_glock_operations *glops = gl->gl_ops;
487         int ret;
488
489 again:
490         if (!test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags))
491                 goto done;
492
493         /*
494          * Since we unlock the lockref lock, we set a flag to indicate
495          * instantiate is in progress.
496          */
497         if (test_and_set_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags)) {
498                 wait_on_bit(&gl->gl_flags, GLF_INSTANTIATE_IN_PROG,
499                             TASK_UNINTERRUPTIBLE);
500                 /*
501                  * Here we just waited for a different instantiate to finish.
502                  * But that may not have been successful, as when a process
503                  * locks an inode glock _before_ it has an actual inode to
504                  * instantiate into. So we check again. This process might
505                  * have an inode to instantiate, so might be successful.
506                  */
507                 goto again;
508         }
509
510         ret = glops->go_instantiate(gl);
511         if (!ret)
512                 clear_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags);
513         clear_and_wake_up_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags);
514         if (ret)
515                 return ret;
516
517 done:
518         if (glops->go_held)
519                 return glops->go_held(gh);
520         return 0;
521 }
522
523 /**
524  * do_promote - promote as many requests as possible on the current queue
525  * @gl: The glock
526  * 
527  * Returns: 1 if there is a blocked holder at the head of the list
528  */
529
530 static int do_promote(struct gfs2_glock *gl)
531 {
532         struct gfs2_holder *gh, *tmp, *first_gh;
533         bool incompat_holders_demoted = false;
534
535         first_gh = find_first_strong_holder(gl);
536         list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
537                 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
538                         continue;
539                 if (!may_grant(gl, first_gh, gh)) {
540                         /*
541                          * If we get here, it means we may not grant this holder for
542                          * some reason. If this holder is the head of the list, it
543                          * means we have a blocked holder at the head, so return 1.
544                          */
545                         if (list_is_first(&gh->gh_list, &gl->gl_holders))
546                                 return 1;
547                         do_error(gl, 0);
548                         break;
549                 }
550                 if (!incompat_holders_demoted) {
551                         demote_incompat_holders(gl, first_gh);
552                         incompat_holders_demoted = true;
553                         first_gh = gh;
554                 }
555                 set_bit(HIF_HOLDER, &gh->gh_iflags);
556                 trace_gfs2_promote(gh);
557                 gfs2_holder_wake(gh);
558         }
559         return 0;
560 }
561
562 /**
563  * find_first_waiter - find the first gh that's waiting for the glock
564  * @gl: the glock
565  */
566
567 static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
568 {
569         struct gfs2_holder *gh;
570
571         list_for_each_entry(gh, &gl->gl_holders, gh_list) {
572                 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
573                         return gh;
574         }
575         return NULL;
576 }
577
578 /**
579  * state_change - record that the glock is now in a different state
580  * @gl: the glock
581  * @new_state: the new state
582  */
583
584 static void state_change(struct gfs2_glock *gl, unsigned int new_state)
585 {
586         int held1, held2;
587
588         held1 = (gl->gl_state != LM_ST_UNLOCKED);
589         held2 = (new_state != LM_ST_UNLOCKED);
590
591         if (held1 != held2) {
592                 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
593                 if (held2)
594                         gl->gl_lockref.count++;
595                 else
596                         gl->gl_lockref.count--;
597         }
598         if (new_state != gl->gl_target)
599                 /* shorten our minimum hold time */
600                 gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
601                                        GL_GLOCK_MIN_HOLD);
602         gl->gl_state = new_state;
603         gl->gl_tchange = jiffies;
604 }
605
606 static void gfs2_set_demote(struct gfs2_glock *gl)
607 {
608         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
609
610         set_bit(GLF_DEMOTE, &gl->gl_flags);
611         smp_mb();
612         wake_up(&sdp->sd_async_glock_wait);
613 }
614
615 static void gfs2_demote_wake(struct gfs2_glock *gl)
616 {
617         gl->gl_demote_state = LM_ST_EXCLUSIVE;
618         clear_bit(GLF_DEMOTE, &gl->gl_flags);
619         smp_mb__after_atomic();
620         wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
621 }
622
623 /**
624  * finish_xmote - The DLM has replied to one of our lock requests
625  * @gl: The glock
626  * @ret: The status from the DLM
627  *
628  */
629
630 static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
631 {
632         const struct gfs2_glock_operations *glops = gl->gl_ops;
633         struct gfs2_holder *gh;
634         unsigned state = ret & LM_OUT_ST_MASK;
635
636         spin_lock(&gl->gl_lockref.lock);
637         trace_gfs2_glock_state_change(gl, state);
638         state_change(gl, state);
639         gh = find_first_waiter(gl);
640
641         /* Demote to UN request arrived during demote to SH or DF */
642         if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
643             state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED)
644                 gl->gl_target = LM_ST_UNLOCKED;
645
646         /* Check for state != intended state */
647         if (unlikely(state != gl->gl_target)) {
648                 if (gh && (ret & LM_OUT_CANCELED))
649                         gfs2_holder_wake(gh);
650                 if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
651                         /* move to back of queue and try next entry */
652                         if (ret & LM_OUT_CANCELED) {
653                                 if ((gh->gh_flags & LM_FLAG_PRIORITY) == 0)
654                                         list_move_tail(&gh->gh_list, &gl->gl_holders);
655                                 gh = find_first_waiter(gl);
656                                 gl->gl_target = gh->gh_state;
657                                 goto retry;
658                         }
659                         /* Some error or failed "try lock" - report it */
660                         if ((ret & LM_OUT_ERROR) ||
661                             (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
662                                 gl->gl_target = gl->gl_state;
663                                 do_error(gl, ret);
664                                 goto out;
665                         }
666                 }
667                 switch(state) {
668                 /* Unlocked due to conversion deadlock, try again */
669                 case LM_ST_UNLOCKED:
670 retry:
671                         do_xmote(gl, gh, gl->gl_target);
672                         break;
673                 /* Conversion fails, unlock and try again */
674                 case LM_ST_SHARED:
675                 case LM_ST_DEFERRED:
676                         do_xmote(gl, gh, LM_ST_UNLOCKED);
677                         break;
678                 default: /* Everything else */
679                         fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n",
680                                gl->gl_target, state);
681                         GLOCK_BUG_ON(gl, 1);
682                 }
683                 spin_unlock(&gl->gl_lockref.lock);
684                 return;
685         }
686
687         /* Fast path - we got what we asked for */
688         if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags))
689                 gfs2_demote_wake(gl);
690         if (state != LM_ST_UNLOCKED) {
691                 if (glops->go_xmote_bh) {
692                         int rv;
693
694                         spin_unlock(&gl->gl_lockref.lock);
695                         rv = glops->go_xmote_bh(gl);
696                         spin_lock(&gl->gl_lockref.lock);
697                         if (rv) {
698                                 do_error(gl, rv);
699                                 goto out;
700                         }
701                 }
702                 do_promote(gl);
703         }
704 out:
705         clear_bit(GLF_LOCK, &gl->gl_flags);
706         spin_unlock(&gl->gl_lockref.lock);
707 }
708
709 static bool is_system_glock(struct gfs2_glock *gl)
710 {
711         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
712         struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
713
714         if (gl == m_ip->i_gl)
715                 return true;
716         return false;
717 }
718
719 /**
720  * do_xmote - Calls the DLM to change the state of a lock
721  * @gl: The lock state
722  * @gh: The holder (only for promotes)
723  * @target: The target lock state
724  *
725  */
726
727 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target)
728 __releases(&gl->gl_lockref.lock)
729 __acquires(&gl->gl_lockref.lock)
730 {
731         const struct gfs2_glock_operations *glops = gl->gl_ops;
732         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
733         unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
734         int ret;
735
736         if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) &&
737             gh && !(gh->gh_flags & LM_FLAG_NOEXP))
738                 return;
739         lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP |
740                       LM_FLAG_PRIORITY);
741         GLOCK_BUG_ON(gl, gl->gl_state == target);
742         GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
743         if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
744             glops->go_inval) {
745                 /*
746                  * If another process is already doing the invalidate, let that
747                  * finish first.  The glock state machine will get back to this
748                  * holder again later.
749                  */
750                 if (test_and_set_bit(GLF_INVALIDATE_IN_PROGRESS,
751                                      &gl->gl_flags))
752                         return;
753                 do_error(gl, 0); /* Fail queued try locks */
754         }
755         gl->gl_req = target;
756         set_bit(GLF_BLOCKING, &gl->gl_flags);
757         if ((gl->gl_req == LM_ST_UNLOCKED) ||
758             (gl->gl_state == LM_ST_EXCLUSIVE) ||
759             (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
760                 clear_bit(GLF_BLOCKING, &gl->gl_flags);
761         spin_unlock(&gl->gl_lockref.lock);
762         if (glops->go_sync) {
763                 ret = glops->go_sync(gl);
764                 /* If we had a problem syncing (due to io errors or whatever,
765                  * we should not invalidate the metadata or tell dlm to
766                  * release the glock to other nodes.
767                  */
768                 if (ret) {
769                         if (cmpxchg(&sdp->sd_log_error, 0, ret)) {
770                                 fs_err(sdp, "Error %d syncing glock \n", ret);
771                                 gfs2_dump_glock(NULL, gl, true);
772                         }
773                         goto skip_inval;
774                 }
775         }
776         if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) {
777                 /*
778                  * The call to go_sync should have cleared out the ail list.
779                  * If there are still items, we have a problem. We ought to
780                  * withdraw, but we can't because the withdraw code also uses
781                  * glocks. Warn about the error, dump the glock, then fall
782                  * through and wait for logd to do the withdraw for us.
783                  */
784                 if ((atomic_read(&gl->gl_ail_count) != 0) &&
785                     (!cmpxchg(&sdp->sd_log_error, 0, -EIO))) {
786                         gfs2_glock_assert_warn(gl,
787                                                !atomic_read(&gl->gl_ail_count));
788                         gfs2_dump_glock(NULL, gl, true);
789                 }
790                 glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
791                 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
792         }
793
794 skip_inval:
795         gfs2_glock_hold(gl);
796         /*
797          * Check for an error encountered since we called go_sync and go_inval.
798          * If so, we can't withdraw from the glock code because the withdraw
799          * code itself uses glocks (see function signal_our_withdraw) to
800          * change the mount to read-only. Most importantly, we must not call
801          * dlm to unlock the glock until the journal is in a known good state
802          * (after journal replay) otherwise other nodes may use the object
803          * (rgrp or dinode) and then later, journal replay will corrupt the
804          * file system. The best we can do here is wait for the logd daemon
805          * to see sd_log_error and withdraw, and in the meantime, requeue the
806          * work for later.
807          *
808          * We make a special exception for some system glocks, such as the
809          * system statfs inode glock, which needs to be granted before the
810          * gfs2_quotad daemon can exit, and that exit needs to finish before
811          * we can unmount the withdrawn file system.
812          *
813          * However, if we're just unlocking the lock (say, for unmount, when
814          * gfs2_gl_hash_clear calls clear_glock) and recovery is complete
815          * then it's okay to tell dlm to unlock it.
816          */
817         if (unlikely(sdp->sd_log_error && !gfs2_withdrawn(sdp)))
818                 gfs2_withdraw_delayed(sdp);
819         if (glock_blocked_by_withdraw(gl) &&
820             (target != LM_ST_UNLOCKED ||
821              test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags))) {
822                 if (!is_system_glock(gl)) {
823                         gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD);
824                         goto out;
825                 } else {
826                         clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
827                 }
828         }
829
830         if (sdp->sd_lockstruct.ls_ops->lm_lock) {
831                 /* lock_dlm */
832                 ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags);
833                 if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
834                     target == LM_ST_UNLOCKED &&
835                     test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) {
836                         finish_xmote(gl, target);
837                         gfs2_glock_queue_work(gl, 0);
838                 } else if (ret) {
839                         fs_err(sdp, "lm_lock ret %d\n", ret);
840                         GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp));
841                 }
842         } else { /* lock_nolock */
843                 finish_xmote(gl, target);
844                 gfs2_glock_queue_work(gl, 0);
845         }
846 out:
847         spin_lock(&gl->gl_lockref.lock);
848 }
849
850 /**
851  * run_queue - do all outstanding tasks related to a glock
852  * @gl: The glock in question
853  * @nonblock: True if we must not block in run_queue
854  *
855  */
856
857 static void run_queue(struct gfs2_glock *gl, const int nonblock)
858 __releases(&gl->gl_lockref.lock)
859 __acquires(&gl->gl_lockref.lock)
860 {
861         struct gfs2_holder *gh = NULL;
862
863         if (test_and_set_bit(GLF_LOCK, &gl->gl_flags))
864                 return;
865
866         GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
867
868         if (test_bit(GLF_DEMOTE, &gl->gl_flags) &&
869             gl->gl_demote_state != gl->gl_state) {
870                 if (find_first_holder(gl))
871                         goto out_unlock;
872                 if (nonblock)
873                         goto out_sched;
874                 set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
875                 GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
876                 gl->gl_target = gl->gl_demote_state;
877         } else {
878                 if (test_bit(GLF_DEMOTE, &gl->gl_flags))
879                         gfs2_demote_wake(gl);
880                 if (do_promote(gl) == 0)
881                         goto out_unlock;
882                 gh = find_first_waiter(gl);
883                 gl->gl_target = gh->gh_state;
884                 if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
885                         do_error(gl, 0); /* Fail queued try locks */
886         }
887         do_xmote(gl, gh, gl->gl_target);
888         return;
889
890 out_sched:
891         clear_bit(GLF_LOCK, &gl->gl_flags);
892         smp_mb__after_atomic();
893         gl->gl_lockref.count++;
894         __gfs2_glock_queue_work(gl, 0);
895         return;
896
897 out_unlock:
898         clear_bit(GLF_LOCK, &gl->gl_flags);
899         smp_mb__after_atomic();
900         return;
901 }
902
903 void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation)
904 {
905         struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
906
907         if (ri->ri_magic == 0)
908                 ri->ri_magic = cpu_to_be32(GFS2_MAGIC);
909         if (ri->ri_magic == cpu_to_be32(GFS2_MAGIC))
910                 ri->ri_generation_deleted = cpu_to_be64(generation);
911 }
912
913 bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation)
914 {
915         struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
916
917         if (ri->ri_magic != cpu_to_be32(GFS2_MAGIC))
918                 return false;
919         return generation <= be64_to_cpu(ri->ri_generation_deleted);
920 }
921
922 static void gfs2_glock_poke(struct gfs2_glock *gl)
923 {
924         int flags = LM_FLAG_TRY_1CB | LM_FLAG_ANY | GL_SKIP;
925         struct gfs2_holder gh;
926         int error;
927
928         __gfs2_holder_init(gl, LM_ST_SHARED, flags, &gh, _RET_IP_);
929         error = gfs2_glock_nq(&gh);
930         if (!error)
931                 gfs2_glock_dq(&gh);
932         gfs2_holder_uninit(&gh);
933 }
934
935 static bool gfs2_try_evict(struct gfs2_glock *gl)
936 {
937         struct gfs2_inode *ip;
938         bool evicted = false;
939
940         /*
941          * If there is contention on the iopen glock and we have an inode, try
942          * to grab and release the inode so that it can be evicted.  This will
943          * allow the remote node to go ahead and delete the inode without us
944          * having to do it, which will avoid rgrp glock thrashing.
945          *
946          * The remote node is likely still holding the corresponding inode
947          * glock, so it will run before we get to verify that the delete has
948          * happened below.
949          */
950         spin_lock(&gl->gl_lockref.lock);
951         ip = gl->gl_object;
952         if (ip && !igrab(&ip->i_inode))
953                 ip = NULL;
954         spin_unlock(&gl->gl_lockref.lock);
955         if (ip) {
956                 struct gfs2_glock *inode_gl = NULL;
957
958                 gl->gl_no_formal_ino = ip->i_no_formal_ino;
959                 set_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
960                 d_prune_aliases(&ip->i_inode);
961                 iput(&ip->i_inode);
962
963                 /* If the inode was evicted, gl->gl_object will now be NULL. */
964                 spin_lock(&gl->gl_lockref.lock);
965                 ip = gl->gl_object;
966                 if (ip) {
967                         inode_gl = ip->i_gl;
968                         lockref_get(&inode_gl->gl_lockref);
969                         clear_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
970                 }
971                 spin_unlock(&gl->gl_lockref.lock);
972                 if (inode_gl) {
973                         gfs2_glock_poke(inode_gl);
974                         gfs2_glock_put(inode_gl);
975                 }
976                 evicted = !ip;
977         }
978         return evicted;
979 }
980
981 static void delete_work_func(struct work_struct *work)
982 {
983         struct delayed_work *dwork = to_delayed_work(work);
984         struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete);
985         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
986         struct inode *inode;
987         u64 no_addr = gl->gl_name.ln_number;
988
989         spin_lock(&gl->gl_lockref.lock);
990         clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
991         spin_unlock(&gl->gl_lockref.lock);
992
993         if (test_bit(GLF_DEMOTE, &gl->gl_flags)) {
994                 /*
995                  * If we can evict the inode, give the remote node trying to
996                  * delete the inode some time before verifying that the delete
997                  * has happened.  Otherwise, if we cause contention on the inode glock
998                  * immediately, the remote node will think that we still have
999                  * the inode in use, and so it will give up waiting.
1000                  *
1001                  * If we can't evict the inode, signal to the remote node that
1002                  * the inode is still in use.  We'll later try to delete the
1003                  * inode locally in gfs2_evict_inode.
1004                  *
1005                  * FIXME: We only need to verify that the remote node has
1006                  * deleted the inode because nodes before this remote delete
1007                  * rework won't cooperate.  At a later time, when we no longer
1008                  * care about compatibility with such nodes, we can skip this
1009                  * step entirely.
1010                  */
1011                 if (gfs2_try_evict(gl)) {
1012                         if (gfs2_queue_delete_work(gl, 5 * HZ))
1013                                 return;
1014                 }
1015                 goto out;
1016         }
1017
1018         inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino,
1019                                     GFS2_BLKST_UNLINKED);
1020         if (!IS_ERR_OR_NULL(inode)) {
1021                 d_prune_aliases(inode);
1022                 iput(inode);
1023         }
1024 out:
1025         gfs2_glock_put(gl);
1026 }
1027
1028 static void glock_work_func(struct work_struct *work)
1029 {
1030         unsigned long delay = 0;
1031         struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
1032         unsigned int drop_refs = 1;
1033
1034         if (test_and_clear_bit(GLF_REPLY_PENDING, &gl->gl_flags)) {
1035                 finish_xmote(gl, gl->gl_reply);
1036                 drop_refs++;
1037         }
1038         spin_lock(&gl->gl_lockref.lock);
1039         if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1040             gl->gl_state != LM_ST_UNLOCKED &&
1041             gl->gl_demote_state != LM_ST_EXCLUSIVE) {
1042                 unsigned long holdtime, now = jiffies;
1043
1044                 holdtime = gl->gl_tchange + gl->gl_hold_time;
1045                 if (time_before(now, holdtime))
1046                         delay = holdtime - now;
1047
1048                 if (!delay) {
1049                         clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1050                         gfs2_set_demote(gl);
1051                 }
1052         }
1053         run_queue(gl, 0);
1054         if (delay) {
1055                 /* Keep one glock reference for the work we requeue. */
1056                 drop_refs--;
1057                 if (gl->gl_name.ln_type != LM_TYPE_INODE)
1058                         delay = 0;
1059                 __gfs2_glock_queue_work(gl, delay);
1060         }
1061
1062         /*
1063          * Drop the remaining glock references manually here. (Mind that
1064          * __gfs2_glock_queue_work depends on the lockref spinlock begin held
1065          * here as well.)
1066          */
1067         gl->gl_lockref.count -= drop_refs;
1068         if (!gl->gl_lockref.count) {
1069                 __gfs2_glock_put(gl);
1070                 return;
1071         }
1072         spin_unlock(&gl->gl_lockref.lock);
1073 }
1074
1075 static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
1076                                             struct gfs2_glock *new)
1077 {
1078         struct wait_glock_queue wait;
1079         wait_queue_head_t *wq = glock_waitqueue(name);
1080         struct gfs2_glock *gl;
1081
1082         wait.name = name;
1083         init_wait(&wait.wait);
1084         wait.wait.func = glock_wake_function;
1085
1086 again:
1087         prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
1088         rcu_read_lock();
1089         if (new) {
1090                 gl = rhashtable_lookup_get_insert_fast(&gl_hash_table,
1091                         &new->gl_node, ht_parms);
1092                 if (IS_ERR(gl))
1093                         goto out;
1094         } else {
1095                 gl = rhashtable_lookup_fast(&gl_hash_table,
1096                         name, ht_parms);
1097         }
1098         if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
1099                 rcu_read_unlock();
1100                 schedule();
1101                 goto again;
1102         }
1103 out:
1104         rcu_read_unlock();
1105         finish_wait(wq, &wait.wait);
1106         return gl;
1107 }
1108
1109 /**
1110  * gfs2_glock_get() - Get a glock, or create one if one doesn't exist
1111  * @sdp: The GFS2 superblock
1112  * @number: the lock number
1113  * @glops: The glock_operations to use
1114  * @create: If 0, don't create the glock if it doesn't exist
1115  * @glp: the glock is returned here
1116  *
1117  * This does not lock a glock, just finds/creates structures for one.
1118  *
1119  * Returns: errno
1120  */
1121
1122 int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
1123                    const struct gfs2_glock_operations *glops, int create,
1124                    struct gfs2_glock **glp)
1125 {
1126         struct super_block *s = sdp->sd_vfs;
1127         struct lm_lockname name = { .ln_number = number,
1128                                     .ln_type = glops->go_type,
1129                                     .ln_sbd = sdp };
1130         struct gfs2_glock *gl, *tmp;
1131         struct address_space *mapping;
1132         int ret = 0;
1133
1134         gl = find_insert_glock(&name, NULL);
1135         if (gl) {
1136                 *glp = gl;
1137                 return 0;
1138         }
1139         if (!create)
1140                 return -ENOENT;
1141
1142         if (glops->go_flags & GLOF_ASPACE) {
1143                 struct gfs2_glock_aspace *gla =
1144                         kmem_cache_alloc(gfs2_glock_aspace_cachep, GFP_NOFS);
1145                 if (!gla)
1146                         return -ENOMEM;
1147                 gl = &gla->glock;
1148         } else {
1149                 gl = kmem_cache_alloc(gfs2_glock_cachep, GFP_NOFS);
1150                 if (!gl)
1151                         return -ENOMEM;
1152         }
1153         memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
1154         gl->gl_ops = glops;
1155
1156         if (glops->go_flags & GLOF_LVB) {
1157                 gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS);
1158                 if (!gl->gl_lksb.sb_lvbptr) {
1159                         gfs2_glock_dealloc(&gl->gl_rcu);
1160                         return -ENOMEM;
1161                 }
1162         }
1163
1164         atomic_inc(&sdp->sd_glock_disposal);
1165         gl->gl_node.next = NULL;
1166         gl->gl_flags = glops->go_instantiate ? BIT(GLF_INSTANTIATE_NEEDED) : 0;
1167         gl->gl_name = name;
1168         lockdep_set_subclass(&gl->gl_lockref.lock, glops->go_subclass);
1169         gl->gl_lockref.count = 1;
1170         gl->gl_state = LM_ST_UNLOCKED;
1171         gl->gl_target = LM_ST_UNLOCKED;
1172         gl->gl_demote_state = LM_ST_EXCLUSIVE;
1173         gl->gl_dstamp = 0;
1174         preempt_disable();
1175         /* We use the global stats to estimate the initial per-glock stats */
1176         gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
1177         preempt_enable();
1178         gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
1179         gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
1180         gl->gl_tchange = jiffies;
1181         gl->gl_object = NULL;
1182         gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
1183         INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
1184         if (gl->gl_name.ln_type == LM_TYPE_IOPEN)
1185                 INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func);
1186
1187         mapping = gfs2_glock2aspace(gl);
1188         if (mapping) {
1189                 mapping->a_ops = &gfs2_meta_aops;
1190                 mapping->host = s->s_bdev->bd_inode;
1191                 mapping->flags = 0;
1192                 mapping_set_gfp_mask(mapping, GFP_NOFS);
1193                 mapping->private_data = NULL;
1194                 mapping->writeback_index = 0;
1195         }
1196
1197         tmp = find_insert_glock(&name, gl);
1198         if (!tmp) {
1199                 *glp = gl;
1200                 goto out;
1201         }
1202         if (IS_ERR(tmp)) {
1203                 ret = PTR_ERR(tmp);
1204                 goto out_free;
1205         }
1206         *glp = tmp;
1207
1208 out_free:
1209         gfs2_glock_dealloc(&gl->gl_rcu);
1210         if (atomic_dec_and_test(&sdp->sd_glock_disposal))
1211                 wake_up(&sdp->sd_glock_wait);
1212
1213 out:
1214         return ret;
1215 }
1216
1217 /**
1218  * __gfs2_holder_init - initialize a struct gfs2_holder in the default way
1219  * @gl: the glock
1220  * @state: the state we're requesting
1221  * @flags: the modifier flags
1222  * @gh: the holder structure
1223  *
1224  */
1225
1226 void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
1227                         struct gfs2_holder *gh, unsigned long ip)
1228 {
1229         INIT_LIST_HEAD(&gh->gh_list);
1230         gh->gh_gl = gl;
1231         gh->gh_ip = ip;
1232         gh->gh_owner_pid = get_pid(task_pid(current));
1233         gh->gh_state = state;
1234         gh->gh_flags = flags;
1235         gh->gh_iflags = 0;
1236         gfs2_glock_hold(gl);
1237 }
1238
1239 /**
1240  * gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it
1241  * @state: the state we're requesting
1242  * @flags: the modifier flags
1243  * @gh: the holder structure
1244  *
1245  * Don't mess with the glock.
1246  *
1247  */
1248
1249 void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
1250 {
1251         gh->gh_state = state;
1252         gh->gh_flags = flags;
1253         gh->gh_iflags = 0;
1254         gh->gh_ip = _RET_IP_;
1255         put_pid(gh->gh_owner_pid);
1256         gh->gh_owner_pid = get_pid(task_pid(current));
1257 }
1258
1259 /**
1260  * gfs2_holder_uninit - uninitialize a holder structure (drop glock reference)
1261  * @gh: the holder structure
1262  *
1263  */
1264
1265 void gfs2_holder_uninit(struct gfs2_holder *gh)
1266 {
1267         put_pid(gh->gh_owner_pid);
1268         gfs2_glock_put(gh->gh_gl);
1269         gfs2_holder_mark_uninitialized(gh);
1270         gh->gh_ip = 0;
1271 }
1272
1273 static void gfs2_glock_update_hold_time(struct gfs2_glock *gl,
1274                                         unsigned long start_time)
1275 {
1276         /* Have we waited longer that a second? */
1277         if (time_after(jiffies, start_time + HZ)) {
1278                 /* Lengthen the minimum hold time. */
1279                 gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR,
1280                                        GL_GLOCK_MAX_HOLD);
1281         }
1282 }
1283
1284 /**
1285  * gfs2_glock_holder_ready - holder is ready and its error code can be collected
1286  * @gh: the glock holder
1287  *
1288  * Called when a glock holder no longer needs to be waited for because it is
1289  * now either held (HIF_HOLDER set; gh_error == 0), or acquiring the lock has
1290  * failed (gh_error != 0).
1291  */
1292
1293 int gfs2_glock_holder_ready(struct gfs2_holder *gh)
1294 {
1295         if (gh->gh_error || (gh->gh_flags & GL_SKIP))
1296                 return gh->gh_error;
1297         gh->gh_error = gfs2_instantiate(gh);
1298         if (gh->gh_error)
1299                 gfs2_glock_dq(gh);
1300         return gh->gh_error;
1301 }
1302
1303 /**
1304  * gfs2_glock_wait - wait on a glock acquisition
1305  * @gh: the glock holder
1306  *
1307  * Returns: 0 on success
1308  */
1309
1310 int gfs2_glock_wait(struct gfs2_holder *gh)
1311 {
1312         unsigned long start_time = jiffies;
1313
1314         might_sleep();
1315         wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1316         gfs2_glock_update_hold_time(gh->gh_gl, start_time);
1317         return gfs2_glock_holder_ready(gh);
1318 }
1319
1320 static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
1321 {
1322         int i;
1323
1324         for (i = 0; i < num_gh; i++)
1325                 if (test_bit(HIF_WAIT, &ghs[i].gh_iflags))
1326                         return 1;
1327         return 0;
1328 }
1329
1330 /**
1331  * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions
1332  * @num_gh: the number of holders in the array
1333  * @ghs: the glock holder array
1334  *
1335  * Returns: 0 on success, meaning all glocks have been granted and are held.
1336  *          -ESTALE if the request timed out, meaning all glocks were released,
1337  *          and the caller should retry the operation.
1338  */
1339
1340 int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs)
1341 {
1342         struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
1343         int i, ret = 0, timeout = 0;
1344         unsigned long start_time = jiffies;
1345
1346         might_sleep();
1347         /*
1348          * Total up the (minimum hold time * 2) of all glocks and use that to
1349          * determine the max amount of time we should wait.
1350          */
1351         for (i = 0; i < num_gh; i++)
1352                 timeout += ghs[i].gh_gl->gl_hold_time << 1;
1353
1354         if (!wait_event_timeout(sdp->sd_async_glock_wait,
1355                                 !glocks_pending(num_gh, ghs), timeout)) {
1356                 ret = -ESTALE; /* request timed out. */
1357                 goto out;
1358         }
1359
1360         for (i = 0; i < num_gh; i++) {
1361                 struct gfs2_holder *gh = &ghs[i];
1362                 int ret2;
1363
1364                 if (test_bit(HIF_HOLDER, &gh->gh_iflags)) {
1365                         gfs2_glock_update_hold_time(gh->gh_gl,
1366                                                     start_time);
1367                 }
1368                 ret2 = gfs2_glock_holder_ready(gh);
1369                 if (!ret)
1370                         ret = ret2;
1371         }
1372
1373 out:
1374         if (ret) {
1375                 for (i = 0; i < num_gh; i++) {
1376                         struct gfs2_holder *gh = &ghs[i];
1377
1378                         gfs2_glock_dq(gh);
1379                 }
1380         }
1381         return ret;
1382 }
1383
1384 /**
1385  * handle_callback - process a demote request
1386  * @gl: the glock
1387  * @state: the state the caller wants us to change to
1388  * @delay: zero to demote immediately; otherwise pending demote
1389  * @remote: true if this came from a different cluster node
1390  *
1391  * There are only two requests that we are going to see in actual
1392  * practise: LM_ST_SHARED and LM_ST_UNLOCKED
1393  */
1394
1395 static void handle_callback(struct gfs2_glock *gl, unsigned int state,
1396                             unsigned long delay, bool remote)
1397 {
1398         if (delay)
1399                 set_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1400         else
1401                 gfs2_set_demote(gl);
1402         if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
1403                 gl->gl_demote_state = state;
1404                 gl->gl_demote_time = jiffies;
1405         } else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
1406                         gl->gl_demote_state != state) {
1407                 gl->gl_demote_state = LM_ST_UNLOCKED;
1408         }
1409         if (gl->gl_ops->go_callback)
1410                 gl->gl_ops->go_callback(gl, remote);
1411         trace_gfs2_demote_rq(gl, remote);
1412 }
1413
1414 void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
1415 {
1416         struct va_format vaf;
1417         va_list args;
1418
1419         va_start(args, fmt);
1420
1421         if (seq) {
1422                 seq_vprintf(seq, fmt, args);
1423         } else {
1424                 vaf.fmt = fmt;
1425                 vaf.va = &args;
1426
1427                 pr_err("%pV", &vaf);
1428         }
1429
1430         va_end(args);
1431 }
1432
1433 /**
1434  * add_to_queue - Add a holder to the wait queue (but look for recursion)
1435  * @gh: the holder structure to add
1436  *
1437  * Eventually we should move the recursive locking trap to a
1438  * debugging option or something like that. This is the fast
1439  * path and needs to have the minimum number of distractions.
1440  * 
1441  */
1442
1443 static inline void add_to_queue(struct gfs2_holder *gh)
1444 __releases(&gl->gl_lockref.lock)
1445 __acquires(&gl->gl_lockref.lock)
1446 {
1447         struct gfs2_glock *gl = gh->gh_gl;
1448         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1449         struct list_head *insert_pt = NULL;
1450         struct gfs2_holder *gh2;
1451         int try_futile = 0;
1452
1453         GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
1454         if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))
1455                 GLOCK_BUG_ON(gl, true);
1456
1457         if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
1458                 if (test_bit(GLF_LOCK, &gl->gl_flags)) {
1459                         struct gfs2_holder *first_gh;
1460
1461                         first_gh = find_first_strong_holder(gl);
1462                         try_futile = !may_grant(gl, first_gh, gh);
1463                 }
1464                 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
1465                         goto fail;
1466         }
1467
1468         list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1469                 if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&
1470                     (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK) &&
1471                     !test_bit(HIF_MAY_DEMOTE, &gh2->gh_iflags)))
1472                         goto trap_recursive;
1473                 if (try_futile &&
1474                     !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
1475 fail:
1476                         gh->gh_error = GLR_TRYFAILED;
1477                         gfs2_holder_wake(gh);
1478                         return;
1479                 }
1480                 if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1481                         continue;
1482                 if (unlikely((gh->gh_flags & LM_FLAG_PRIORITY) && !insert_pt))
1483                         insert_pt = &gh2->gh_list;
1484         }
1485         trace_gfs2_glock_queue(gh, 1);
1486         gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT);
1487         gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT);
1488         if (likely(insert_pt == NULL)) {
1489                 list_add_tail(&gh->gh_list, &gl->gl_holders);
1490                 if (unlikely(gh->gh_flags & LM_FLAG_PRIORITY))
1491                         goto do_cancel;
1492                 return;
1493         }
1494         list_add_tail(&gh->gh_list, insert_pt);
1495 do_cancel:
1496         gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
1497         if (!(gh->gh_flags & LM_FLAG_PRIORITY)) {
1498                 spin_unlock(&gl->gl_lockref.lock);
1499                 if (sdp->sd_lockstruct.ls_ops->lm_cancel)
1500                         sdp->sd_lockstruct.ls_ops->lm_cancel(gl);
1501                 spin_lock(&gl->gl_lockref.lock);
1502         }
1503         return;
1504
1505 trap_recursive:
1506         fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1507         fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1508         fs_err(sdp, "lock type: %d req lock state : %d\n",
1509                gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1510         fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1511         fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1512         fs_err(sdp, "lock type: %d req lock state : %d\n",
1513                gh->gh_gl->gl_name.ln_type, gh->gh_state);
1514         gfs2_dump_glock(NULL, gl, true);
1515         BUG();
1516 }
1517
1518 /**
1519  * gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock)
1520  * @gh: the holder structure
1521  *
1522  * if (gh->gh_flags & GL_ASYNC), this never returns an error
1523  *
1524  * Returns: 0, GLR_TRYFAILED, or errno on failure
1525  */
1526
1527 int gfs2_glock_nq(struct gfs2_holder *gh)
1528 {
1529         struct gfs2_glock *gl = gh->gh_gl;
1530         int error = 0;
1531
1532         if (glock_blocked_by_withdraw(gl) && !(gh->gh_flags & LM_FLAG_NOEXP))
1533                 return -EIO;
1534
1535         if (test_bit(GLF_LRU, &gl->gl_flags))
1536                 gfs2_glock_remove_from_lru(gl);
1537
1538         gh->gh_error = 0;
1539         spin_lock(&gl->gl_lockref.lock);
1540         add_to_queue(gh);
1541         if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&
1542                      test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) {
1543                 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1544                 gl->gl_lockref.count++;
1545                 __gfs2_glock_queue_work(gl, 0);
1546         }
1547         run_queue(gl, 1);
1548         spin_unlock(&gl->gl_lockref.lock);
1549
1550         if (!(gh->gh_flags & GL_ASYNC))
1551                 error = gfs2_glock_wait(gh);
1552
1553         return error;
1554 }
1555
1556 /**
1557  * gfs2_glock_poll - poll to see if an async request has been completed
1558  * @gh: the holder
1559  *
1560  * Returns: 1 if the request is ready to be gfs2_glock_wait()ed on
1561  */
1562
1563 int gfs2_glock_poll(struct gfs2_holder *gh)
1564 {
1565         return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1566 }
1567
1568 static inline bool needs_demote(struct gfs2_glock *gl)
1569 {
1570         return (test_bit(GLF_DEMOTE, &gl->gl_flags) ||
1571                 test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags));
1572 }
1573
1574 static void __gfs2_glock_dq(struct gfs2_holder *gh)
1575 {
1576         struct gfs2_glock *gl = gh->gh_gl;
1577         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1578         unsigned delay = 0;
1579         int fast_path = 0;
1580
1581         /*
1582          * This while loop is similar to function demote_incompat_holders:
1583          * If the glock is due to be demoted (which may be from another node
1584          * or even if this holder is GL_NOCACHE), the weak holders are
1585          * demoted as well, allowing the glock to be demoted.
1586          */
1587         while (gh) {
1588                 /*
1589                  * If we're in the process of file system withdraw, we cannot
1590                  * just dequeue any glocks until our journal is recovered, lest
1591                  * we introduce file system corruption. We need two exceptions
1592                  * to this rule: We need to allow unlocking of nondisk glocks
1593                  * and the glock for our own journal that needs recovery.
1594                  */
1595                 if (test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags) &&
1596                     glock_blocked_by_withdraw(gl) &&
1597                     gh->gh_gl != sdp->sd_jinode_gl) {
1598                         sdp->sd_glock_dqs_held++;
1599                         spin_unlock(&gl->gl_lockref.lock);
1600                         might_sleep();
1601                         wait_on_bit(&sdp->sd_flags, SDF_WITHDRAW_RECOVERY,
1602                                     TASK_UNINTERRUPTIBLE);
1603                         spin_lock(&gl->gl_lockref.lock);
1604                 }
1605
1606                 /*
1607                  * This holder should not be cached, so mark it for demote.
1608                  * Note: this should be done before the check for needs_demote
1609                  * below.
1610                  */
1611                 if (gh->gh_flags & GL_NOCACHE)
1612                         handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1613
1614                 list_del_init(&gh->gh_list);
1615                 clear_bit(HIF_HOLDER, &gh->gh_iflags);
1616                 trace_gfs2_glock_queue(gh, 0);
1617
1618                 /*
1619                  * If there hasn't been a demote request we are done.
1620                  * (Let the remaining holders, if any, keep holding it.)
1621                  */
1622                 if (!needs_demote(gl)) {
1623                         if (list_empty(&gl->gl_holders))
1624                                 fast_path = 1;
1625                         break;
1626                 }
1627                 /*
1628                  * If we have another strong holder (we cannot auto-demote)
1629                  * we are done. It keeps holding it until it is done.
1630                  */
1631                 if (find_first_strong_holder(gl))
1632                         break;
1633
1634                 /*
1635                  * If we have a weak holder at the head of the list, it
1636                  * (and all others like it) must be auto-demoted. If there
1637                  * are no more weak holders, we exit the while loop.
1638                  */
1639                 gh = find_first_holder(gl);
1640         }
1641
1642         if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
1643                 gfs2_glock_add_to_lru(gl);
1644
1645         if (unlikely(!fast_path)) {
1646                 gl->gl_lockref.count++;
1647                 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1648                     !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1649                     gl->gl_name.ln_type == LM_TYPE_INODE)
1650                         delay = gl->gl_hold_time;
1651                 __gfs2_glock_queue_work(gl, delay);
1652         }
1653 }
1654
1655 /**
1656  * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
1657  * @gh: the glock holder
1658  *
1659  */
1660 void gfs2_glock_dq(struct gfs2_holder *gh)
1661 {
1662         struct gfs2_glock *gl = gh->gh_gl;
1663
1664         spin_lock(&gl->gl_lockref.lock);
1665         if (list_is_first(&gh->gh_list, &gl->gl_holders) &&
1666             !test_bit(HIF_HOLDER, &gh->gh_iflags)) {
1667                 spin_unlock(&gl->gl_lockref.lock);
1668                 gl->gl_name.ln_sbd->sd_lockstruct.ls_ops->lm_cancel(gl);
1669                 wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1670                 spin_lock(&gl->gl_lockref.lock);
1671         }
1672
1673         __gfs2_glock_dq(gh);
1674         spin_unlock(&gl->gl_lockref.lock);
1675 }
1676
1677 void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1678 {
1679         struct gfs2_glock *gl = gh->gh_gl;
1680         gfs2_glock_dq(gh);
1681         might_sleep();
1682         wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1683 }
1684
1685 /**
1686  * gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it
1687  * @gh: the holder structure
1688  *
1689  */
1690
1691 void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1692 {
1693         gfs2_glock_dq(gh);
1694         gfs2_holder_uninit(gh);
1695 }
1696
1697 /**
1698  * gfs2_glock_nq_num - acquire a glock based on lock number
1699  * @sdp: the filesystem
1700  * @number: the lock number
1701  * @glops: the glock operations for the type of glock
1702  * @state: the state to acquire the glock in
1703  * @flags: modifier flags for the acquisition
1704  * @gh: the struct gfs2_holder
1705  *
1706  * Returns: errno
1707  */
1708
1709 int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1710                       const struct gfs2_glock_operations *glops,
1711                       unsigned int state, u16 flags, struct gfs2_holder *gh)
1712 {
1713         struct gfs2_glock *gl;
1714         int error;
1715
1716         error = gfs2_glock_get(sdp, number, glops, CREATE, &gl);
1717         if (!error) {
1718                 error = gfs2_glock_nq_init(gl, state, flags, gh);
1719                 gfs2_glock_put(gl);
1720         }
1721
1722         return error;
1723 }
1724
1725 /**
1726  * glock_compare - Compare two struct gfs2_glock structures for sorting
1727  * @arg_a: the first structure
1728  * @arg_b: the second structure
1729  *
1730  */
1731
1732 static int glock_compare(const void *arg_a, const void *arg_b)
1733 {
1734         const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1735         const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1736         const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1737         const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1738
1739         if (a->ln_number > b->ln_number)
1740                 return 1;
1741         if (a->ln_number < b->ln_number)
1742                 return -1;
1743         BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1744         return 0;
1745 }
1746
1747 /**
1748  * nq_m_sync - synchonously acquire more than one glock in deadlock free order
1749  * @num_gh: the number of structures
1750  * @ghs: an array of struct gfs2_holder structures
1751  * @p: placeholder for the holder structure to pass back
1752  *
1753  * Returns: 0 on success (all glocks acquired),
1754  *          errno on failure (no glocks acquired)
1755  */
1756
1757 static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1758                      struct gfs2_holder **p)
1759 {
1760         unsigned int x;
1761         int error = 0;
1762
1763         for (x = 0; x < num_gh; x++)
1764                 p[x] = &ghs[x];
1765
1766         sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL);
1767
1768         for (x = 0; x < num_gh; x++) {
1769                 p[x]->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1770
1771                 error = gfs2_glock_nq(p[x]);
1772                 if (error) {
1773                         while (x--)
1774                                 gfs2_glock_dq(p[x]);
1775                         break;
1776                 }
1777         }
1778
1779         return error;
1780 }
1781
1782 /**
1783  * gfs2_glock_nq_m - acquire multiple glocks
1784  * @num_gh: the number of structures
1785  * @ghs: an array of struct gfs2_holder structures
1786  *
1787  *
1788  * Returns: 0 on success (all glocks acquired),
1789  *          errno on failure (no glocks acquired)
1790  */
1791
1792 int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1793 {
1794         struct gfs2_holder *tmp[4];
1795         struct gfs2_holder **pph = tmp;
1796         int error = 0;
1797
1798         switch(num_gh) {
1799         case 0:
1800                 return 0;
1801         case 1:
1802                 ghs->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1803                 return gfs2_glock_nq(ghs);
1804         default:
1805                 if (num_gh <= 4)
1806                         break;
1807                 pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *),
1808                                     GFP_NOFS);
1809                 if (!pph)
1810                         return -ENOMEM;
1811         }
1812
1813         error = nq_m_sync(num_gh, ghs, pph);
1814
1815         if (pph != tmp)
1816                 kfree(pph);
1817
1818         return error;
1819 }
1820
1821 /**
1822  * gfs2_glock_dq_m - release multiple glocks
1823  * @num_gh: the number of structures
1824  * @ghs: an array of struct gfs2_holder structures
1825  *
1826  */
1827
1828 void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1829 {
1830         while (num_gh--)
1831                 gfs2_glock_dq(&ghs[num_gh]);
1832 }
1833
1834 void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1835 {
1836         unsigned long delay = 0;
1837         unsigned long holdtime;
1838         unsigned long now = jiffies;
1839
1840         gfs2_glock_hold(gl);
1841         spin_lock(&gl->gl_lockref.lock);
1842         holdtime = gl->gl_tchange + gl->gl_hold_time;
1843         if (!list_empty(&gl->gl_holders) &&
1844             gl->gl_name.ln_type == LM_TYPE_INODE) {
1845                 if (time_before(now, holdtime))
1846                         delay = holdtime - now;
1847                 if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
1848                         delay = gl->gl_hold_time;
1849         }
1850         /*
1851          * Note 1: We cannot call demote_incompat_holders from handle_callback
1852          * or gfs2_set_demote due to recursion problems like: gfs2_glock_dq ->
1853          * handle_callback -> demote_incompat_holders -> gfs2_glock_dq
1854          * Plus, we only want to demote the holders if the request comes from
1855          * a remote cluster node because local holder conflicts are resolved
1856          * elsewhere.
1857          *
1858          * Note 2: if a remote node wants this glock in EX mode, lock_dlm will
1859          * request that we set our state to UNLOCKED. Here we mock up a holder
1860          * to make it look like someone wants the lock EX locally. Any SH
1861          * and DF requests should be able to share the lock without demoting.
1862          *
1863          * Note 3: We only want to demote the demoteable holders when there
1864          * are no more strong holders. The demoteable holders might as well
1865          * keep the glock until the last strong holder is done with it.
1866          */
1867         if (!find_first_strong_holder(gl)) {
1868                 struct gfs2_holder mock_gh = {
1869                         .gh_gl = gl,
1870                         .gh_state = (state == LM_ST_UNLOCKED) ?
1871                                     LM_ST_EXCLUSIVE : state,
1872                         .gh_iflags = BIT(HIF_HOLDER)
1873                 };
1874
1875                 demote_incompat_holders(gl, &mock_gh);
1876         }
1877         handle_callback(gl, state, delay, true);
1878         __gfs2_glock_queue_work(gl, delay);
1879         spin_unlock(&gl->gl_lockref.lock);
1880 }
1881
1882 /**
1883  * gfs2_should_freeze - Figure out if glock should be frozen
1884  * @gl: The glock in question
1885  *
1886  * Glocks are not frozen if (a) the result of the dlm operation is
1887  * an error, (b) the locking operation was an unlock operation or
1888  * (c) if there is a "noexp" flagged request anywhere in the queue
1889  *
1890  * Returns: 1 if freezing should occur, 0 otherwise
1891  */
1892
1893 static int gfs2_should_freeze(const struct gfs2_glock *gl)
1894 {
1895         const struct gfs2_holder *gh;
1896
1897         if (gl->gl_reply & ~LM_OUT_ST_MASK)
1898                 return 0;
1899         if (gl->gl_target == LM_ST_UNLOCKED)
1900                 return 0;
1901
1902         list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1903                 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1904                         continue;
1905                 if (LM_FLAG_NOEXP & gh->gh_flags)
1906                         return 0;
1907         }
1908
1909         return 1;
1910 }
1911
1912 /**
1913  * gfs2_glock_complete - Callback used by locking
1914  * @gl: Pointer to the glock
1915  * @ret: The return value from the dlm
1916  *
1917  * The gl_reply field is under the gl_lockref.lock lock so that it is ok
1918  * to use a bitfield shared with other glock state fields.
1919  */
1920
1921 void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1922 {
1923         struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1924
1925         spin_lock(&gl->gl_lockref.lock);
1926         gl->gl_reply = ret;
1927
1928         if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1929                 if (gfs2_should_freeze(gl)) {
1930                         set_bit(GLF_FROZEN, &gl->gl_flags);
1931                         spin_unlock(&gl->gl_lockref.lock);
1932                         return;
1933                 }
1934         }
1935
1936         gl->gl_lockref.count++;
1937         set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1938         __gfs2_glock_queue_work(gl, 0);
1939         spin_unlock(&gl->gl_lockref.lock);
1940 }
1941
1942 static int glock_cmp(void *priv, const struct list_head *a,
1943                      const struct list_head *b)
1944 {
1945         struct gfs2_glock *gla, *glb;
1946
1947         gla = list_entry(a, struct gfs2_glock, gl_lru);
1948         glb = list_entry(b, struct gfs2_glock, gl_lru);
1949
1950         if (gla->gl_name.ln_number > glb->gl_name.ln_number)
1951                 return 1;
1952         if (gla->gl_name.ln_number < glb->gl_name.ln_number)
1953                 return -1;
1954
1955         return 0;
1956 }
1957
1958 /**
1959  * gfs2_dispose_glock_lru - Demote a list of glocks
1960  * @list: The list to dispose of
1961  *
1962  * Disposing of glocks may involve disk accesses, so that here we sort
1963  * the glocks by number (i.e. disk location of the inodes) so that if
1964  * there are any such accesses, they'll be sent in order (mostly).
1965  *
1966  * Must be called under the lru_lock, but may drop and retake this
1967  * lock. While the lru_lock is dropped, entries may vanish from the
1968  * list, but no new entries will appear on the list (since it is
1969  * private)
1970  */
1971
1972 static void gfs2_dispose_glock_lru(struct list_head *list)
1973 __releases(&lru_lock)
1974 __acquires(&lru_lock)
1975 {
1976         struct gfs2_glock *gl;
1977
1978         list_sort(NULL, list, glock_cmp);
1979
1980         while(!list_empty(list)) {
1981                 gl = list_first_entry(list, struct gfs2_glock, gl_lru);
1982                 list_del_init(&gl->gl_lru);
1983                 clear_bit(GLF_LRU, &gl->gl_flags);
1984                 if (!spin_trylock(&gl->gl_lockref.lock)) {
1985 add_back_to_lru:
1986                         list_add(&gl->gl_lru, &lru_list);
1987                         set_bit(GLF_LRU, &gl->gl_flags);
1988                         atomic_inc(&lru_count);
1989                         continue;
1990                 }
1991                 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) {
1992                         spin_unlock(&gl->gl_lockref.lock);
1993                         goto add_back_to_lru;
1994                 }
1995                 gl->gl_lockref.count++;
1996                 if (demote_ok(gl))
1997                         handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1998                 WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags));
1999                 __gfs2_glock_queue_work(gl, 0);
2000                 spin_unlock(&gl->gl_lockref.lock);
2001                 cond_resched_lock(&lru_lock);
2002         }
2003 }
2004
2005 /**
2006  * gfs2_scan_glock_lru - Scan the LRU looking for locks to demote
2007  * @nr: The number of entries to scan
2008  *
2009  * This function selects the entries on the LRU which are able to
2010  * be demoted, and then kicks off the process by calling
2011  * gfs2_dispose_glock_lru() above.
2012  */
2013
2014 static long gfs2_scan_glock_lru(int nr)
2015 {
2016         struct gfs2_glock *gl;
2017         LIST_HEAD(skipped);
2018         LIST_HEAD(dispose);
2019         long freed = 0;
2020
2021         spin_lock(&lru_lock);
2022         while ((nr-- >= 0) && !list_empty(&lru_list)) {
2023                 gl = list_first_entry(&lru_list, struct gfs2_glock, gl_lru);
2024
2025                 /* Test for being demotable */
2026                 if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
2027                         list_move(&gl->gl_lru, &dispose);
2028                         atomic_dec(&lru_count);
2029                         freed++;
2030                         continue;
2031                 }
2032
2033                 list_move(&gl->gl_lru, &skipped);
2034         }
2035         list_splice(&skipped, &lru_list);
2036         if (!list_empty(&dispose))
2037                 gfs2_dispose_glock_lru(&dispose);
2038         spin_unlock(&lru_lock);
2039
2040         return freed;
2041 }
2042
2043 static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
2044                                             struct shrink_control *sc)
2045 {
2046         if (!(sc->gfp_mask & __GFP_FS))
2047                 return SHRINK_STOP;
2048         return gfs2_scan_glock_lru(sc->nr_to_scan);
2049 }
2050
2051 static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
2052                                              struct shrink_control *sc)
2053 {
2054         return vfs_pressure_ratio(atomic_read(&lru_count));
2055 }
2056
2057 static struct shrinker glock_shrinker = {
2058         .seeks = DEFAULT_SEEKS,
2059         .count_objects = gfs2_glock_shrink_count,
2060         .scan_objects = gfs2_glock_shrink_scan,
2061 };
2062
2063 /**
2064  * glock_hash_walk - Call a function for glock in a hash bucket
2065  * @examiner: the function
2066  * @sdp: the filesystem
2067  *
2068  * Note that the function can be called multiple times on the same
2069  * object.  So the user must ensure that the function can cope with
2070  * that.
2071  */
2072
2073 static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
2074 {
2075         struct gfs2_glock *gl;
2076         struct rhashtable_iter iter;
2077
2078         rhashtable_walk_enter(&gl_hash_table, &iter);
2079
2080         do {
2081                 rhashtable_walk_start(&iter);
2082
2083                 while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) {
2084                         if (gl->gl_name.ln_sbd == sdp)
2085                                 examiner(gl);
2086                 }
2087
2088                 rhashtable_walk_stop(&iter);
2089         } while (cond_resched(), gl == ERR_PTR(-EAGAIN));
2090
2091         rhashtable_walk_exit(&iter);
2092 }
2093
2094 bool gfs2_queue_delete_work(struct gfs2_glock *gl, unsigned long delay)
2095 {
2096         bool queued;
2097
2098         spin_lock(&gl->gl_lockref.lock);
2099         queued = queue_delayed_work(gfs2_delete_workqueue,
2100                                     &gl->gl_delete, delay);
2101         if (queued)
2102                 set_bit(GLF_PENDING_DELETE, &gl->gl_flags);
2103         spin_unlock(&gl->gl_lockref.lock);
2104         return queued;
2105 }
2106
2107 void gfs2_cancel_delete_work(struct gfs2_glock *gl)
2108 {
2109         if (cancel_delayed_work(&gl->gl_delete)) {
2110                 clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
2111                 gfs2_glock_put(gl);
2112         }
2113 }
2114
2115 bool gfs2_delete_work_queued(const struct gfs2_glock *gl)
2116 {
2117         return test_bit(GLF_PENDING_DELETE, &gl->gl_flags);
2118 }
2119
2120 static void flush_delete_work(struct gfs2_glock *gl)
2121 {
2122         if (gl->gl_name.ln_type == LM_TYPE_IOPEN) {
2123                 if (cancel_delayed_work(&gl->gl_delete)) {
2124                         queue_delayed_work(gfs2_delete_workqueue,
2125                                            &gl->gl_delete, 0);
2126                 }
2127         }
2128 }
2129
2130 void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
2131 {
2132         glock_hash_walk(flush_delete_work, sdp);
2133         flush_workqueue(gfs2_delete_workqueue);
2134 }
2135
2136 /**
2137  * thaw_glock - thaw out a glock which has an unprocessed reply waiting
2138  * @gl: The glock to thaw
2139  *
2140  */
2141
2142 static void thaw_glock(struct gfs2_glock *gl)
2143 {
2144         if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))
2145                 return;
2146         if (!lockref_get_not_dead(&gl->gl_lockref))
2147                 return;
2148         set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
2149         gfs2_glock_queue_work(gl, 0);
2150 }
2151
2152 /**
2153  * clear_glock - look at a glock and see if we can free it from glock cache
2154  * @gl: the glock to look at
2155  *
2156  */
2157
2158 static void clear_glock(struct gfs2_glock *gl)
2159 {
2160         gfs2_glock_remove_from_lru(gl);
2161
2162         spin_lock(&gl->gl_lockref.lock);
2163         if (!__lockref_is_dead(&gl->gl_lockref)) {
2164                 gl->gl_lockref.count++;
2165                 if (gl->gl_state != LM_ST_UNLOCKED)
2166                         handle_callback(gl, LM_ST_UNLOCKED, 0, false);
2167                 __gfs2_glock_queue_work(gl, 0);
2168         }
2169         spin_unlock(&gl->gl_lockref.lock);
2170 }
2171
2172 /**
2173  * gfs2_glock_thaw - Thaw any frozen glocks
2174  * @sdp: The super block
2175  *
2176  */
2177
2178 void gfs2_glock_thaw(struct gfs2_sbd *sdp)
2179 {
2180         glock_hash_walk(thaw_glock, sdp);
2181 }
2182
2183 static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2184 {
2185         spin_lock(&gl->gl_lockref.lock);
2186         gfs2_dump_glock(seq, gl, fsid);
2187         spin_unlock(&gl->gl_lockref.lock);
2188 }
2189
2190 static void dump_glock_func(struct gfs2_glock *gl)
2191 {
2192         dump_glock(NULL, gl, true);
2193 }
2194
2195 /**
2196  * gfs2_gl_hash_clear - Empty out the glock hash table
2197  * @sdp: the filesystem
2198  *
2199  * Called when unmounting the filesystem.
2200  */
2201
2202 void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
2203 {
2204         set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
2205         flush_workqueue(glock_workqueue);
2206         glock_hash_walk(clear_glock, sdp);
2207         flush_workqueue(glock_workqueue);
2208         wait_event_timeout(sdp->sd_glock_wait,
2209                            atomic_read(&sdp->sd_glock_disposal) == 0,
2210                            HZ * 600);
2211         glock_hash_walk(dump_glock_func, sdp);
2212 }
2213
2214 static const char *state2str(unsigned state)
2215 {
2216         switch(state) {
2217         case LM_ST_UNLOCKED:
2218                 return "UN";
2219         case LM_ST_SHARED:
2220                 return "SH";
2221         case LM_ST_DEFERRED:
2222                 return "DF";
2223         case LM_ST_EXCLUSIVE:
2224                 return "EX";
2225         }
2226         return "??";
2227 }
2228
2229 static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
2230 {
2231         char *p = buf;
2232         if (flags & LM_FLAG_TRY)
2233                 *p++ = 't';
2234         if (flags & LM_FLAG_TRY_1CB)
2235                 *p++ = 'T';
2236         if (flags & LM_FLAG_NOEXP)
2237                 *p++ = 'e';
2238         if (flags & LM_FLAG_ANY)
2239                 *p++ = 'A';
2240         if (flags & LM_FLAG_PRIORITY)
2241                 *p++ = 'p';
2242         if (flags & LM_FLAG_NODE_SCOPE)
2243                 *p++ = 'n';
2244         if (flags & GL_ASYNC)
2245                 *p++ = 'a';
2246         if (flags & GL_EXACT)
2247                 *p++ = 'E';
2248         if (flags & GL_NOCACHE)
2249                 *p++ = 'c';
2250         if (test_bit(HIF_HOLDER, &iflags))
2251                 *p++ = 'H';
2252         if (test_bit(HIF_WAIT, &iflags))
2253                 *p++ = 'W';
2254         if (test_bit(HIF_MAY_DEMOTE, &iflags))
2255                 *p++ = 'D';
2256         if (flags & GL_SKIP)
2257                 *p++ = 's';
2258         *p = 0;
2259         return buf;
2260 }
2261
2262 /**
2263  * dump_holder - print information about a glock holder
2264  * @seq: the seq_file struct
2265  * @gh: the glock holder
2266  * @fs_id_buf: pointer to file system id (if requested)
2267  *
2268  */
2269
2270 static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
2271                         const char *fs_id_buf)
2272 {
2273         struct task_struct *gh_owner = NULL;
2274         char flags_buf[32];
2275
2276         rcu_read_lock();
2277         if (gh->gh_owner_pid)
2278                 gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
2279         gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
2280                        fs_id_buf, state2str(gh->gh_state),
2281                        hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
2282                        gh->gh_error,
2283                        gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1,
2284                        gh_owner ? gh_owner->comm : "(ended)",
2285                        (void *)gh->gh_ip);
2286         rcu_read_unlock();
2287 }
2288
2289 static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
2290 {
2291         const unsigned long *gflags = &gl->gl_flags;
2292         char *p = buf;
2293
2294         if (test_bit(GLF_LOCK, gflags))
2295                 *p++ = 'l';
2296         if (test_bit(GLF_DEMOTE, gflags))
2297                 *p++ = 'D';
2298         if (test_bit(GLF_PENDING_DEMOTE, gflags))
2299                 *p++ = 'd';
2300         if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
2301                 *p++ = 'p';
2302         if (test_bit(GLF_DIRTY, gflags))
2303                 *p++ = 'y';
2304         if (test_bit(GLF_LFLUSH, gflags))
2305                 *p++ = 'f';
2306         if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags))
2307                 *p++ = 'i';
2308         if (test_bit(GLF_REPLY_PENDING, gflags))
2309                 *p++ = 'r';
2310         if (test_bit(GLF_INITIAL, gflags))
2311                 *p++ = 'I';
2312         if (test_bit(GLF_FROZEN, gflags))
2313                 *p++ = 'F';
2314         if (!list_empty(&gl->gl_holders))
2315                 *p++ = 'q';
2316         if (test_bit(GLF_LRU, gflags))
2317                 *p++ = 'L';
2318         if (gl->gl_object)
2319                 *p++ = 'o';
2320         if (test_bit(GLF_BLOCKING, gflags))
2321                 *p++ = 'b';
2322         if (test_bit(GLF_PENDING_DELETE, gflags))
2323                 *p++ = 'P';
2324         if (test_bit(GLF_FREEING, gflags))
2325                 *p++ = 'x';
2326         if (test_bit(GLF_INSTANTIATE_NEEDED, gflags))
2327                 *p++ = 'n';
2328         if (test_bit(GLF_INSTANTIATE_IN_PROG, gflags))
2329                 *p++ = 'N';
2330         *p = 0;
2331         return buf;
2332 }
2333
2334 /**
2335  * gfs2_dump_glock - print information about a glock
2336  * @seq: The seq_file struct
2337  * @gl: the glock
2338  * @fsid: If true, also dump the file system id
2339  *
2340  * The file format is as follows:
2341  * One line per object, capital letters are used to indicate objects
2342  * G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented,
2343  * other objects are indented by a single space and follow the glock to
2344  * which they are related. Fields are indicated by lower case letters
2345  * followed by a colon and the field value, except for strings which are in
2346  * [] so that its possible to see if they are composed of spaces for
2347  * example. The field's are n = number (id of the object), f = flags,
2348  * t = type, s = state, r = refcount, e = error, p = pid.
2349  *
2350  */
2351
2352 void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2353 {
2354         const struct gfs2_glock_operations *glops = gl->gl_ops;
2355         unsigned long long dtime;
2356         const struct gfs2_holder *gh;
2357         char gflags_buf[32];
2358         struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2359         char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
2360         unsigned long nrpages = 0;
2361
2362         if (gl->gl_ops->go_flags & GLOF_ASPACE) {
2363                 struct address_space *mapping = gfs2_glock2aspace(gl);
2364
2365                 nrpages = mapping->nrpages;
2366         }
2367         memset(fs_id_buf, 0, sizeof(fs_id_buf));
2368         if (fsid && sdp) /* safety precaution */
2369                 sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname);
2370         dtime = jiffies - gl->gl_demote_time;
2371         dtime *= 1000000/HZ; /* demote time in uSec */
2372         if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
2373                 dtime = 0;
2374         gfs2_print_dbg(seq, "%sG:  s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
2375                        "v:%d r:%d m:%ld p:%lu\n",
2376                        fs_id_buf, state2str(gl->gl_state),
2377                        gl->gl_name.ln_type,
2378                        (unsigned long long)gl->gl_name.ln_number,
2379                        gflags2str(gflags_buf, gl),
2380                        state2str(gl->gl_target),
2381                        state2str(gl->gl_demote_state), dtime,
2382                        atomic_read(&gl->gl_ail_count),
2383                        atomic_read(&gl->gl_revokes),
2384                        (int)gl->gl_lockref.count, gl->gl_hold_time, nrpages);
2385
2386         list_for_each_entry(gh, &gl->gl_holders, gh_list)
2387                 dump_holder(seq, gh, fs_id_buf);
2388
2389         if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
2390                 glops->go_dump(seq, gl, fs_id_buf);
2391 }
2392
2393 static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
2394 {
2395         struct gfs2_glock *gl = iter_ptr;
2396
2397         seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
2398                    gl->gl_name.ln_type,
2399                    (unsigned long long)gl->gl_name.ln_number,
2400                    (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
2401                    (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
2402                    (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
2403                    (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
2404                    (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
2405                    (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
2406                    (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
2407                    (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
2408         return 0;
2409 }
2410
2411 static const char *gfs2_gltype[] = {
2412         "type",
2413         "reserved",
2414         "nondisk",
2415         "inode",
2416         "rgrp",
2417         "meta",
2418         "iopen",
2419         "flock",
2420         "plock",
2421         "quota",
2422         "journal",
2423 };
2424
2425 static const char *gfs2_stype[] = {
2426         [GFS2_LKS_SRTT]         = "srtt",
2427         [GFS2_LKS_SRTTVAR]      = "srttvar",
2428         [GFS2_LKS_SRTTB]        = "srttb",
2429         [GFS2_LKS_SRTTVARB]     = "srttvarb",
2430         [GFS2_LKS_SIRT]         = "sirt",
2431         [GFS2_LKS_SIRTVAR]      = "sirtvar",
2432         [GFS2_LKS_DCOUNT]       = "dlm",
2433         [GFS2_LKS_QCOUNT]       = "queue",
2434 };
2435
2436 #define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
2437
2438 static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
2439 {
2440         struct gfs2_sbd *sdp = seq->private;
2441         loff_t pos = *(loff_t *)iter_ptr;
2442         unsigned index = pos >> 3;
2443         unsigned subindex = pos & 0x07;
2444         int i;
2445
2446         if (index == 0 && subindex != 0)
2447                 return 0;
2448
2449         seq_printf(seq, "%-10s %8s:", gfs2_gltype[index],
2450                    (index == 0) ? "cpu": gfs2_stype[subindex]);
2451
2452         for_each_possible_cpu(i) {
2453                 const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
2454
2455                 if (index == 0)
2456                         seq_printf(seq, " %15u", i);
2457                 else
2458                         seq_printf(seq, " %15llu", (unsigned long long)lkstats->
2459                                    lkstats[index - 1].stats[subindex]);
2460         }
2461         seq_putc(seq, '\n');
2462         return 0;
2463 }
2464
2465 int __init gfs2_glock_init(void)
2466 {
2467         int i, ret;
2468
2469         ret = rhashtable_init(&gl_hash_table, &ht_parms);
2470         if (ret < 0)
2471                 return ret;
2472
2473         glock_workqueue = alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM |
2474                                           WQ_HIGHPRI | WQ_FREEZABLE, 0);
2475         if (!glock_workqueue) {
2476                 rhashtable_destroy(&gl_hash_table);
2477                 return -ENOMEM;
2478         }
2479         gfs2_delete_workqueue = alloc_workqueue("delete_workqueue",
2480                                                 WQ_MEM_RECLAIM | WQ_FREEZABLE,
2481                                                 0);
2482         if (!gfs2_delete_workqueue) {
2483                 destroy_workqueue(glock_workqueue);
2484                 rhashtable_destroy(&gl_hash_table);
2485                 return -ENOMEM;
2486         }
2487
2488         ret = register_shrinker(&glock_shrinker);
2489         if (ret) {
2490                 destroy_workqueue(gfs2_delete_workqueue);
2491                 destroy_workqueue(glock_workqueue);
2492                 rhashtable_destroy(&gl_hash_table);
2493                 return ret;
2494         }
2495
2496         for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
2497                 init_waitqueue_head(glock_wait_table + i);
2498
2499         return 0;
2500 }
2501
2502 void gfs2_glock_exit(void)
2503 {
2504         unregister_shrinker(&glock_shrinker);
2505         rhashtable_destroy(&gl_hash_table);
2506         destroy_workqueue(glock_workqueue);
2507         destroy_workqueue(gfs2_delete_workqueue);
2508 }
2509
2510 static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
2511 {
2512         struct gfs2_glock *gl = gi->gl;
2513
2514         if (gl) {
2515                 if (n == 0)
2516                         return;
2517                 if (!lockref_put_not_zero(&gl->gl_lockref))
2518                         gfs2_glock_queue_put(gl);
2519         }
2520         for (;;) {
2521                 gl = rhashtable_walk_next(&gi->hti);
2522                 if (IS_ERR_OR_NULL(gl)) {
2523                         if (gl == ERR_PTR(-EAGAIN)) {
2524                                 n = 1;
2525                                 continue;
2526                         }
2527                         gl = NULL;
2528                         break;
2529                 }
2530                 if (gl->gl_name.ln_sbd != gi->sdp)
2531                         continue;
2532                 if (n <= 1) {
2533                         if (!lockref_get_not_dead(&gl->gl_lockref))
2534                                 continue;
2535                         break;
2536                 } else {
2537                         if (__lockref_is_dead(&gl->gl_lockref))
2538                                 continue;
2539                         n--;
2540                 }
2541         }
2542         gi->gl = gl;
2543 }
2544
2545 static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
2546         __acquires(RCU)
2547 {
2548         struct gfs2_glock_iter *gi = seq->private;
2549         loff_t n;
2550
2551         /*
2552          * We can either stay where we are, skip to the next hash table
2553          * entry, or start from the beginning.
2554          */
2555         if (*pos < gi->last_pos) {
2556                 rhashtable_walk_exit(&gi->hti);
2557                 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2558                 n = *pos + 1;
2559         } else {
2560                 n = *pos - gi->last_pos;
2561         }
2562
2563         rhashtable_walk_start(&gi->hti);
2564
2565         gfs2_glock_iter_next(gi, n);
2566         gi->last_pos = *pos;
2567         return gi->gl;
2568 }
2569
2570 static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
2571                                  loff_t *pos)
2572 {
2573         struct gfs2_glock_iter *gi = seq->private;
2574
2575         (*pos)++;
2576         gi->last_pos = *pos;
2577         gfs2_glock_iter_next(gi, 1);
2578         return gi->gl;
2579 }
2580
2581 static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2582         __releases(RCU)
2583 {
2584         struct gfs2_glock_iter *gi = seq->private;
2585
2586         rhashtable_walk_stop(&gi->hti);
2587 }
2588
2589 static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2590 {
2591         dump_glock(seq, iter_ptr, false);
2592         return 0;
2593 }
2594
2595 static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2596 {
2597         preempt_disable();
2598         if (*pos >= GFS2_NR_SBSTATS)
2599                 return NULL;
2600         return pos;
2601 }
2602
2603 static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2604                                    loff_t *pos)
2605 {
2606         (*pos)++;
2607         if (*pos >= GFS2_NR_SBSTATS)
2608                 return NULL;
2609         return pos;
2610 }
2611
2612 static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2613 {
2614         preempt_enable();
2615 }
2616
2617 static const struct seq_operations gfs2_glock_seq_ops = {
2618         .start = gfs2_glock_seq_start,
2619         .next  = gfs2_glock_seq_next,
2620         .stop  = gfs2_glock_seq_stop,
2621         .show  = gfs2_glock_seq_show,
2622 };
2623
2624 static const struct seq_operations gfs2_glstats_seq_ops = {
2625         .start = gfs2_glock_seq_start,
2626         .next  = gfs2_glock_seq_next,
2627         .stop  = gfs2_glock_seq_stop,
2628         .show  = gfs2_glstats_seq_show,
2629 };
2630
2631 static const struct seq_operations gfs2_sbstats_sops = {
2632         .start = gfs2_sbstats_seq_start,
2633         .next  = gfs2_sbstats_seq_next,
2634         .stop  = gfs2_sbstats_seq_stop,
2635         .show  = gfs2_sbstats_seq_show,
2636 };
2637
2638 #define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2639
2640 static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2641                               const struct seq_operations *ops)
2642 {
2643         int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2644         if (ret == 0) {
2645                 struct seq_file *seq = file->private_data;
2646                 struct gfs2_glock_iter *gi = seq->private;
2647
2648                 gi->sdp = inode->i_private;
2649                 seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2650                 if (seq->buf)
2651                         seq->size = GFS2_SEQ_GOODSIZE;
2652                 /*
2653                  * Initially, we are "before" the first hash table entry; the
2654                  * first call to rhashtable_walk_next gets us the first entry.
2655                  */
2656                 gi->last_pos = -1;
2657                 gi->gl = NULL;
2658                 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2659         }
2660         return ret;
2661 }
2662
2663 static int gfs2_glocks_open(struct inode *inode, struct file *file)
2664 {
2665         return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops);
2666 }
2667
2668 static int gfs2_glocks_release(struct inode *inode, struct file *file)
2669 {
2670         struct seq_file *seq = file->private_data;
2671         struct gfs2_glock_iter *gi = seq->private;
2672
2673         if (gi->gl)
2674                 gfs2_glock_put(gi->gl);
2675         rhashtable_walk_exit(&gi->hti);
2676         return seq_release_private(inode, file);
2677 }
2678
2679 static int gfs2_glstats_open(struct inode *inode, struct file *file)
2680 {
2681         return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops);
2682 }
2683
2684 static const struct file_operations gfs2_glocks_fops = {
2685         .owner   = THIS_MODULE,
2686         .open    = gfs2_glocks_open,
2687         .read    = seq_read,
2688         .llseek  = seq_lseek,
2689         .release = gfs2_glocks_release,
2690 };
2691
2692 static const struct file_operations gfs2_glstats_fops = {
2693         .owner   = THIS_MODULE,
2694         .open    = gfs2_glstats_open,
2695         .read    = seq_read,
2696         .llseek  = seq_lseek,
2697         .release = gfs2_glocks_release,
2698 };
2699
2700 DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats);
2701
2702 void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2703 {
2704         sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root);
2705
2706         debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2707                             &gfs2_glocks_fops);
2708
2709         debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2710                             &gfs2_glstats_fops);
2711
2712         debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2713                             &gfs2_sbstats_fops);
2714 }
2715
2716 void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2717 {
2718         debugfs_remove_recursive(sdp->debugfs_dir);
2719         sdp->debugfs_dir = NULL;
2720 }
2721
2722 void gfs2_register_debugfs(void)
2723 {
2724         gfs2_root = debugfs_create_dir("gfs2", NULL);
2725 }
2726
2727 void gfs2_unregister_debugfs(void)
2728 {
2729         debugfs_remove(gfs2_root);
2730         gfs2_root = NULL;
2731 }