1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4	* Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5	*/
6
7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9	#include <linux/sched.h>
10	#include <linux/slab.h>
11	#include <linux/spinlock.h>
12	#include <linux/buffer_head.h>
13	#include <linux/delay.h>
14	#include <linux/sort.h>
15	#include <linux/hash.h>
16	#include <linux/jhash.h>
17	#include <linux/kallsyms.h>
18	#include <linux/gfs2_ondisk.h>
19	#include <linux/list.h>
20	#include <linux/wait.h>
21	#include <linux/module.h>
22	#include <linux/uaccess.h>
23	#include <linux/seq_file.h>
24	#include <linux/debugfs.h>
25	#include <linux/kthread.h>
26	#include <linux/freezer.h>
27	#include <linux/workqueue.h>
28	#include <linux/jiffies.h>
29	#include <linux/rcupdate.h>
30	#include <linux/rculist_bl.h>
31	#include <linux/bit_spinlock.h>
32	#include <linux/percpu.h>
33	#include <linux/list_sort.h>
34	#include <linux/lockref.h>
35	#include <linux/rhashtable.h>
36	#include <linux/pid_namespace.h>
37	#include <linux/fdtable.h>
38	#include <linux/file.h>
39
40	#include "gfs2.h"
41	#include "incore.h"
42	#include "glock.h"
43	#include "glops.h"
44	#include "inode.h"
45	#include "lops.h"
46	#include "meta_io.h"
47	#include "quota.h"
48	#include "super.h"
49	#include "util.h"
50	#include "bmap.h"
51	#define CREATE_TRACE_POINTS
52	#include "trace_gfs2.h"
53
54	struct gfs2_glock_iter {
55	struct gfs2_sbd *sdp; /* incore superblock */
56	struct rhashtable_iter hti; /* rhashtable iterator */
57	struct gfs2_glock *gl; /* current glock struct */
58	loff_t last_pos; /* last position */
59	};
60
61	typedef void (*glock_examiner) (struct gfs2_glock * gl);
62
63	static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
64	static void __gfs2_glock_dq(struct gfs2_holder *gh);
65	static void handle_callback(struct gfs2_glock *gl, unsigned int state,
66	unsigned long delay, bool remote);
67
68	static struct dentry *gfs2_root;
69	static struct workqueue_struct *glock_workqueue;
70	static LIST_HEAD(lru_list);
71	static atomic_t lru_count = ATOMIC_INIT(0);
72	static DEFINE_SPINLOCK(lru_lock);
73
74	#define GFS2_GL_HASH_SHIFT 15
75	#define GFS2_GL_HASH_SIZE BIT(GFS2_GL_HASH_SHIFT)
76
77	static const struct rhashtable_params ht_parms = {
78	.nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
79	.key_len = offsetofend(struct lm_lockname, ln_type),
80	.key_offset = offsetof(struct gfs2_glock, gl_name),
81	.head_offset = offsetof(struct gfs2_glock, gl_node),
82	};
83
84	static struct rhashtable gl_hash_table;
85
86	#define GLOCK_WAIT_TABLE_BITS 12
87	#define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
88	static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
89
90	struct wait_glock_queue {
91	struct lm_lockname *name;
92	wait_queue_entry_t wait;
93	};
94
95	static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
96	int sync, void *key)
97	{
98	struct wait_glock_queue *wait_glock =
99	container_of(wait, struct wait_glock_queue, wait);
100	struct lm_lockname *wait_name = wait_glock->name;
101	struct lm_lockname *wake_name = key;
102
103	if (wake_name->ln_sbd != wait_name->ln_sbd ||
104	wake_name->ln_number != wait_name->ln_number ||
105	wake_name->ln_type != wait_name->ln_type)
106	return 0;
107	return autoremove_wake_function(wq_entry: wait, mode, sync, key);
108	}
109
110	static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
111	{
112	u32 hash = jhash2(k: (u32 *)name, length: ht_parms.key_len / 4, initval: 0);
113
114	return glock_wait_table + hash_32(val: hash, GLOCK_WAIT_TABLE_BITS);
115	}
116
117	/**
118	* wake_up_glock - Wake up waiters on a glock
119	* @gl: the glock
120	*/
121	static void wake_up_glock(struct gfs2_glock *gl)
122	{
123	wait_queue_head_t *wq = glock_waitqueue(name: &gl->gl_name);
124
125	if (waitqueue_active(wq_head: wq))
126	__wake_up(wq_head: wq, TASK_NORMAL, nr: 1, key: &gl->gl_name);
127	}
128
129	static void gfs2_glock_dealloc(struct rcu_head *rcu)
130	{
131	struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
132
133	kfree(objp: gl->gl_lksb.sb_lvbptr);
134	if (gl->gl_ops->go_flags & GLOF_ASPACE) {
135	struct gfs2_glock_aspace *gla =
136	container_of(gl, struct gfs2_glock_aspace, glock);
137	kmem_cache_free(s: gfs2_glock_aspace_cachep, objp: gla);
138	} else
139	kmem_cache_free(s: gfs2_glock_cachep, objp: gl);
140	}
141
142	/**
143	* glock_blocked_by_withdraw - determine if we can still use a glock
144	* @gl: the glock
145	*
146	* We need to allow some glocks to be enqueued, dequeued, promoted, and demoted
147	* when we're withdrawn. For example, to maintain metadata integrity, we should
148	* disallow the use of inode and rgrp glocks when withdrawn. Other glocks like
149	* the iopen or freeze glock may be safely used because none of their
150	* metadata goes through the journal. So in general, we should disallow all
151	* glocks that are journaled, and allow all the others. One exception is:
152	* we need to allow our active journal to be promoted and demoted so others
153	* may recover it and we can reacquire it when they're done.
154	*/
155	static bool glock_blocked_by_withdraw(struct gfs2_glock *gl)
156	{
157	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
158
159	if (!gfs2_withdrawing_or_withdrawn(sdp))
160	return false;
161	if (gl->gl_ops->go_flags & GLOF_NONDISK)
162	return false;
163	if (!sdp->sd_jdesc ||
164	gl->gl_name.ln_number == sdp->sd_jdesc->jd_no_addr)
165	return false;
166	return true;
167	}
168
169	void gfs2_glock_free(struct gfs2_glock *gl)
170	{
171	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
172
173	gfs2_glock_assert_withdraw(gl, atomic_read(&gl->gl_revokes) == 0);
174	rhashtable_remove_fast(ht: &gl_hash_table, obj: &gl->gl_node, params: ht_parms);
175	smp_mb();
176	wake_up_glock(gl);
177	call_rcu(head: &gl->gl_rcu, func: gfs2_glock_dealloc);
178	if (atomic_dec_and_test(v: &sdp->sd_glock_disposal))
179	wake_up(&sdp->sd_kill_wait);
180	}
181
182	/**
183	* gfs2_glock_hold() - increment reference count on glock
184	* @gl: The glock to hold
185	*
186	*/
187
188	struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl)
189	{
190	GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
191	lockref_get(&gl->gl_lockref);
192	return gl;
193	}
194
195	/**
196	* demote_ok - Check to see if it's ok to unlock a glock
197	* @gl: the glock
198	*
199	* Returns: 1 if it's ok
200	*/
201
202	static int demote_ok(const struct gfs2_glock *gl)
203	{
204	const struct gfs2_glock_operations *glops = gl->gl_ops;
205
206	if (gl->gl_state == LM_ST_UNLOCKED)
207	return 0;
208	if (!list_empty(head: &gl->gl_holders))
209	return 0;
210	if (glops->go_demote_ok)
211	return glops->go_demote_ok(gl);
212	return 1;
213	}
214
215
216	void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
217	{
218	if (!(gl->gl_ops->go_flags & GLOF_LRU))
219	return;
220
221	spin_lock(lock: &lru_lock);
222
223	list_move_tail(list: &gl->gl_lru, head: &lru_list);
224
225	if (!test_bit(GLF_LRU, &gl->gl_flags)) {
226	set_bit(nr: GLF_LRU, addr: &gl->gl_flags);
227	atomic_inc(v: &lru_count);
228	}
229
230	spin_unlock(lock: &lru_lock);
231	}
232
233	static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
234	{
235	if (!(gl->gl_ops->go_flags & GLOF_LRU))
236	return;
237
238	spin_lock(lock: &lru_lock);
239	if (test_bit(GLF_LRU, &gl->gl_flags)) {
240	list_del_init(entry: &gl->gl_lru);
241	atomic_dec(v: &lru_count);
242	clear_bit(nr: GLF_LRU, addr: &gl->gl_flags);
243	}
244	spin_unlock(lock: &lru_lock);
245	}
246
247	/*
248	* Enqueue the glock on the work queue. Passes one glock reference on to the
249	* work queue.
250	*/
251	static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
252	if (!queue_delayed_work(wq: glock_workqueue, dwork: &gl->gl_work, delay)) {
253	/*
254	* We are holding the lockref spinlock, and the work was still
255	* queued above. The queued work (glock_work_func) takes that
256	* spinlock before dropping its glock reference(s), so it
257	* cannot have dropped them in the meantime.
258	*/
259	GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
260	gl->gl_lockref.count--;
261	}
262	}
263
264	static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
265	spin_lock(lock: &gl->gl_lockref.lock);
266	__gfs2_glock_queue_work(gl, delay);
267	spin_unlock(lock: &gl->gl_lockref.lock);
268	}
269
270	static void __gfs2_glock_put(struct gfs2_glock *gl)
271	{
272	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
273	struct address_space *mapping = gfs2_glock2aspace(gl);
274
275	lockref_mark_dead(&gl->gl_lockref);
276	spin_unlock(lock: &gl->gl_lockref.lock);
277	gfs2_glock_remove_from_lru(gl);
278	GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
279	if (mapping) {
280	truncate_inode_pages_final(mapping);
281	if (!gfs2_withdrawing_or_withdrawn(sdp))
282	GLOCK_BUG_ON(gl, !mapping_empty(mapping));
283	}
284	trace_gfs2_glock_put(gl);
285	sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
286	}
287
288	/*
289	* Cause the glock to be put in work queue context.
290	*/
291	void gfs2_glock_queue_put(struct gfs2_glock *gl)
292	{
293	gfs2_glock_queue_work(gl, delay: 0);
294	}
295
296	/**
297	* gfs2_glock_put() - Decrement reference count on glock
298	* @gl: The glock to put
299	*
300	*/
301
302	void gfs2_glock_put(struct gfs2_glock *gl)
303	{
304	if (lockref_put_or_lock(&gl->gl_lockref))
305	return;
306
307	__gfs2_glock_put(gl);
308	}
309
310	/**
311	* may_grant - check if it's ok to grant a new lock
312	* @gl: The glock
313	* @current_gh: One of the current holders of @gl
314	* @gh: The lock request which we wish to grant
315	*
316	* With our current compatibility rules, if a glock has one or more active
317	* holders (HIF_HOLDER flag set), any of those holders can be passed in as
318	* @current_gh; they are all the same as far as compatibility with the new @gh
319	* goes.
320	*
321	* Returns true if it's ok to grant the lock.
322	*/
323
324	static inline bool may_grant(struct gfs2_glock *gl,
325	struct gfs2_holder *current_gh,
326	struct gfs2_holder *gh)
327	{
328	if (current_gh) {
329	GLOCK_BUG_ON(gl, !test_bit(HIF_HOLDER, &current_gh->gh_iflags));
330
331	switch(current_gh->gh_state) {
332	case LM_ST_EXCLUSIVE:
333	/*
334	* Here we make a special exception to grant holders
335	* who agree to share the EX lock with other holders
336	* who also have the bit set. If the original holder
337	* has the LM_FLAG_NODE_SCOPE bit set, we grant more
338	* holders with the bit set.
339	*/
340	return gh->gh_state == LM_ST_EXCLUSIVE &&
341	(current_gh->gh_flags & LM_FLAG_NODE_SCOPE) &&
342	(gh->gh_flags & LM_FLAG_NODE_SCOPE);
343
344	case LM_ST_SHARED:
345	case LM_ST_DEFERRED:
346	return gh->gh_state == current_gh->gh_state;
347
348	default:
349	return false;
350	}
351	}
352
353	if (gl->gl_state == gh->gh_state)
354	return true;
355	if (gh->gh_flags & GL_EXACT)
356	return false;
357	if (gl->gl_state == LM_ST_EXCLUSIVE) {
358	return gh->gh_state == LM_ST_SHARED ||
359	gh->gh_state == LM_ST_DEFERRED;
360	}
361	if (gh->gh_flags & LM_FLAG_ANY)
362	return gl->gl_state != LM_ST_UNLOCKED;
363	return false;
364	}
365
366	static void gfs2_holder_wake(struct gfs2_holder *gh)
367	{
368	clear_bit(nr: HIF_WAIT, addr: &gh->gh_iflags);
369	smp_mb__after_atomic();
370	wake_up_bit(word: &gh->gh_iflags, bit: HIF_WAIT);
371	if (gh->gh_flags & GL_ASYNC) {
372	struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
373
374	wake_up(&sdp->sd_async_glock_wait);
375	}
376	}
377
378	/**
379	* do_error - Something unexpected has happened during a lock request
380	* @gl: The glock
381	* @ret: The status from the DLM
382	*/
383
384	static void do_error(struct gfs2_glock *gl, const int ret)
385	{
386	struct gfs2_holder *gh, *tmp;
387
388	list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
389	if (test_bit(HIF_HOLDER, &gh->gh_iflags))
390	continue;
391	if (ret & LM_OUT_ERROR)
392	gh->gh_error = -EIO;
393	else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
394	gh->gh_error = GLR_TRYFAILED;
395	else
396	continue;
397	list_del_init(entry: &gh->gh_list);
398	trace_gfs2_glock_queue(gh, queue: 0);
399	gfs2_holder_wake(gh);
400	}
401	}
402
403	/**
404	* find_first_holder - find the first "holder" gh
405	* @gl: the glock
406	*/
407
408	static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
409	{
410	struct gfs2_holder *gh;
411
412	if (!list_empty(head: &gl->gl_holders)) {
413	gh = list_first_entry(&gl->gl_holders, struct gfs2_holder,
414	gh_list);
415	if (test_bit(HIF_HOLDER, &gh->gh_iflags))
416	return gh;
417	}
418	return NULL;
419	}
420
421	/*
422	* gfs2_instantiate - Call the glops instantiate function
423	* @gh: The glock holder
424	*
425	* Returns: 0 if instantiate was successful, or error.
426	*/
427	int gfs2_instantiate(struct gfs2_holder *gh)
428	{
429	struct gfs2_glock *gl = gh->gh_gl;
430	const struct gfs2_glock_operations *glops = gl->gl_ops;
431	int ret;
432
433	again:
434	if (!test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags))
435	goto done;
436
437	/*
438	* Since we unlock the lockref lock, we set a flag to indicate
439	* instantiate is in progress.
440	*/
441	if (test_and_set_bit(nr: GLF_INSTANTIATE_IN_PROG, addr: &gl->gl_flags)) {
442	wait_on_bit(word: &gl->gl_flags, bit: GLF_INSTANTIATE_IN_PROG,
443	TASK_UNINTERRUPTIBLE);
444	/*
445	* Here we just waited for a different instantiate to finish.
446	* But that may not have been successful, as when a process
447	* locks an inode glock _before_ it has an actual inode to
448	* instantiate into. So we check again. This process might
449	* have an inode to instantiate, so might be successful.
450	*/
451	goto again;
452	}
453
454	ret = glops->go_instantiate(gl);
455	if (!ret)
456	clear_bit(nr: GLF_INSTANTIATE_NEEDED, addr: &gl->gl_flags);
457	clear_and_wake_up_bit(bit: GLF_INSTANTIATE_IN_PROG, word: &gl->gl_flags);
458	if (ret)
459	return ret;
460
461	done:
462	if (glops->go_held)
463	return glops->go_held(gh);
464	return 0;
465	}
466
467	/**
468	* do_promote - promote as many requests as possible on the current queue
469	* @gl: The glock
470	*
471	* Returns true on success (i.e., progress was made or there are no waiters).
472	*/
473
474	static bool do_promote(struct gfs2_glock *gl)
475	{
476	struct gfs2_holder *gh, *current_gh;
477
478	current_gh = find_first_holder(gl);
479	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
480	if (test_bit(HIF_HOLDER, &gh->gh_iflags))
481	continue;
482	if (!may_grant(gl, current_gh, gh)) {
483	/*
484	* If we get here, it means we may not grant this
485	* holder for some reason. If this holder is at the
486	* head of the list, it means we have a blocked holder
487	* at the head, so return false.
488	*/
489	if (list_is_first(list: &gh->gh_list, head: &gl->gl_holders))
490	return false;
491	do_error(gl, ret: 0);
492	break;
493	}
494	set_bit(nr: HIF_HOLDER, addr: &gh->gh_iflags);
495	trace_gfs2_promote(gh);
496	gfs2_holder_wake(gh);
497	if (!current_gh)
498	current_gh = gh;
499	}
500	return true;
501	}
502
503	/**
504	* find_first_waiter - find the first gh that's waiting for the glock
505	* @gl: the glock
506	*/
507
508	static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
509	{
510	struct gfs2_holder *gh;
511
512	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
513	if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
514	return gh;
515	}
516	return NULL;
517	}
518
519	/**
520	* find_last_waiter - find the last gh that's waiting for the glock
521	* @gl: the glock
522	*
523	* This also is a fast way of finding out if there are any waiters.
524	*/
525
526	static inline struct gfs2_holder *find_last_waiter(const struct gfs2_glock *gl)
527	{
528	struct gfs2_holder *gh;
529
530	if (list_empty(head: &gl->gl_holders))
531	return NULL;
532	gh = list_last_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
533	return test_bit(HIF_HOLDER, &gh->gh_iflags) ? NULL : gh;
534	}
535
536	/**
537	* state_change - record that the glock is now in a different state
538	* @gl: the glock
539	* @new_state: the new state
540	*/
541
542	static void state_change(struct gfs2_glock *gl, unsigned int new_state)
543	{
544	int held1, held2;
545
546	held1 = (gl->gl_state != LM_ST_UNLOCKED);
547	held2 = (new_state != LM_ST_UNLOCKED);
548
549	if (held1 != held2) {
550	GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
551	if (held2)
552	gl->gl_lockref.count++;
553	else
554	gl->gl_lockref.count--;
555	}
556	if (new_state != gl->gl_target)
557	/* shorten our minimum hold time */
558	gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
559	GL_GLOCK_MIN_HOLD);
560	gl->gl_state = new_state;
561	gl->gl_tchange = jiffies;
562	}
563
564	static void gfs2_set_demote(struct gfs2_glock *gl)
565	{
566	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
567
568	set_bit(nr: GLF_DEMOTE, addr: &gl->gl_flags);
569	smp_mb();
570	wake_up(&sdp->sd_async_glock_wait);
571	}
572
573	static void gfs2_demote_wake(struct gfs2_glock *gl)
574	{
575	gl->gl_demote_state = LM_ST_EXCLUSIVE;
576	clear_bit(nr: GLF_DEMOTE, addr: &gl->gl_flags);
577	smp_mb__after_atomic();
578	wake_up_bit(word: &gl->gl_flags, bit: GLF_DEMOTE);
579	}
580
581	/**
582	* finish_xmote - The DLM has replied to one of our lock requests
583	* @gl: The glock
584	* @ret: The status from the DLM
585	*
586	*/
587
588	static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
589	{
590	const struct gfs2_glock_operations *glops = gl->gl_ops;
591	struct gfs2_holder *gh;
592	unsigned state = ret & LM_OUT_ST_MASK;
593
594	spin_lock(lock: &gl->gl_lockref.lock);
595	trace_gfs2_glock_state_change(gl, new_state: state);
596	state_change(gl, new_state: state);
597	gh = find_first_waiter(gl);
598
599	/* Demote to UN request arrived during demote to SH or DF */
600	if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
601	state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED)
602	gl->gl_target = LM_ST_UNLOCKED;
603
604	/* Check for state != intended state */
605	if (unlikely(state != gl->gl_target)) {
606	if (gh && (ret & LM_OUT_CANCELED))
607	gfs2_holder_wake(gh);
608	if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
609	/* move to back of queue and try next entry */
610	if (ret & LM_OUT_CANCELED) {
611	list_move_tail(list: &gh->gh_list, head: &gl->gl_holders);
612	gh = find_first_waiter(gl);
613	gl->gl_target = gh->gh_state;
614	if (do_promote(gl))
615	goto out;
616	goto retry;
617	}
618	/* Some error or failed "try lock" - report it */
619	if ((ret & LM_OUT_ERROR) ||
620	(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
621	gl->gl_target = gl->gl_state;
622	do_error(gl, ret);
623	goto out;
624	}
625	}
626	switch(state) {
627	/* Unlocked due to conversion deadlock, try again */
628	case LM_ST_UNLOCKED:
629	retry:
630	do_xmote(gl, gh, target: gl->gl_target);
631	break;
632	/* Conversion fails, unlock and try again */
633	case LM_ST_SHARED:
634	case LM_ST_DEFERRED:
635	do_xmote(gl, gh, LM_ST_UNLOCKED);
636	break;
637	default: /* Everything else */
638	fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n",
639	gl->gl_target, state);
640	GLOCK_BUG_ON(gl, 1);
641	}
642	spin_unlock(lock: &gl->gl_lockref.lock);
643	return;
644	}
645
646	/* Fast path - we got what we asked for */
647	if (test_and_clear_bit(nr: GLF_DEMOTE_IN_PROGRESS, addr: &gl->gl_flags))
648	gfs2_demote_wake(gl);
649	if (state != LM_ST_UNLOCKED) {
650	if (glops->go_xmote_bh) {
651	int rv;
652
653	spin_unlock(lock: &gl->gl_lockref.lock);
654	rv = glops->go_xmote_bh(gl);
655	spin_lock(lock: &gl->gl_lockref.lock);
656	if (rv) {
657	do_error(gl, ret: rv);
658	goto out;
659	}
660	}
661	do_promote(gl);
662	}
663	out:
664	clear_bit(nr: GLF_LOCK, addr: &gl->gl_flags);
665	spin_unlock(lock: &gl->gl_lockref.lock);
666	}
667
668	static bool is_system_glock(struct gfs2_glock *gl)
669	{
670	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
671	struct gfs2_inode *m_ip = GFS2_I(inode: sdp->sd_statfs_inode);
672
673	if (gl == m_ip->i_gl)
674	return true;
675	return false;
676	}
677
678	/**
679	* do_xmote - Calls the DLM to change the state of a lock
680	* @gl: The lock state
681	* @gh: The holder (only for promotes)
682	* @target: The target lock state
683	*
684	*/
685
686	static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh,
687	unsigned int target)
688	__releases(&gl->gl_lockref.lock)
689	__acquires(&gl->gl_lockref.lock)
690	{
691	const struct gfs2_glock_operations *glops = gl->gl_ops;
692	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
693	unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
694	int ret;
695
696	if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) &&
697	gh && !(gh->gh_flags & LM_FLAG_NOEXP))
698	goto skip_inval;
699
700	lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP);
701	GLOCK_BUG_ON(gl, gl->gl_state == target);
702	GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
703	if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
704	glops->go_inval) {
705	/*
706	* If another process is already doing the invalidate, let that
707	* finish first. The glock state machine will get back to this
708	* holder again later.
709	*/
710	if (test_and_set_bit(nr: GLF_INVALIDATE_IN_PROGRESS,
711	addr: &gl->gl_flags))
712	return;
713	do_error(gl, ret: 0); /* Fail queued try locks */
714	}
715	gl->gl_req = target;
716	set_bit(nr: GLF_BLOCKING, addr: &gl->gl_flags);
717	if ((gl->gl_req == LM_ST_UNLOCKED) ||
718	(gl->gl_state == LM_ST_EXCLUSIVE) ||
719	(lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
720	clear_bit(nr: GLF_BLOCKING, addr: &gl->gl_flags);
721	spin_unlock(lock: &gl->gl_lockref.lock);
722	if (glops->go_sync) {
723	ret = glops->go_sync(gl);
724	/* If we had a problem syncing (due to io errors or whatever,
725	* we should not invalidate the metadata or tell dlm to
726	* release the glock to other nodes.
727	*/
728	if (ret) {
729	if (cmpxchg(&sdp->sd_log_error, 0, ret)) {
730	fs_err(sdp, "Error %d syncing glock \n", ret);
731	gfs2_dump_glock(NULL, gl, fsid: true);
732	}
733	goto skip_inval;
734	}
735	}
736	if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) {
737	/*
738	* The call to go_sync should have cleared out the ail list.
739	* If there are still items, we have a problem. We ought to
740	* withdraw, but we can't because the withdraw code also uses
741	* glocks. Warn about the error, dump the glock, then fall
742	* through and wait for logd to do the withdraw for us.
743	*/
744	if ((atomic_read(v: &gl->gl_ail_count) != 0) &&
745	(!cmpxchg(&sdp->sd_log_error, 0, -EIO))) {
746	gfs2_glock_assert_warn(gl,
747	!atomic_read(&gl->gl_ail_count));
748	gfs2_dump_glock(NULL, gl, fsid: true);
749	}
750	glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
751	clear_bit(nr: GLF_INVALIDATE_IN_PROGRESS, addr: &gl->gl_flags);
752	}
753
754	skip_inval:
755	gfs2_glock_hold(gl);
756	/*
757	* Check for an error encountered since we called go_sync and go_inval.
758	* If so, we can't withdraw from the glock code because the withdraw
759	* code itself uses glocks (see function signal_our_withdraw) to
760	* change the mount to read-only. Most importantly, we must not call
761	* dlm to unlock the glock until the journal is in a known good state
762	* (after journal replay) otherwise other nodes may use the object
763	* (rgrp or dinode) and then later, journal replay will corrupt the
764	* file system. The best we can do here is wait for the logd daemon
765	* to see sd_log_error and withdraw, and in the meantime, requeue the
766	* work for later.
767	*
768	* We make a special exception for some system glocks, such as the
769	* system statfs inode glock, which needs to be granted before the
770	* gfs2_quotad daemon can exit, and that exit needs to finish before
771	* we can unmount the withdrawn file system.
772	*
773	* However, if we're just unlocking the lock (say, for unmount, when
774	* gfs2_gl_hash_clear calls clear_glock) and recovery is complete
775	* then it's okay to tell dlm to unlock it.
776	*/
777	if (unlikely(sdp->sd_log_error) && !gfs2_withdrawing_or_withdrawn(sdp))
778	gfs2_withdraw_delayed(sdp);
779	if (glock_blocked_by_withdraw(gl) &&
780	(target != LM_ST_UNLOCKED ||
781	test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags))) {
782	if (!is_system_glock(gl)) {
783	handle_callback(gl, LM_ST_UNLOCKED, delay: 0, remote: false); /* sets demote */
784	/*
785	* Ordinarily, we would call dlm and its callback would call
786	* finish_xmote, which would call state_change() to the new state.
787	* Since we withdrew, we won't call dlm, so call state_change
788	* manually, but to the UNLOCKED state we desire.
789	*/
790	state_change(gl, LM_ST_UNLOCKED);
791	/*
792	* We skip telling dlm to do the locking, so we won't get a
793	* reply that would otherwise clear GLF_LOCK. So we clear it here.
794	*/
795	clear_bit(nr: GLF_LOCK, addr: &gl->gl_flags);
796	clear_bit(nr: GLF_DEMOTE_IN_PROGRESS, addr: &gl->gl_flags);
797	gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD);
798	goto out;
799	} else {
800	clear_bit(nr: GLF_INVALIDATE_IN_PROGRESS, addr: &gl->gl_flags);
801	}
802	}
803
804	if (sdp->sd_lockstruct.ls_ops->lm_lock) {
805	/* lock_dlm */
806	ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags);
807	if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
808	target == LM_ST_UNLOCKED &&
809	test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) {
810	finish_xmote(gl, ret: target);
811	gfs2_glock_queue_work(gl, delay: 0);
812	} else if (ret) {
813	fs_err(sdp, "lm_lock ret %d\n", ret);
814	GLOCK_BUG_ON(gl, !gfs2_withdrawing_or_withdrawn(sdp));
815	}
816	} else { /* lock_nolock */
817	finish_xmote(gl, ret: target);
818	gfs2_glock_queue_work(gl, delay: 0);
819	}
820	out:
821	spin_lock(lock: &gl->gl_lockref.lock);
822	}
823
824	/**
825	* run_queue - do all outstanding tasks related to a glock
826	* @gl: The glock in question
827	* @nonblock: True if we must not block in run_queue
828	*
829	*/
830
831	static void run_queue(struct gfs2_glock *gl, const int nonblock)
832	__releases(&gl->gl_lockref.lock)
833	__acquires(&gl->gl_lockref.lock)
834	{
835	struct gfs2_holder *gh = NULL;
836
837	if (test_and_set_bit(nr: GLF_LOCK, addr: &gl->gl_flags))
838	return;
839
840	GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
841
842	if (test_bit(GLF_DEMOTE, &gl->gl_flags) &&
843	gl->gl_demote_state != gl->gl_state) {
844	if (find_first_holder(gl))
845	goto out_unlock;
846	if (nonblock)
847	goto out_sched;
848	set_bit(nr: GLF_DEMOTE_IN_PROGRESS, addr: &gl->gl_flags);
849	GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
850	gl->gl_target = gl->gl_demote_state;
851	} else {
852	if (test_bit(GLF_DEMOTE, &gl->gl_flags))
853	gfs2_demote_wake(gl);
854	if (do_promote(gl))
855	goto out_unlock;
856	gh = find_first_waiter(gl);
857	gl->gl_target = gh->gh_state;
858	if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
859	do_error(gl, ret: 0); /* Fail queued try locks */
860	}
861	do_xmote(gl, gh, target: gl->gl_target);
862	return;
863
864	out_sched:
865	clear_bit(nr: GLF_LOCK, addr: &gl->gl_flags);
866	smp_mb__after_atomic();
867	gl->gl_lockref.count++;
868	__gfs2_glock_queue_work(gl, delay: 0);
869	return;
870
871	out_unlock:
872	clear_bit(nr: GLF_LOCK, addr: &gl->gl_flags);
873	smp_mb__after_atomic();
874	return;
875	}
876
877	/**
878	* glock_set_object - set the gl_object field of a glock
879	* @gl: the glock
880	* @object: the object
881	*/
882	void glock_set_object(struct gfs2_glock *gl, void *object)
883	{
884	void *prev_object;
885
886	spin_lock(lock: &gl->gl_lockref.lock);
887	prev_object = gl->gl_object;
888	gl->gl_object = object;
889	spin_unlock(lock: &gl->gl_lockref.lock);
890	if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == NULL)) {
891	pr_warn("glock=%u/%llx\n",
892	gl->gl_name.ln_type,
893	(unsigned long long)gl->gl_name.ln_number);
894	gfs2_dump_glock(NULL, gl, fsid: true);
895	}
896	}
897
898	/**
899	* glock_clear_object - clear the gl_object field of a glock
900	* @gl: the glock
901	* @object: object the glock currently points at
902	*/
903	void glock_clear_object(struct gfs2_glock *gl, void *object)
904	{
905	void *prev_object;
906
907	spin_lock(lock: &gl->gl_lockref.lock);
908	prev_object = gl->gl_object;
909	gl->gl_object = NULL;
910	spin_unlock(lock: &gl->gl_lockref.lock);
911	if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == object)) {
912	pr_warn("glock=%u/%llx\n",
913	gl->gl_name.ln_type,
914	(unsigned long long)gl->gl_name.ln_number);
915	gfs2_dump_glock(NULL, gl, fsid: true);
916	}
917	}
918
919	void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation)
920	{
921	struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
922
923	if (ri->ri_magic == 0)
924	ri->ri_magic = cpu_to_be32(GFS2_MAGIC);
925	if (ri->ri_magic == cpu_to_be32(GFS2_MAGIC))
926	ri->ri_generation_deleted = cpu_to_be64(generation);
927	}
928
929	bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation)
930	{
931	struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
932
933	if (ri->ri_magic != cpu_to_be32(GFS2_MAGIC))
934	return false;
935	return generation <= be64_to_cpu(ri->ri_generation_deleted);
936	}
937
938	static void gfs2_glock_poke(struct gfs2_glock *gl)
939	{
940	int flags = LM_FLAG_TRY_1CB | LM_FLAG_ANY | GL_SKIP;
941	struct gfs2_holder gh;
942	int error;
943
944	__gfs2_holder_init(gl, LM_ST_SHARED, flags, gh: &gh, _RET_IP_);
945	error = gfs2_glock_nq(gh: &gh);
946	if (!error)
947	gfs2_glock_dq(gh: &gh);
948	gfs2_holder_uninit(gh: &gh);
949	}
950
951	static bool gfs2_try_evict(struct gfs2_glock *gl)
952	{
953	struct gfs2_inode *ip;
954	bool evicted = false;
955
956	/*
957	* If there is contention on the iopen glock and we have an inode, try
958	* to grab and release the inode so that it can be evicted. This will
959	* allow the remote node to go ahead and delete the inode without us
960	* having to do it, which will avoid rgrp glock thrashing.
961	*
962	* The remote node is likely still holding the corresponding inode
963	* glock, so it will run before we get to verify that the delete has
964	* happened below.
965	*/
966	spin_lock(lock: &gl->gl_lockref.lock);
967	ip = gl->gl_object;
968	if (ip && !igrab(&ip->i_inode))
969	ip = NULL;
970	spin_unlock(lock: &gl->gl_lockref.lock);
971	if (ip) {
972	gl->gl_no_formal_ino = ip->i_no_formal_ino;
973	set_bit(nr: GIF_DEFERRED_DELETE, addr: &ip->i_flags);
974	d_prune_aliases(&ip->i_inode);
975	iput(&ip->i_inode);
976
977	/* If the inode was evicted, gl->gl_object will now be NULL. */
978	spin_lock(lock: &gl->gl_lockref.lock);
979	ip = gl->gl_object;
980	if (ip) {
981	clear_bit(nr: GIF_DEFERRED_DELETE, addr: &ip->i_flags);
982	if (!igrab(&ip->i_inode))
983	ip = NULL;
984	}
985	spin_unlock(lock: &gl->gl_lockref.lock);
986	if (ip) {
987	gfs2_glock_poke(gl: ip->i_gl);
988	iput(&ip->i_inode);
989	}
990	evicted = !ip;
991	}
992	return evicted;
993	}
994
995	bool gfs2_queue_try_to_evict(struct gfs2_glock *gl)
996	{
997	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
998
999	if (test_and_set_bit(nr: GLF_TRY_TO_EVICT, addr: &gl->gl_flags))
1000	return false;
1001	return queue_delayed_work(wq: sdp->sd_delete_wq,
1002	dwork: &gl->gl_delete, delay: 0);
1003	}
1004
1005	static bool gfs2_queue_verify_evict(struct gfs2_glock *gl)
1006	{
1007	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1008
1009	if (test_and_set_bit(nr: GLF_VERIFY_EVICT, addr: &gl->gl_flags))
1010	return false;
1011	return queue_delayed_work(wq: sdp->sd_delete_wq,
1012	dwork: &gl->gl_delete, delay: 5 * HZ);
1013	}
1014
1015	static void delete_work_func(struct work_struct *work)
1016	{
1017	struct delayed_work *dwork = to_delayed_work(work);
1018	struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete);
1019	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1020	struct inode *inode;
1021	u64 no_addr = gl->gl_name.ln_number;
1022
1023	if (test_and_clear_bit(nr: GLF_TRY_TO_EVICT, addr: &gl->gl_flags)) {
1024	/*
1025	* If we can evict the inode, give the remote node trying to
1026	* delete the inode some time before verifying that the delete
1027	* has happened. Otherwise, if we cause contention on the inode glock
1028	* immediately, the remote node will think that we still have
1029	* the inode in use, and so it will give up waiting.
1030	*
1031	* If we can't evict the inode, signal to the remote node that
1032	* the inode is still in use. We'll later try to delete the
1033	* inode locally in gfs2_evict_inode.
1034	*
1035	* FIXME: We only need to verify that the remote node has
1036	* deleted the inode because nodes before this remote delete
1037	* rework won't cooperate. At a later time, when we no longer
1038	* care about compatibility with such nodes, we can skip this
1039	* step entirely.
1040	*/
1041	if (gfs2_try_evict(gl)) {
1042	if (test_bit(SDF_KILL, &sdp->sd_flags))
1043	goto out;
1044	if (gfs2_queue_verify_evict(gl))
1045	return;
1046	}
1047	goto out;
1048	}
1049
1050	if (test_and_clear_bit(nr: GLF_VERIFY_EVICT, addr: &gl->gl_flags)) {
1051	inode = gfs2_lookup_by_inum(sdp, no_addr, no_formal_ino: gl->gl_no_formal_ino,
1052	GFS2_BLKST_UNLINKED);
1053	if (IS_ERR(ptr: inode)) {
1054	if (PTR_ERR(ptr: inode) == -EAGAIN &&
1055	!test_bit(SDF_KILL, &sdp->sd_flags) &&
1056	gfs2_queue_verify_evict(gl))
1057	return;
1058	} else {
1059	d_prune_aliases(inode);
1060	iput(inode);
1061	}
1062	}
1063
1064	out:
1065	gfs2_glock_put(gl);
1066	}
1067
1068	static void glock_work_func(struct work_struct *work)
1069	{
1070	unsigned long delay = 0;
1071	struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
1072	unsigned int drop_refs = 1;
1073
1074	if (test_and_clear_bit(nr: GLF_REPLY_PENDING, addr: &gl->gl_flags)) {
1075	finish_xmote(gl, ret: gl->gl_reply);
1076	drop_refs++;
1077	}
1078	spin_lock(lock: &gl->gl_lockref.lock);
1079	if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1080	gl->gl_state != LM_ST_UNLOCKED &&
1081	gl->gl_demote_state != LM_ST_EXCLUSIVE) {
1082	unsigned long holdtime, now = jiffies;
1083
1084	holdtime = gl->gl_tchange + gl->gl_hold_time;
1085	if (time_before(now, holdtime))
1086	delay = holdtime - now;
1087
1088	if (!delay) {
1089	clear_bit(nr: GLF_PENDING_DEMOTE, addr: &gl->gl_flags);
1090	gfs2_set_demote(gl);
1091	}
1092	}
1093	run_queue(gl, nonblock: 0);
1094	if (delay) {
1095	/* Keep one glock reference for the work we requeue. */
1096	drop_refs--;
1097	if (gl->gl_name.ln_type != LM_TYPE_INODE)
1098	delay = 0;
1099	__gfs2_glock_queue_work(gl, delay);
1100	}
1101
1102	/*
1103	* Drop the remaining glock references manually here. (Mind that
1104	* __gfs2_glock_queue_work depends on the lockref spinlock begin held
1105	* here as well.)
1106	*/
1107	gl->gl_lockref.count -= drop_refs;
1108	if (!gl->gl_lockref.count) {
1109	__gfs2_glock_put(gl);
1110	return;
1111	}
1112	spin_unlock(lock: &gl->gl_lockref.lock);
1113	}
1114
1115	static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
1116	struct gfs2_glock *new)
1117	{
1118	struct wait_glock_queue wait;
1119	wait_queue_head_t *wq = glock_waitqueue(name);
1120	struct gfs2_glock *gl;
1121
1122	wait.name = name;
1123	init_wait(&wait.wait);
1124	wait.wait.func = glock_wake_function;
1125
1126	again:
1127	prepare_to_wait(wq_head: wq, wq_entry: &wait.wait, TASK_UNINTERRUPTIBLE);
1128	rcu_read_lock();
1129	if (new) {
1130	gl = rhashtable_lookup_get_insert_fast(ht: &gl_hash_table,
1131	obj: &new->gl_node, params: ht_parms);
1132	if (IS_ERR(ptr: gl))
1133	goto out;
1134	} else {
1135	gl = rhashtable_lookup_fast(ht: &gl_hash_table,
1136	key: name, params: ht_parms);
1137	}
1138	if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
1139	rcu_read_unlock();
1140	schedule();
1141	goto again;
1142	}
1143	out:
1144	rcu_read_unlock();
1145	finish_wait(wq_head: wq, wq_entry: &wait.wait);
1146	return gl;
1147	}
1148
1149	/**
1150	* gfs2_glock_get() - Get a glock, or create one if one doesn't exist
1151	* @sdp: The GFS2 superblock
1152	* @number: the lock number
1153	* @glops: The glock_operations to use
1154	* @create: If 0, don't create the glock if it doesn't exist
1155	* @glp: the glock is returned here
1156	*
1157	* This does not lock a glock, just finds/creates structures for one.
1158	*
1159	* Returns: errno
1160	*/
1161
1162	int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
1163	const struct gfs2_glock_operations *glops, int create,
1164	struct gfs2_glock **glp)
1165	{
1166	struct super_block *s = sdp->sd_vfs;
1167	struct lm_lockname name = { .ln_number = number,
1168	.ln_type = glops->go_type,
1169	.ln_sbd = sdp };
1170	struct gfs2_glock *gl, *tmp;
1171	struct address_space *mapping;
1172	int ret = 0;
1173
1174	gl = find_insert_glock(name: &name, NULL);
1175	if (gl) {
1176	*glp = gl;
1177	return 0;
1178	}
1179	if (!create)
1180	return -ENOENT;
1181
1182	if (glops->go_flags & GLOF_ASPACE) {
1183	struct gfs2_glock_aspace *gla =
1184	kmem_cache_alloc(cachep: gfs2_glock_aspace_cachep, GFP_NOFS);
1185	if (!gla)
1186	return -ENOMEM;
1187	gl = &gla->glock;
1188	} else {
1189	gl = kmem_cache_alloc(cachep: gfs2_glock_cachep, GFP_NOFS);
1190	if (!gl)
1191	return -ENOMEM;
1192	}
1193	memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
1194	gl->gl_ops = glops;
1195
1196	if (glops->go_flags & GLOF_LVB) {
1197	gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS);
1198	if (!gl->gl_lksb.sb_lvbptr) {
1199	gfs2_glock_dealloc(rcu: &gl->gl_rcu);
1200	return -ENOMEM;
1201	}
1202	}
1203
1204	atomic_inc(v: &sdp->sd_glock_disposal);
1205	gl->gl_node.next = NULL;
1206	gl->gl_flags = glops->go_instantiate ? BIT(GLF_INSTANTIATE_NEEDED) : 0;
1207	gl->gl_name = name;
1208	lockdep_set_subclass(&gl->gl_lockref.lock, glops->go_subclass);
1209	gl->gl_lockref.count = 1;
1210	gl->gl_state = LM_ST_UNLOCKED;
1211	gl->gl_target = LM_ST_UNLOCKED;
1212	gl->gl_demote_state = LM_ST_EXCLUSIVE;
1213	gl->gl_dstamp = 0;
1214	preempt_disable();
1215	/* We use the global stats to estimate the initial per-glock stats */
1216	gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
1217	preempt_enable();
1218	gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
1219	gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
1220	gl->gl_tchange = jiffies;
1221	gl->gl_object = NULL;
1222	gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
1223	INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
1224	if (gl->gl_name.ln_type == LM_TYPE_IOPEN)
1225	INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func);
1226
1227	mapping = gfs2_glock2aspace(gl);
1228	if (mapping) {
1229	mapping->a_ops = &gfs2_meta_aops;
1230	mapping->host = s->s_bdev->bd_inode;
1231	mapping->flags = 0;
1232	mapping_set_gfp_mask(m: mapping, GFP_NOFS);
1233	mapping->i_private_data = NULL;
1234	mapping->writeback_index = 0;
1235	}
1236
1237	tmp = find_insert_glock(name: &name, new: gl);
1238	if (!tmp) {
1239	*glp = gl;
1240	goto out;
1241	}
1242	if (IS_ERR(ptr: tmp)) {
1243	ret = PTR_ERR(ptr: tmp);
1244	goto out_free;
1245	}
1246	*glp = tmp;
1247
1248	out_free:
1249	gfs2_glock_dealloc(rcu: &gl->gl_rcu);
1250	if (atomic_dec_and_test(v: &sdp->sd_glock_disposal))
1251	wake_up(&sdp->sd_kill_wait);
1252
1253	out:
1254	return ret;
1255	}
1256
1257	/**
1258	* __gfs2_holder_init - initialize a struct gfs2_holder in the default way
1259	* @gl: the glock
1260	* @state: the state we're requesting
1261	* @flags: the modifier flags
1262	* @gh: the holder structure
1263	*
1264	*/
1265
1266	void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
1267	struct gfs2_holder *gh, unsigned long ip)
1268	{
1269	INIT_LIST_HEAD(list: &gh->gh_list);
1270	gh->gh_gl = gfs2_glock_hold(gl);
1271	gh->gh_ip = ip;
1272	gh->gh_owner_pid = get_pid(pid: task_pid(current));
1273	gh->gh_state = state;
1274	gh->gh_flags = flags;
1275	gh->gh_iflags = 0;
1276	}
1277
1278	/**
1279	* gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it
1280	* @state: the state we're requesting
1281	* @flags: the modifier flags
1282	* @gh: the holder structure
1283	*
1284	* Don't mess with the glock.
1285	*
1286	*/
1287
1288	void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
1289	{
1290	gh->gh_state = state;
1291	gh->gh_flags = flags;
1292	gh->gh_iflags = 0;
1293	gh->gh_ip = _RET_IP_;
1294	put_pid(pid: gh->gh_owner_pid);
1295	gh->gh_owner_pid = get_pid(pid: task_pid(current));
1296	}
1297
1298	/**
1299	* gfs2_holder_uninit - uninitialize a holder structure (drop glock reference)
1300	* @gh: the holder structure
1301	*
1302	*/
1303
1304	void gfs2_holder_uninit(struct gfs2_holder *gh)
1305	{
1306	put_pid(pid: gh->gh_owner_pid);
1307	gfs2_glock_put(gl: gh->gh_gl);
1308	gfs2_holder_mark_uninitialized(gh);
1309	gh->gh_ip = 0;
1310	}
1311
1312	static void gfs2_glock_update_hold_time(struct gfs2_glock *gl,
1313	unsigned long start_time)
1314	{
1315	/* Have we waited longer that a second? */
1316	if (time_after(jiffies, start_time + HZ)) {
1317	/* Lengthen the minimum hold time. */
1318	gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR,
1319	GL_GLOCK_MAX_HOLD);
1320	}
1321	}
1322
1323	/**
1324	* gfs2_glock_holder_ready - holder is ready and its error code can be collected
1325	* @gh: the glock holder
1326	*
1327	* Called when a glock holder no longer needs to be waited for because it is
1328	* now either held (HIF_HOLDER set; gh_error == 0), or acquiring the lock has
1329	* failed (gh_error != 0).
1330	*/
1331
1332	int gfs2_glock_holder_ready(struct gfs2_holder *gh)
1333	{
1334	if (gh->gh_error || (gh->gh_flags & GL_SKIP))
1335	return gh->gh_error;
1336	gh->gh_error = gfs2_instantiate(gh);
1337	if (gh->gh_error)
1338	gfs2_glock_dq(gh);
1339	return gh->gh_error;
1340	}
1341
1342	/**
1343	* gfs2_glock_wait - wait on a glock acquisition
1344	* @gh: the glock holder
1345	*
1346	* Returns: 0 on success
1347	*/
1348
1349	int gfs2_glock_wait(struct gfs2_holder *gh)
1350	{
1351	unsigned long start_time = jiffies;
1352
1353	might_sleep();
1354	wait_on_bit(word: &gh->gh_iflags, bit: HIF_WAIT, TASK_UNINTERRUPTIBLE);
1355	gfs2_glock_update_hold_time(gl: gh->gh_gl, start_time);
1356	return gfs2_glock_holder_ready(gh);
1357	}
1358
1359	static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
1360	{
1361	int i;
1362
1363	for (i = 0; i < num_gh; i++)
1364	if (test_bit(HIF_WAIT, &ghs[i].gh_iflags))
1365	return 1;
1366	return 0;
1367	}
1368
1369	/**
1370	* gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions
1371	* @num_gh: the number of holders in the array
1372	* @ghs: the glock holder array
1373	*
1374	* Returns: 0 on success, meaning all glocks have been granted and are held.
1375	* -ESTALE if the request timed out, meaning all glocks were released,
1376	* and the caller should retry the operation.
1377	*/
1378
1379	int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs)
1380	{
1381	struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
1382	int i, ret = 0, timeout = 0;
1383	unsigned long start_time = jiffies;
1384
1385	might_sleep();
1386	/*
1387	* Total up the (minimum hold time * 2) of all glocks and use that to
1388	* determine the max amount of time we should wait.
1389	*/
1390	for (i = 0; i < num_gh; i++)
1391	timeout += ghs[i].gh_gl->gl_hold_time << 1;
1392
1393	if (!wait_event_timeout(sdp->sd_async_glock_wait,
1394	!glocks_pending(num_gh, ghs), timeout)) {
1395	ret = -ESTALE; /* request timed out. */
1396	goto out;
1397	}
1398
1399	for (i = 0; i < num_gh; i++) {
1400	struct gfs2_holder *gh = &ghs[i];
1401	int ret2;
1402
1403	if (test_bit(HIF_HOLDER, &gh->gh_iflags)) {
1404	gfs2_glock_update_hold_time(gl: gh->gh_gl,
1405	start_time);
1406	}
1407	ret2 = gfs2_glock_holder_ready(gh);
1408	if (!ret)
1409	ret = ret2;
1410	}
1411
1412	out:
1413	if (ret) {
1414	for (i = 0; i < num_gh; i++) {
1415	struct gfs2_holder *gh = &ghs[i];
1416
1417	gfs2_glock_dq(gh);
1418	}
1419	}
1420	return ret;
1421	}
1422
1423	/**
1424	* handle_callback - process a demote request
1425	* @gl: the glock
1426	* @state: the state the caller wants us to change to
1427	* @delay: zero to demote immediately; otherwise pending demote
1428	* @remote: true if this came from a different cluster node
1429	*
1430	* There are only two requests that we are going to see in actual
1431	* practise: LM_ST_SHARED and LM_ST_UNLOCKED
1432	*/
1433
1434	static void handle_callback(struct gfs2_glock *gl, unsigned int state,
1435	unsigned long delay, bool remote)
1436	{
1437	if (delay)
1438	set_bit(nr: GLF_PENDING_DEMOTE, addr: &gl->gl_flags);
1439	else
1440	gfs2_set_demote(gl);
1441	if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
1442	gl->gl_demote_state = state;
1443	gl->gl_demote_time = jiffies;
1444	} else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
1445	gl->gl_demote_state != state) {
1446	gl->gl_demote_state = LM_ST_UNLOCKED;
1447	}
1448	if (gl->gl_ops->go_callback)
1449	gl->gl_ops->go_callback(gl, remote);
1450	trace_gfs2_demote_rq(gl, remote);
1451	}
1452
1453	void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
1454	{
1455	struct va_format vaf;
1456	va_list args;
1457
1458	va_start(args, fmt);
1459
1460	if (seq) {
1461	seq_vprintf(m: seq, fmt, args);
1462	} else {
1463	vaf.fmt = fmt;
1464	vaf.va = &args;
1465
1466	pr_err("%pV", &vaf);
1467	}
1468
1469	va_end(args);
1470	}
1471
1472	static inline bool pid_is_meaningful(const struct gfs2_holder *gh)
1473	{
1474	if (!(gh->gh_flags & GL_NOPID))
1475	return true;
1476	if (gh->gh_state == LM_ST_UNLOCKED)
1477	return true;
1478	return false;
1479	}
1480
1481	/**
1482	* add_to_queue - Add a holder to the wait queue (but look for recursion)
1483	* @gh: the holder structure to add
1484	*
1485	* Eventually we should move the recursive locking trap to a
1486	* debugging option or something like that. This is the fast
1487	* path and needs to have the minimum number of distractions.
1488	*
1489	*/
1490
1491	static inline void add_to_queue(struct gfs2_holder *gh)
1492	__releases(&gl->gl_lockref.lock)
1493	__acquires(&gl->gl_lockref.lock)
1494	{
1495	struct gfs2_glock *gl = gh->gh_gl;
1496	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1497	struct list_head *insert_pt = NULL;
1498	struct gfs2_holder *gh2;
1499	int try_futile = 0;
1500
1501	GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
1502	if (test_and_set_bit(nr: HIF_WAIT, addr: &gh->gh_iflags))
1503	GLOCK_BUG_ON(gl, true);
1504
1505	if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
1506	if (test_bit(GLF_LOCK, &gl->gl_flags)) {
1507	struct gfs2_holder *current_gh;
1508
1509	current_gh = find_first_holder(gl);
1510	try_futile = !may_grant(gl, current_gh, gh);
1511	}
1512	if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
1513	goto fail;
1514	}
1515
1516	list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1517	if (likely(gh2->gh_owner_pid != gh->gh_owner_pid))
1518	continue;
1519	if (gh->gh_gl->gl_ops->go_type == LM_TYPE_FLOCK)
1520	continue;
1521	if (!pid_is_meaningful(gh: gh2))
1522	continue;
1523	goto trap_recursive;
1524	}
1525	list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1526	if (try_futile &&
1527	!(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
1528	fail:
1529	gh->gh_error = GLR_TRYFAILED;
1530	gfs2_holder_wake(gh);
1531	return;
1532	}
1533	if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1534	continue;
1535	}
1536	trace_gfs2_glock_queue(gh, queue: 1);
1537	gfs2_glstats_inc(gl, which: GFS2_LKS_QCOUNT);
1538	gfs2_sbstats_inc(gl, which: GFS2_LKS_QCOUNT);
1539	if (likely(insert_pt == NULL)) {
1540	list_add_tail(new: &gh->gh_list, head: &gl->gl_holders);
1541	return;
1542	}
1543	list_add_tail(new: &gh->gh_list, head: insert_pt);
1544	spin_unlock(lock: &gl->gl_lockref.lock);
1545	if (sdp->sd_lockstruct.ls_ops->lm_cancel)
1546	sdp->sd_lockstruct.ls_ops->lm_cancel(gl);
1547	spin_lock(lock: &gl->gl_lockref.lock);
1548	return;
1549
1550	trap_recursive:
1551	fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1552	fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1553	fs_err(sdp, "lock type: %d req lock state : %d\n",
1554	gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1555	fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1556	fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1557	fs_err(sdp, "lock type: %d req lock state : %d\n",
1558	gh->gh_gl->gl_name.ln_type, gh->gh_state);
1559	gfs2_dump_glock(NULL, gl, fsid: true);
1560	BUG();
1561	}
1562
1563	/**
1564	* gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock)
1565	* @gh: the holder structure
1566	*
1567	* if (gh->gh_flags & GL_ASYNC), this never returns an error
1568	*
1569	* Returns: 0, GLR_TRYFAILED, or errno on failure
1570	*/
1571
1572	int gfs2_glock_nq(struct gfs2_holder *gh)
1573	{
1574	struct gfs2_glock *gl = gh->gh_gl;
1575	int error;
1576
1577	if (glock_blocked_by_withdraw(gl) && !(gh->gh_flags & LM_FLAG_NOEXP))
1578	return -EIO;
1579
1580	if (gh->gh_flags & GL_NOBLOCK) {
1581	struct gfs2_holder *current_gh;
1582
1583	error = -ECHILD;
1584	spin_lock(lock: &gl->gl_lockref.lock);
1585	if (find_last_waiter(gl))
1586	goto unlock;
1587	current_gh = find_first_holder(gl);
1588	if (!may_grant(gl, current_gh, gh))
1589	goto unlock;
1590	set_bit(nr: HIF_HOLDER, addr: &gh->gh_iflags);
1591	list_add_tail(new: &gh->gh_list, head: &gl->gl_holders);
1592	trace_gfs2_promote(gh);
1593	error = 0;
1594	unlock:
1595	spin_unlock(lock: &gl->gl_lockref.lock);
1596	return error;
1597	}
1598
1599	if (test_bit(GLF_LRU, &gl->gl_flags))
1600	gfs2_glock_remove_from_lru(gl);
1601
1602	gh->gh_error = 0;
1603	spin_lock(lock: &gl->gl_lockref.lock);
1604	add_to_queue(gh);
1605	if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&
1606	test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) {
1607	set_bit(nr: GLF_REPLY_PENDING, addr: &gl->gl_flags);
1608	gl->gl_lockref.count++;
1609	__gfs2_glock_queue_work(gl, delay: 0);
1610	}
1611	run_queue(gl, nonblock: 1);
1612	spin_unlock(lock: &gl->gl_lockref.lock);
1613
1614	error = 0;
1615	if (!(gh->gh_flags & GL_ASYNC))
1616	error = gfs2_glock_wait(gh);
1617
1618	return error;
1619	}
1620
1621	/**
1622	* gfs2_glock_poll - poll to see if an async request has been completed
1623	* @gh: the holder
1624	*
1625	* Returns: 1 if the request is ready to be gfs2_glock_wait()ed on
1626	*/
1627
1628	int gfs2_glock_poll(struct gfs2_holder *gh)
1629	{
1630	return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1631	}
1632
1633	static inline bool needs_demote(struct gfs2_glock *gl)
1634	{
1635	return (test_bit(GLF_DEMOTE, &gl->gl_flags) ||
1636	test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags));
1637	}
1638
1639	static void __gfs2_glock_dq(struct gfs2_holder *gh)
1640	{
1641	struct gfs2_glock *gl = gh->gh_gl;
1642	unsigned delay = 0;
1643	int fast_path = 0;
1644
1645	/*
1646	* This holder should not be cached, so mark it for demote.
1647	* Note: this should be done before the check for needs_demote
1648	* below.
1649	*/
1650	if (gh->gh_flags & GL_NOCACHE)
1651	handle_callback(gl, LM_ST_UNLOCKED, delay: 0, remote: false);
1652
1653	list_del_init(entry: &gh->gh_list);
1654	clear_bit(nr: HIF_HOLDER, addr: &gh->gh_iflags);
1655	trace_gfs2_glock_queue(gh, queue: 0);
1656
1657	/*
1658	* If there hasn't been a demote request we are done.
1659	* (Let the remaining holders, if any, keep holding it.)
1660	*/
1661	if (!needs_demote(gl)) {
1662	if (list_empty(head: &gl->gl_holders))
1663	fast_path = 1;
1664	}
1665
1666	if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
1667	gfs2_glock_add_to_lru(gl);
1668
1669	if (unlikely(!fast_path)) {
1670	gl->gl_lockref.count++;
1671	if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1672	!test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1673	gl->gl_name.ln_type == LM_TYPE_INODE)
1674	delay = gl->gl_hold_time;
1675	__gfs2_glock_queue_work(gl, delay);
1676	}
1677	}
1678
1679	/**
1680	* gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock)
1681	* @gh: the glock holder
1682	*
1683	*/
1684	void gfs2_glock_dq(struct gfs2_holder *gh)
1685	{
1686	struct gfs2_glock *gl = gh->gh_gl;
1687	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1688
1689	spin_lock(lock: &gl->gl_lockref.lock);
1690	if (!gfs2_holder_queued(gh)) {
1691	/*
1692	* May have already been dequeued because the locking request
1693	* was GL_ASYNC and it has failed in the meantime.
1694	*/
1695	goto out;
1696	}
1697
1698	if (list_is_first(list: &gh->gh_list, head: &gl->gl_holders) &&
1699	!test_bit(HIF_HOLDER, &gh->gh_iflags)) {
1700	spin_unlock(lock: &gl->gl_lockref.lock);
1701	gl->gl_name.ln_sbd->sd_lockstruct.ls_ops->lm_cancel(gl);
1702	wait_on_bit(word: &gh->gh_iflags, bit: HIF_WAIT, TASK_UNINTERRUPTIBLE);
1703	spin_lock(lock: &gl->gl_lockref.lock);
1704	}
1705
1706	/*
1707	* If we're in the process of file system withdraw, we cannot just
1708	* dequeue any glocks until our journal is recovered, lest we introduce
1709	* file system corruption. We need two exceptions to this rule: We need
1710	* to allow unlocking of nondisk glocks and the glock for our own
1711	* journal that needs recovery.
1712	*/
1713	if (test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags) &&
1714	glock_blocked_by_withdraw(gl) &&
1715	gh->gh_gl != sdp->sd_jinode_gl) {
1716	sdp->sd_glock_dqs_held++;
1717	spin_unlock(lock: &gl->gl_lockref.lock);
1718	might_sleep();
1719	wait_on_bit(word: &sdp->sd_flags, bit: SDF_WITHDRAW_RECOVERY,
1720	TASK_UNINTERRUPTIBLE);
1721	spin_lock(lock: &gl->gl_lockref.lock);
1722	}
1723
1724	__gfs2_glock_dq(gh);
1725	out:
1726	spin_unlock(lock: &gl->gl_lockref.lock);
1727	}
1728
1729	void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1730	{
1731	struct gfs2_glock *gl = gh->gh_gl;
1732	gfs2_glock_dq(gh);
1733	might_sleep();
1734	wait_on_bit(word: &gl->gl_flags, bit: GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1735	}
1736
1737	/**
1738	* gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it
1739	* @gh: the holder structure
1740	*
1741	*/
1742
1743	void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1744	{
1745	gfs2_glock_dq(gh);
1746	gfs2_holder_uninit(gh);
1747	}
1748
1749	/**
1750	* gfs2_glock_nq_num - acquire a glock based on lock number
1751	* @sdp: the filesystem
1752	* @number: the lock number
1753	* @glops: the glock operations for the type of glock
1754	* @state: the state to acquire the glock in
1755	* @flags: modifier flags for the acquisition
1756	* @gh: the struct gfs2_holder
1757	*
1758	* Returns: errno
1759	*/
1760
1761	int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1762	const struct gfs2_glock_operations *glops,
1763	unsigned int state, u16 flags, struct gfs2_holder *gh)
1764	{
1765	struct gfs2_glock *gl;
1766	int error;
1767
1768	error = gfs2_glock_get(sdp, number, glops, create: CREATE, glp: &gl);
1769	if (!error) {
1770	error = gfs2_glock_nq_init(gl, state, flags, gh);
1771	gfs2_glock_put(gl);
1772	}
1773
1774	return error;
1775	}
1776
1777	/**
1778	* glock_compare - Compare two struct gfs2_glock structures for sorting
1779	* @arg_a: the first structure
1780	* @arg_b: the second structure
1781	*
1782	*/
1783
1784	static int glock_compare(const void *arg_a, const void *arg_b)
1785	{
1786	const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1787	const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1788	const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1789	const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1790
1791	if (a->ln_number > b->ln_number)
1792	return 1;
1793	if (a->ln_number < b->ln_number)
1794	return -1;
1795	BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1796	return 0;
1797	}
1798
1799	/**
1800	* nq_m_sync - synchronously acquire more than one glock in deadlock free order
1801	* @num_gh: the number of structures
1802	* @ghs: an array of struct gfs2_holder structures
1803	* @p: placeholder for the holder structure to pass back
1804	*
1805	* Returns: 0 on success (all glocks acquired),
1806	* errno on failure (no glocks acquired)
1807	*/
1808
1809	static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1810	struct gfs2_holder **p)
1811	{
1812	unsigned int x;
1813	int error = 0;
1814
1815	for (x = 0; x < num_gh; x++)
1816	p[x] = &ghs[x];
1817
1818	sort(base: p, num: num_gh, size: sizeof(struct gfs2_holder *), cmp_func: glock_compare, NULL);
1819
1820	for (x = 0; x < num_gh; x++) {
1821	error = gfs2_glock_nq(gh: p[x]);
1822	if (error) {
1823	while (x--)
1824	gfs2_glock_dq(gh: p[x]);
1825	break;
1826	}
1827	}
1828
1829	return error;
1830	}
1831
1832	/**
1833	* gfs2_glock_nq_m - acquire multiple glocks
1834	* @num_gh: the number of structures
1835	* @ghs: an array of struct gfs2_holder structures
1836	*
1837	* Returns: 0 on success (all glocks acquired),
1838	* errno on failure (no glocks acquired)
1839	*/
1840
1841	int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1842	{
1843	struct gfs2_holder *tmp[4];
1844	struct gfs2_holder **pph = tmp;
1845	int error = 0;
1846
1847	switch(num_gh) {
1848	case 0:
1849	return 0;
1850	case 1:
1851	return gfs2_glock_nq(gh: ghs);
1852	default:
1853	if (num_gh <= 4)
1854	break;
1855	pph = kmalloc_array(n: num_gh, size: sizeof(struct gfs2_holder *),
1856	GFP_NOFS);
1857	if (!pph)
1858	return -ENOMEM;
1859	}
1860
1861	error = nq_m_sync(num_gh, ghs, p: pph);
1862
1863	if (pph != tmp)
1864	kfree(objp: pph);
1865
1866	return error;
1867	}
1868
1869	/**
1870	* gfs2_glock_dq_m - release multiple glocks
1871	* @num_gh: the number of structures
1872	* @ghs: an array of struct gfs2_holder structures
1873	*
1874	*/
1875
1876	void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1877	{
1878	while (num_gh--)
1879	gfs2_glock_dq(gh: &ghs[num_gh]);
1880	}
1881
1882	void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1883	{
1884	unsigned long delay = 0;
1885	unsigned long holdtime;
1886	unsigned long now = jiffies;
1887
1888	gfs2_glock_hold(gl);
1889	spin_lock(lock: &gl->gl_lockref.lock);
1890	holdtime = gl->gl_tchange + gl->gl_hold_time;
1891	if (!list_empty(head: &gl->gl_holders) &&
1892	gl->gl_name.ln_type == LM_TYPE_INODE) {
1893	if (time_before(now, holdtime))
1894	delay = holdtime - now;
1895	if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
1896	delay = gl->gl_hold_time;
1897	}
1898	handle_callback(gl, state, delay, remote: true);
1899	__gfs2_glock_queue_work(gl, delay);
1900	spin_unlock(lock: &gl->gl_lockref.lock);
1901	}
1902
1903	/**
1904	* gfs2_should_freeze - Figure out if glock should be frozen
1905	* @gl: The glock in question
1906	*
1907	* Glocks are not frozen if (a) the result of the dlm operation is
1908	* an error, (b) the locking operation was an unlock operation or
1909	* (c) if there is a "noexp" flagged request anywhere in the queue
1910	*
1911	* Returns: 1 if freezing should occur, 0 otherwise
1912	*/
1913
1914	static int gfs2_should_freeze(const struct gfs2_glock *gl)
1915	{
1916	const struct gfs2_holder *gh;
1917
1918	if (gl->gl_reply & ~LM_OUT_ST_MASK)
1919	return 0;
1920	if (gl->gl_target == LM_ST_UNLOCKED)
1921	return 0;
1922
1923	list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1924	if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1925	continue;
1926	if (LM_FLAG_NOEXP & gh->gh_flags)
1927	return 0;
1928	}
1929
1930	return 1;
1931	}
1932
1933	/**
1934	* gfs2_glock_complete - Callback used by locking
1935	* @gl: Pointer to the glock
1936	* @ret: The return value from the dlm
1937	*
1938	* The gl_reply field is under the gl_lockref.lock lock so that it is ok
1939	* to use a bitfield shared with other glock state fields.
1940	*/
1941
1942	void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1943	{
1944	struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1945
1946	spin_lock(lock: &gl->gl_lockref.lock);
1947	gl->gl_reply = ret;
1948
1949	if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1950	if (gfs2_should_freeze(gl)) {
1951	set_bit(nr: GLF_FROZEN, addr: &gl->gl_flags);
1952	spin_unlock(lock: &gl->gl_lockref.lock);
1953	return;
1954	}
1955	}
1956
1957	gl->gl_lockref.count++;
1958	set_bit(nr: GLF_REPLY_PENDING, addr: &gl->gl_flags);
1959	__gfs2_glock_queue_work(gl, delay: 0);
1960	spin_unlock(lock: &gl->gl_lockref.lock);
1961	}
1962
1963	static int glock_cmp(void *priv, const struct list_head *a,
1964	const struct list_head *b)
1965	{
1966	struct gfs2_glock *gla, *glb;
1967
1968	gla = list_entry(a, struct gfs2_glock, gl_lru);
1969	glb = list_entry(b, struct gfs2_glock, gl_lru);
1970
1971	if (gla->gl_name.ln_number > glb->gl_name.ln_number)
1972	return 1;
1973	if (gla->gl_name.ln_number < glb->gl_name.ln_number)
1974	return -1;
1975
1976	return 0;
1977	}
1978
1979	/**
1980	* gfs2_dispose_glock_lru - Demote a list of glocks
1981	* @list: The list to dispose of
1982	*
1983	* Disposing of glocks may involve disk accesses, so that here we sort
1984	* the glocks by number (i.e. disk location of the inodes) so that if
1985	* there are any such accesses, they'll be sent in order (mostly).
1986	*
1987	* Must be called under the lru_lock, but may drop and retake this
1988	* lock. While the lru_lock is dropped, entries may vanish from the
1989	* list, but no new entries will appear on the list (since it is
1990	* private)
1991	*/
1992
1993	static void gfs2_dispose_glock_lru(struct list_head *list)
1994	__releases(&lru_lock)
1995	__acquires(&lru_lock)
1996	{
1997	struct gfs2_glock *gl;
1998
1999	list_sort(NULL, head: list, cmp: glock_cmp);
2000
2001	while(!list_empty(head: list)) {
2002	gl = list_first_entry(list, struct gfs2_glock, gl_lru);
2003	list_del_init(entry: &gl->gl_lru);
2004	clear_bit(nr: GLF_LRU, addr: &gl->gl_flags);
2005	if (!spin_trylock(lock: &gl->gl_lockref.lock)) {
2006	add_back_to_lru:
2007	list_add(new: &gl->gl_lru, head: &lru_list);
2008	set_bit(nr: GLF_LRU, addr: &gl->gl_flags);
2009	atomic_inc(v: &lru_count);
2010	continue;
2011	}
2012	if (test_and_set_bit(nr: GLF_LOCK, addr: &gl->gl_flags)) {
2013	spin_unlock(lock: &gl->gl_lockref.lock);
2014	goto add_back_to_lru;
2015	}
2016	gl->gl_lockref.count++;
2017	if (demote_ok(gl))
2018	handle_callback(gl, LM_ST_UNLOCKED, delay: 0, remote: false);
2019	WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags));
2020	__gfs2_glock_queue_work(gl, delay: 0);
2021	spin_unlock(lock: &gl->gl_lockref.lock);
2022	cond_resched_lock(&lru_lock);
2023	}
2024	}
2025
2026	/**
2027	* gfs2_scan_glock_lru - Scan the LRU looking for locks to demote
2028	* @nr: The number of entries to scan
2029	*
2030	* This function selects the entries on the LRU which are able to
2031	* be demoted, and then kicks off the process by calling
2032	* gfs2_dispose_glock_lru() above.
2033	*/
2034
2035	static long gfs2_scan_glock_lru(int nr)
2036	{
2037	struct gfs2_glock *gl, *next;
2038	LIST_HEAD(dispose);
2039	long freed = 0;
2040
2041	spin_lock(lock: &lru_lock);
2042	list_for_each_entry_safe(gl, next, &lru_list, gl_lru) {
2043	if (nr-- <= 0)
2044	break;
2045	/* Test for being demotable */
2046	if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
2047	if (!spin_trylock(lock: &gl->gl_lockref.lock))
2048	continue;
2049	if (gl->gl_lockref.count <= 1 &&
2050	(gl->gl_state == LM_ST_UNLOCKED ||
2051	demote_ok(gl))) {
2052	list_move(list: &gl->gl_lru, head: &dispose);
2053	atomic_dec(v: &lru_count);
2054	freed++;
2055	}
2056	spin_unlock(lock: &gl->gl_lockref.lock);
2057	}
2058	}
2059	if (!list_empty(head: &dispose))
2060	gfs2_dispose_glock_lru(list: &dispose);
2061	spin_unlock(lock: &lru_lock);
2062
2063	return freed;
2064	}
2065
2066	static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
2067	struct shrink_control *sc)
2068	{
2069	if (!(sc->gfp_mask & __GFP_FS))
2070	return SHRINK_STOP;
2071	return gfs2_scan_glock_lru(nr: sc->nr_to_scan);
2072	}
2073
2074	static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
2075	struct shrink_control *sc)
2076	{
2077	return vfs_pressure_ratio(val: atomic_read(v: &lru_count));
2078	}
2079
2080	static struct shrinker *glock_shrinker;
2081
2082	/**
2083	* glock_hash_walk - Call a function for glock in a hash bucket
2084	* @examiner: the function
2085	* @sdp: the filesystem
2086	*
2087	* Note that the function can be called multiple times on the same
2088	* object. So the user must ensure that the function can cope with
2089	* that.
2090	*/
2091
2092	static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
2093	{
2094	struct gfs2_glock *gl;
2095	struct rhashtable_iter iter;
2096
2097	rhashtable_walk_enter(ht: &gl_hash_table, iter: &iter);
2098
2099	do {
2100	rhashtable_walk_start(iter: &iter);
2101
2102	while ((gl = rhashtable_walk_next(iter: &iter)) && !IS_ERR(ptr: gl)) {
2103	if (gl->gl_name.ln_sbd == sdp)
2104	examiner(gl);
2105	}
2106
2107	rhashtable_walk_stop(iter: &iter);
2108	} while (cond_resched(), gl == ERR_PTR(error: -EAGAIN));
2109
2110	rhashtable_walk_exit(iter: &iter);
2111	}
2112
2113	void gfs2_cancel_delete_work(struct gfs2_glock *gl)
2114	{
2115	clear_bit(nr: GLF_TRY_TO_EVICT, addr: &gl->gl_flags);
2116	clear_bit(nr: GLF_VERIFY_EVICT, addr: &gl->gl_flags);
2117	if (cancel_delayed_work(dwork: &gl->gl_delete))
2118	gfs2_glock_put(gl);
2119	}
2120
2121	static void flush_delete_work(struct gfs2_glock *gl)
2122	{
2123	if (gl->gl_name.ln_type == LM_TYPE_IOPEN) {
2124	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2125
2126	if (cancel_delayed_work(dwork: &gl->gl_delete)) {
2127	queue_delayed_work(wq: sdp->sd_delete_wq,
2128	dwork: &gl->gl_delete, delay: 0);
2129	}
2130	}
2131	}
2132
2133	void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
2134	{
2135	glock_hash_walk(examiner: flush_delete_work, sdp);
2136	flush_workqueue(sdp->sd_delete_wq);
2137	}
2138
2139	/**
2140	* thaw_glock - thaw out a glock which has an unprocessed reply waiting
2141	* @gl: The glock to thaw
2142	*
2143	*/
2144
2145	static void thaw_glock(struct gfs2_glock *gl)
2146	{
2147	if (!test_and_clear_bit(nr: GLF_FROZEN, addr: &gl->gl_flags))
2148	return;
2149	if (!lockref_get_not_dead(&gl->gl_lockref))
2150	return;
2151	set_bit(nr: GLF_REPLY_PENDING, addr: &gl->gl_flags);
2152	gfs2_glock_queue_work(gl, delay: 0);
2153	}
2154
2155	/**
2156	* clear_glock - look at a glock and see if we can free it from glock cache
2157	* @gl: the glock to look at
2158	*
2159	*/
2160
2161	static void clear_glock(struct gfs2_glock *gl)
2162	{
2163	gfs2_glock_remove_from_lru(gl);
2164
2165	spin_lock(lock: &gl->gl_lockref.lock);
2166	if (!__lockref_is_dead(l: &gl->gl_lockref)) {
2167	gl->gl_lockref.count++;
2168	if (gl->gl_state != LM_ST_UNLOCKED)
2169	handle_callback(gl, LM_ST_UNLOCKED, delay: 0, remote: false);
2170	__gfs2_glock_queue_work(gl, delay: 0);
2171	}
2172	spin_unlock(lock: &gl->gl_lockref.lock);
2173	}
2174
2175	/**
2176	* gfs2_glock_thaw - Thaw any frozen glocks
2177	* @sdp: The super block
2178	*
2179	*/
2180
2181	void gfs2_glock_thaw(struct gfs2_sbd *sdp)
2182	{
2183	glock_hash_walk(examiner: thaw_glock, sdp);
2184	}
2185
2186	static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2187	{
2188	spin_lock(lock: &gl->gl_lockref.lock);
2189	gfs2_dump_glock(seq, gl, fsid);
2190	spin_unlock(lock: &gl->gl_lockref.lock);
2191	}
2192
2193	static void dump_glock_func(struct gfs2_glock *gl)
2194	{
2195	dump_glock(NULL, gl, fsid: true);
2196	}
2197
2198	static void withdraw_dq(struct gfs2_glock *gl)
2199	{
2200	spin_lock(lock: &gl->gl_lockref.lock);
2201	if (!__lockref_is_dead(l: &gl->gl_lockref) &&
2202	glock_blocked_by_withdraw(gl))
2203	do_error(gl, LM_OUT_ERROR); /* remove pending waiters */
2204	spin_unlock(lock: &gl->gl_lockref.lock);
2205	}
2206
2207	void gfs2_gl_dq_holders(struct gfs2_sbd *sdp)
2208	{
2209	glock_hash_walk(examiner: withdraw_dq, sdp);
2210	}
2211
2212	/**
2213	* gfs2_gl_hash_clear - Empty out the glock hash table
2214	* @sdp: the filesystem
2215	*
2216	* Called when unmounting the filesystem.
2217	*/
2218
2219	void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
2220	{
2221	set_bit(nr: SDF_SKIP_DLM_UNLOCK, addr: &sdp->sd_flags);
2222	flush_workqueue(glock_workqueue);
2223	glock_hash_walk(examiner: clear_glock, sdp);
2224	flush_workqueue(glock_workqueue);
2225	wait_event_timeout(sdp->sd_kill_wait,
2226	atomic_read(&sdp->sd_glock_disposal) == 0,
2227	HZ * 600);
2228	glock_hash_walk(examiner: dump_glock_func, sdp);
2229	}
2230
2231	static const char *state2str(unsigned state)
2232	{
2233	switch(state) {
2234	case LM_ST_UNLOCKED:
2235	return "UN";
2236	case LM_ST_SHARED:
2237	return "SH";
2238	case LM_ST_DEFERRED:
2239	return "DF";
2240	case LM_ST_EXCLUSIVE:
2241	return "EX";
2242	}
2243	return "??";
2244	}
2245
2246	static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
2247	{
2248	char *p = buf;
2249	if (flags & LM_FLAG_TRY)
2250	*p++ = 't';
2251	if (flags & LM_FLAG_TRY_1CB)
2252	*p++ = 'T';
2253	if (flags & LM_FLAG_NOEXP)
2254	*p++ = 'e';
2255	if (flags & LM_FLAG_ANY)
2256	*p++ = 'A';
2257	if (flags & LM_FLAG_NODE_SCOPE)
2258	*p++ = 'n';
2259	if (flags & GL_ASYNC)
2260	*p++ = 'a';
2261	if (flags & GL_EXACT)
2262	*p++ = 'E';
2263	if (flags & GL_NOCACHE)
2264	*p++ = 'c';
2265	if (test_bit(HIF_HOLDER, &iflags))
2266	*p++ = 'H';
2267	if (test_bit(HIF_WAIT, &iflags))
2268	*p++ = 'W';
2269	if (flags & GL_SKIP)
2270	*p++ = 's';
2271	*p = 0;
2272	return buf;
2273	}
2274
2275	/**
2276	* dump_holder - print information about a glock holder
2277	* @seq: the seq_file struct
2278	* @gh: the glock holder
2279	* @fs_id_buf: pointer to file system id (if requested)
2280	*
2281	*/
2282
2283	static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
2284	const char *fs_id_buf)
2285	{
2286	const char *comm = "(none)";
2287	pid_t owner_pid = 0;
2288	char flags_buf[32];
2289
2290	rcu_read_lock();
2291	if (pid_is_meaningful(gh)) {
2292	struct task_struct *gh_owner;
2293
2294	comm = "(ended)";
2295	owner_pid = pid_nr(pid: gh->gh_owner_pid);
2296	gh_owner = pid_task(pid: gh->gh_owner_pid, PIDTYPE_PID);
2297	if (gh_owner)
2298	comm = gh_owner->comm;
2299	}
2300	gfs2_print_dbg(seq, fmt: "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
2301	fs_id_buf, state2str(state: gh->gh_state),
2302	hflags2str(buf: flags_buf, flags: gh->gh_flags, iflags: gh->gh_iflags),
2303	gh->gh_error, (long)owner_pid, comm, (void *)gh->gh_ip);
2304	rcu_read_unlock();
2305	}
2306
2307	static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
2308	{
2309	const unsigned long *gflags = &gl->gl_flags;
2310	char *p = buf;
2311
2312	if (test_bit(GLF_LOCK, gflags))
2313	*p++ = 'l';
2314	if (test_bit(GLF_DEMOTE, gflags))
2315	*p++ = 'D';
2316	if (test_bit(GLF_PENDING_DEMOTE, gflags))
2317	*p++ = 'd';
2318	if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
2319	*p++ = 'p';
2320	if (test_bit(GLF_DIRTY, gflags))
2321	*p++ = 'y';
2322	if (test_bit(GLF_LFLUSH, gflags))
2323	*p++ = 'f';
2324	if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags))
2325	*p++ = 'i';
2326	if (test_bit(GLF_REPLY_PENDING, gflags))
2327	*p++ = 'r';
2328	if (test_bit(GLF_INITIAL, gflags))
2329	*p++ = 'I';
2330	if (test_bit(GLF_FROZEN, gflags))
2331	*p++ = 'F';
2332	if (!list_empty(head: &gl->gl_holders))
2333	*p++ = 'q';
2334	if (test_bit(GLF_LRU, gflags))
2335	*p++ = 'L';
2336	if (gl->gl_object)
2337	*p++ = 'o';
2338	if (test_bit(GLF_BLOCKING, gflags))
2339	*p++ = 'b';
2340	if (test_bit(GLF_FREEING, gflags))
2341	*p++ = 'x';
2342	if (test_bit(GLF_INSTANTIATE_NEEDED, gflags))
2343	*p++ = 'n';
2344	if (test_bit(GLF_INSTANTIATE_IN_PROG, gflags))
2345	*p++ = 'N';
2346	if (test_bit(GLF_TRY_TO_EVICT, gflags))
2347	*p++ = 'e';
2348	if (test_bit(GLF_VERIFY_EVICT, gflags))
2349	*p++ = 'E';
2350	*p = 0;
2351	return buf;
2352	}
2353
2354	/**
2355	* gfs2_dump_glock - print information about a glock
2356	* @seq: The seq_file struct
2357	* @gl: the glock
2358	* @fsid: If true, also dump the file system id
2359	*
2360	* The file format is as follows:
2361	* One line per object, capital letters are used to indicate objects
2362	* G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented,
2363	* other objects are indented by a single space and follow the glock to
2364	* which they are related. Fields are indicated by lower case letters
2365	* followed by a colon and the field value, except for strings which are in
2366	* [] so that its possible to see if they are composed of spaces for
2367	* example. The field's are n = number (id of the object), f = flags,
2368	* t = type, s = state, r = refcount, e = error, p = pid.
2369	*
2370	*/
2371
2372	void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2373	{
2374	const struct gfs2_glock_operations *glops = gl->gl_ops;
2375	unsigned long long dtime;
2376	const struct gfs2_holder *gh;
2377	char gflags_buf[32];
2378	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2379	char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
2380	unsigned long nrpages = 0;
2381
2382	if (gl->gl_ops->go_flags & GLOF_ASPACE) {
2383	struct address_space *mapping = gfs2_glock2aspace(gl);
2384
2385	nrpages = mapping->nrpages;
2386	}
2387	memset(fs_id_buf, 0, sizeof(fs_id_buf));
2388	if (fsid && sdp) /* safety precaution */
2389	sprintf(buf: fs_id_buf, fmt: "fsid=%s: ", sdp->sd_fsname);
2390	dtime = jiffies - gl->gl_demote_time;
2391	dtime *= 1000000/HZ; /* demote time in uSec */
2392	if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
2393	dtime = 0;
2394	gfs2_print_dbg(seq, fmt: "%sG: s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
2395	"v:%d r:%d m:%ld p:%lu\n",
2396	fs_id_buf, state2str(state: gl->gl_state),
2397	gl->gl_name.ln_type,
2398	(unsigned long long)gl->gl_name.ln_number,
2399	gflags2str(buf: gflags_buf, gl),
2400	state2str(state: gl->gl_target),
2401	state2str(state: gl->gl_demote_state), dtime,
2402	atomic_read(v: &gl->gl_ail_count),
2403	atomic_read(v: &gl->gl_revokes),
2404	(int)gl->gl_lockref.count, gl->gl_hold_time, nrpages);
2405
2406	list_for_each_entry(gh, &gl->gl_holders, gh_list)
2407	dump_holder(seq, gh, fs_id_buf);
2408
2409	if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
2410	glops->go_dump(seq, gl, fs_id_buf);
2411	}
2412
2413	static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
2414	{
2415	struct gfs2_glock *gl = iter_ptr;
2416
2417	seq_printf(m: seq, fmt: "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
2418	gl->gl_name.ln_type,
2419	(unsigned long long)gl->gl_name.ln_number,
2420	(unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
2421	(unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
2422	(unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
2423	(unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
2424	(unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
2425	(unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
2426	(unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
2427	(unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
2428	return 0;
2429	}
2430
2431	static const char *gfs2_gltype[] = {
2432	"type",
2433	"reserved",
2434	"nondisk",
2435	"inode",
2436	"rgrp",
2437	"meta",
2438	"iopen",
2439	"flock",
2440	"plock",
2441	"quota",
2442	"journal",
2443	};
2444
2445	static const char *gfs2_stype[] = {
2446	[GFS2_LKS_SRTT] = "srtt",
2447	[GFS2_LKS_SRTTVAR] = "srttvar",
2448	[GFS2_LKS_SRTTB] = "srttb",
2449	[GFS2_LKS_SRTTVARB] = "srttvarb",
2450	[GFS2_LKS_SIRT] = "sirt",
2451	[GFS2_LKS_SIRTVAR] = "sirtvar",
2452	[GFS2_LKS_DCOUNT] = "dlm",
2453	[GFS2_LKS_QCOUNT] = "queue",
2454	};
2455
2456	#define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
2457
2458	static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
2459	{
2460	struct gfs2_sbd *sdp = seq->private;
2461	loff_t pos = *(loff_t *)iter_ptr;
2462	unsigned index = pos >> 3;
2463	unsigned subindex = pos & 0x07;
2464	int i;
2465
2466	if (index == 0 && subindex != 0)
2467	return 0;
2468
2469	seq_printf(m: seq, fmt: "%-10s %8s:", gfs2_gltype[index],
2470	(index == 0) ? "cpu": gfs2_stype[subindex]);
2471
2472	for_each_possible_cpu(i) {
2473	const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
2474
2475	if (index == 0)
2476	seq_printf(m: seq, fmt: " %15u", i);
2477	else
2478	seq_printf(m: seq, fmt: " %15llu", (unsigned long long)lkstats->
2479	lkstats[index - 1].stats[subindex]);
2480	}
2481	seq_putc(m: seq, c: '\n');
2482	return 0;
2483	}
2484
2485	int __init gfs2_glock_init(void)
2486	{
2487	int i, ret;
2488
2489	ret = rhashtable_init(ht: &gl_hash_table, params: &ht_parms);
2490	if (ret < 0)
2491	return ret;
2492
2493	glock_workqueue = alloc_workqueue(fmt: "glock_workqueue", flags: WQ_MEM_RECLAIM |
2494	WQ_HIGHPRI | WQ_FREEZABLE, max_active: 0);
2495	if (!glock_workqueue) {
2496	rhashtable_destroy(ht: &gl_hash_table);
2497	return -ENOMEM;
2498	}
2499
2500	glock_shrinker = shrinker_alloc(flags: 0, fmt: "gfs2-glock");
2501	if (!glock_shrinker) {
2502	destroy_workqueue(wq: glock_workqueue);
2503	rhashtable_destroy(ht: &gl_hash_table);
2504	return -ENOMEM;
2505	}
2506
2507	glock_shrinker->count_objects = gfs2_glock_shrink_count;
2508	glock_shrinker->scan_objects = gfs2_glock_shrink_scan;
2509
2510	shrinker_register(shrinker: glock_shrinker);
2511
2512	for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
2513	init_waitqueue_head(glock_wait_table + i);
2514
2515	return 0;
2516	}
2517
2518	void gfs2_glock_exit(void)
2519	{
2520	shrinker_free(shrinker: glock_shrinker);
2521	rhashtable_destroy(ht: &gl_hash_table);
2522	destroy_workqueue(wq: glock_workqueue);
2523	}
2524
2525	static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
2526	{
2527	struct gfs2_glock *gl = gi->gl;
2528
2529	if (gl) {
2530	if (n == 0)
2531	return;
2532	if (!lockref_put_not_zero(&gl->gl_lockref))
2533	gfs2_glock_queue_put(gl);
2534	}
2535	for (;;) {
2536	gl = rhashtable_walk_next(iter: &gi->hti);
2537	if (IS_ERR_OR_NULL(ptr: gl)) {
2538	if (gl == ERR_PTR(error: -EAGAIN)) {
2539	n = 1;
2540	continue;
2541	}
2542	gl = NULL;
2543	break;
2544	}
2545	if (gl->gl_name.ln_sbd != gi->sdp)
2546	continue;
2547	if (n <= 1) {
2548	if (!lockref_get_not_dead(&gl->gl_lockref))
2549	continue;
2550	break;
2551	} else {
2552	if (__lockref_is_dead(l: &gl->gl_lockref))
2553	continue;
2554	n--;
2555	}
2556	}
2557	gi->gl = gl;
2558	}
2559
2560	static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
2561	__acquires(RCU)
2562	{
2563	struct gfs2_glock_iter *gi = seq->private;
2564	loff_t n;
2565
2566	/*
2567	* We can either stay where we are, skip to the next hash table
2568	* entry, or start from the beginning.
2569	*/
2570	if (*pos < gi->last_pos) {
2571	rhashtable_walk_exit(iter: &gi->hti);
2572	rhashtable_walk_enter(ht: &gl_hash_table, iter: &gi->hti);
2573	n = *pos + 1;
2574	} else {
2575	n = *pos - gi->last_pos;
2576	}
2577
2578	rhashtable_walk_start(iter: &gi->hti);
2579
2580	gfs2_glock_iter_next(gi, n);
2581	gi->last_pos = *pos;
2582	return gi->gl;
2583	}
2584
2585	static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
2586	loff_t *pos)
2587	{
2588	struct gfs2_glock_iter *gi = seq->private;
2589
2590	(*pos)++;
2591	gi->last_pos = *pos;
2592	gfs2_glock_iter_next(gi, n: 1);
2593	return gi->gl;
2594	}
2595
2596	static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2597	__releases(RCU)
2598	{
2599	struct gfs2_glock_iter *gi = seq->private;
2600
2601	rhashtable_walk_stop(iter: &gi->hti);
2602	}
2603
2604	static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2605	{
2606	dump_glock(seq, gl: iter_ptr, fsid: false);
2607	return 0;
2608	}
2609
2610	static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2611	{
2612	preempt_disable();
2613	if (*pos >= GFS2_NR_SBSTATS)
2614	return NULL;
2615	return pos;
2616	}
2617
2618	static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2619	loff_t *pos)
2620	{
2621	(*pos)++;
2622	if (*pos >= GFS2_NR_SBSTATS)
2623	return NULL;
2624	return pos;
2625	}
2626
2627	static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2628	{
2629	preempt_enable();
2630	}
2631
2632	static const struct seq_operations gfs2_glock_seq_ops = {
2633	.start = gfs2_glock_seq_start,
2634	.next = gfs2_glock_seq_next,
2635	.stop = gfs2_glock_seq_stop,
2636	.show = gfs2_glock_seq_show,
2637	};
2638
2639	static const struct seq_operations gfs2_glstats_seq_ops = {
2640	.start = gfs2_glock_seq_start,
2641	.next = gfs2_glock_seq_next,
2642	.stop = gfs2_glock_seq_stop,
2643	.show = gfs2_glstats_seq_show,
2644	};
2645
2646	static const struct seq_operations gfs2_sbstats_sops = {
2647	.start = gfs2_sbstats_seq_start,
2648	.next = gfs2_sbstats_seq_next,
2649	.stop = gfs2_sbstats_seq_stop,
2650	.show = gfs2_sbstats_seq_show,
2651	};
2652
2653	#define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2654
2655	static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2656	const struct seq_operations *ops)
2657	{
2658	int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2659	if (ret == 0) {
2660	struct seq_file *seq = file->private_data;
2661	struct gfs2_glock_iter *gi = seq->private;
2662
2663	gi->sdp = inode->i_private;
2664	seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2665	if (seq->buf)
2666	seq->size = GFS2_SEQ_GOODSIZE;
2667	/*
2668	* Initially, we are "before" the first hash table entry; the
2669	* first call to rhashtable_walk_next gets us the first entry.
2670	*/
2671	gi->last_pos = -1;
2672	gi->gl = NULL;
2673	rhashtable_walk_enter(ht: &gl_hash_table, iter: &gi->hti);
2674	}
2675	return ret;
2676	}
2677
2678	static int gfs2_glocks_open(struct inode *inode, struct file *file)
2679	{
2680	return __gfs2_glocks_open(inode, file, ops: &gfs2_glock_seq_ops);
2681	}
2682
2683	static int gfs2_glocks_release(struct inode *inode, struct file *file)
2684	{
2685	struct seq_file *seq = file->private_data;
2686	struct gfs2_glock_iter *gi = seq->private;
2687
2688	if (gi->gl)
2689	gfs2_glock_put(gl: gi->gl);
2690	rhashtable_walk_exit(iter: &gi->hti);
2691	return seq_release_private(inode, file);
2692	}
2693
2694	static int gfs2_glstats_open(struct inode *inode, struct file *file)
2695	{
2696	return __gfs2_glocks_open(inode, file, ops: &gfs2_glstats_seq_ops);
2697	}
2698
2699	static const struct file_operations gfs2_glocks_fops = {
2700	.owner = THIS_MODULE,
2701	.open = gfs2_glocks_open,
2702	.read = seq_read,
2703	.llseek = seq_lseek,
2704	.release = gfs2_glocks_release,
2705	};
2706
2707	static const struct file_operations gfs2_glstats_fops = {
2708	.owner = THIS_MODULE,
2709	.open = gfs2_glstats_open,
2710	.read = seq_read,
2711	.llseek = seq_lseek,
2712	.release = gfs2_glocks_release,
2713	};
2714
2715	struct gfs2_glockfd_iter {
2716	struct super_block *sb;
2717	unsigned int tgid;
2718	struct task_struct *task;
2719	unsigned int fd;
2720	struct file *file;
2721	};
2722
2723	static struct task_struct *gfs2_glockfd_next_task(struct gfs2_glockfd_iter *i)
2724	{
2725	struct pid_namespace *ns = task_active_pid_ns(current);
2726	struct pid *pid;
2727
2728	if (i->task)
2729	put_task_struct(t: i->task);
2730
2731	rcu_read_lock();
2732	retry:
2733	i->task = NULL;
2734	pid = find_ge_pid(nr: i->tgid, ns);
2735	if (pid) {
2736	i->tgid = pid_nr_ns(pid, ns);
2737	i->task = pid_task(pid, PIDTYPE_TGID);
2738	if (!i->task) {
2739	i->tgid++;
2740	goto retry;
2741	}
2742	get_task_struct(t: i->task);
2743	}
2744	rcu_read_unlock();
2745	return i->task;
2746	}
2747
2748	static struct file *gfs2_glockfd_next_file(struct gfs2_glockfd_iter *i)
2749	{
2750	if (i->file) {
2751	fput(i->file);
2752	i->file = NULL;
2753	}
2754
2755	rcu_read_lock();
2756	for(;; i->fd++) {
2757	struct inode *inode;
2758
2759	i->file = task_lookup_next_fdget_rcu(task: i->task, fd: &i->fd);
2760	if (!i->file) {
2761	i->fd = 0;
2762	break;
2763	}
2764
2765	inode = file_inode(f: i->file);
2766	if (inode->i_sb == i->sb)
2767	break;
2768
2769	rcu_read_unlock();
2770	fput(i->file);
2771	rcu_read_lock();
2772	}
2773	rcu_read_unlock();
2774	return i->file;
2775	}
2776
2777	static void *gfs2_glockfd_seq_start(struct seq_file *seq, loff_t *pos)
2778	{
2779	struct gfs2_glockfd_iter *i = seq->private;
2780
2781	if (*pos)
2782	return NULL;
2783	while (gfs2_glockfd_next_task(i)) {
2784	if (gfs2_glockfd_next_file(i))
2785	return i;
2786	i->tgid++;
2787	}
2788	return NULL;
2789	}
2790
2791	static void *gfs2_glockfd_seq_next(struct seq_file *seq, void *iter_ptr,
2792	loff_t *pos)
2793	{
2794	struct gfs2_glockfd_iter *i = seq->private;
2795
2796	(*pos)++;
2797	i->fd++;
2798	do {
2799	if (gfs2_glockfd_next_file(i))
2800	return i;
2801	i->tgid++;
2802	} while (gfs2_glockfd_next_task(i));
2803	return NULL;
2804	}
2805
2806	static void gfs2_glockfd_seq_stop(struct seq_file *seq, void *iter_ptr)
2807	{
2808	struct gfs2_glockfd_iter *i = seq->private;
2809
2810	if (i->file)
2811	fput(i->file);
2812	if (i->task)
2813	put_task_struct(t: i->task);
2814	}
2815
2816	static void gfs2_glockfd_seq_show_flock(struct seq_file *seq,
2817	struct gfs2_glockfd_iter *i)
2818	{
2819	struct gfs2_file *fp = i->file->private_data;
2820	struct gfs2_holder *fl_gh = &fp->f_fl_gh;
2821	struct lm_lockname gl_name = { .ln_type = LM_TYPE_RESERVED };
2822
2823	if (!READ_ONCE(fl_gh->gh_gl))
2824	return;
2825
2826	spin_lock(lock: &i->file->f_lock);
2827	if (gfs2_holder_initialized(gh: fl_gh))
2828	gl_name = fl_gh->gh_gl->gl_name;
2829	spin_unlock(lock: &i->file->f_lock);
2830
2831	if (gl_name.ln_type != LM_TYPE_RESERVED) {
2832	seq_printf(m: seq, fmt: "%d %u %u/%llx\n",
2833	i->tgid, i->fd, gl_name.ln_type,
2834	(unsigned long long)gl_name.ln_number);
2835	}
2836	}
2837
2838	static int gfs2_glockfd_seq_show(struct seq_file *seq, void *iter_ptr)
2839	{
2840	struct gfs2_glockfd_iter *i = seq->private;
2841	struct inode *inode = file_inode(f: i->file);
2842	struct gfs2_glock *gl;
2843
2844	inode_lock_shared(inode);
2845	gl = GFS2_I(inode)->i_iopen_gh.gh_gl;
2846	if (gl) {
2847	seq_printf(m: seq, fmt: "%d %u %u/%llx\n",
2848	i->tgid, i->fd, gl->gl_name.ln_type,
2849	(unsigned long long)gl->gl_name.ln_number);
2850	}
2851	gfs2_glockfd_seq_show_flock(seq, i);
2852	inode_unlock_shared(inode);
2853	return 0;
2854	}
2855
2856	static const struct seq_operations gfs2_glockfd_seq_ops = {
2857	.start = gfs2_glockfd_seq_start,
2858	.next = gfs2_glockfd_seq_next,
2859	.stop = gfs2_glockfd_seq_stop,
2860	.show = gfs2_glockfd_seq_show,
2861	};
2862
2863	static int gfs2_glockfd_open(struct inode *inode, struct file *file)
2864	{
2865	struct gfs2_glockfd_iter *i;
2866	struct gfs2_sbd *sdp = inode->i_private;
2867
2868	i = __seq_open_private(file, &gfs2_glockfd_seq_ops,
2869	sizeof(struct gfs2_glockfd_iter));
2870	if (!i)
2871	return -ENOMEM;
2872	i->sb = sdp->sd_vfs;
2873	return 0;
2874	}
2875
2876	static const struct file_operations gfs2_glockfd_fops = {
2877	.owner = THIS_MODULE,
2878	.open = gfs2_glockfd_open,
2879	.read = seq_read,
2880	.llseek = seq_lseek,
2881	.release = seq_release_private,
2882	};
2883
2884	DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats);
2885
2886	void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2887	{
2888	sdp->debugfs_dir = debugfs_create_dir(name: sdp->sd_table_name, parent: gfs2_root);
2889
2890	debugfs_create_file(name: "glocks", S_IFREG | S_IRUGO, parent: sdp->debugfs_dir, data: sdp,
2891	fops: &gfs2_glocks_fops);
2892
2893	debugfs_create_file(name: "glockfd", S_IFREG | S_IRUGO, parent: sdp->debugfs_dir, data: sdp,
2894	fops: &gfs2_glockfd_fops);
2895
2896	debugfs_create_file(name: "glstats", S_IFREG | S_IRUGO, parent: sdp->debugfs_dir, data: sdp,
2897	fops: &gfs2_glstats_fops);
2898
2899	debugfs_create_file(name: "sbstats", S_IFREG | S_IRUGO, parent: sdp->debugfs_dir, data: sdp,
2900	fops: &gfs2_sbstats_fops);
2901	}
2902
2903	void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2904	{
2905	debugfs_remove_recursive(dentry: sdp->debugfs_dir);
2906	sdp->debugfs_dir = NULL;
2907	}
2908
2909	void gfs2_register_debugfs(void)
2910	{
2911	gfs2_root = debugfs_create_dir(name: "gfs2", NULL);
2912	}
2913
2914	void gfs2_unregister_debugfs(void)
2915	{
2916	debugfs_remove(dentry: gfs2_root);
2917	gfs2_root = NULL;
2918	}
2919