1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2016 Trond Myklebust |
4 | * Copyright (c) 2019 Jeff Layton |
5 | * |
6 | * I/O and data path helper functionality. |
7 | * |
8 | * Heavily borrowed from equivalent code in fs/nfs/io.c |
9 | */ |
10 | |
11 | #include <linux/ceph/ceph_debug.h> |
12 | |
13 | #include <linux/types.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/rwsem.h> |
16 | #include <linux/fs.h> |
17 | |
18 | #include "super.h" |
19 | #include "io.h" |
20 | |
21 | /* Call with exclusively locked inode->i_rwsem */ |
22 | static void ceph_block_o_direct(struct ceph_inode_info *ci, struct inode *inode) |
23 | { |
24 | lockdep_assert_held_write(&inode->i_rwsem); |
25 | |
26 | if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT) { |
27 | spin_lock(lock: &ci->i_ceph_lock); |
28 | ci->i_ceph_flags &= ~CEPH_I_ODIRECT; |
29 | spin_unlock(lock: &ci->i_ceph_lock); |
30 | inode_dio_wait(inode); |
31 | } |
32 | } |
33 | |
34 | /** |
35 | * ceph_start_io_read - declare the file is being used for buffered reads |
36 | * @inode: file inode |
37 | * |
38 | * Declare that a buffered read operation is about to start, and ensure |
39 | * that we block all direct I/O. |
40 | * On exit, the function ensures that the CEPH_I_ODIRECT flag is unset, |
41 | * and holds a shared lock on inode->i_rwsem to ensure that the flag |
42 | * cannot be changed. |
43 | * In practice, this means that buffered read operations are allowed to |
44 | * execute in parallel, thanks to the shared lock, whereas direct I/O |
45 | * operations need to wait to grab an exclusive lock in order to set |
46 | * CEPH_I_ODIRECT. |
47 | * Note that buffered writes and truncates both take a write lock on |
48 | * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. |
49 | */ |
50 | void |
51 | ceph_start_io_read(struct inode *inode) |
52 | { |
53 | struct ceph_inode_info *ci = ceph_inode(inode); |
54 | |
55 | /* Be an optimist! */ |
56 | down_read(sem: &inode->i_rwsem); |
57 | if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT)) |
58 | return; |
59 | up_read(sem: &inode->i_rwsem); |
60 | /* Slow path.... */ |
61 | down_write(sem: &inode->i_rwsem); |
62 | ceph_block_o_direct(ci, inode); |
63 | downgrade_write(sem: &inode->i_rwsem); |
64 | } |
65 | |
66 | /** |
67 | * ceph_end_io_read - declare that the buffered read operation is done |
68 | * @inode: file inode |
69 | * |
70 | * Declare that a buffered read operation is done, and release the shared |
71 | * lock on inode->i_rwsem. |
72 | */ |
73 | void |
74 | ceph_end_io_read(struct inode *inode) |
75 | { |
76 | up_read(sem: &inode->i_rwsem); |
77 | } |
78 | |
79 | /** |
80 | * ceph_start_io_write - declare the file is being used for buffered writes |
81 | * @inode: file inode |
82 | * |
83 | * Declare that a buffered write operation is about to start, and ensure |
84 | * that we block all direct I/O. |
85 | */ |
86 | void |
87 | ceph_start_io_write(struct inode *inode) |
88 | { |
89 | down_write(sem: &inode->i_rwsem); |
90 | ceph_block_o_direct(ci: ceph_inode(inode), inode); |
91 | } |
92 | |
93 | /** |
94 | * ceph_end_io_write - declare that the buffered write operation is done |
95 | * @inode: file inode |
96 | * |
97 | * Declare that a buffered write operation is done, and release the |
98 | * lock on inode->i_rwsem. |
99 | */ |
100 | void |
101 | ceph_end_io_write(struct inode *inode) |
102 | { |
103 | up_write(sem: &inode->i_rwsem); |
104 | } |
105 | |
106 | /* Call with exclusively locked inode->i_rwsem */ |
107 | static void ceph_block_buffered(struct ceph_inode_info *ci, struct inode *inode) |
108 | { |
109 | lockdep_assert_held_write(&inode->i_rwsem); |
110 | |
111 | if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT)) { |
112 | spin_lock(lock: &ci->i_ceph_lock); |
113 | ci->i_ceph_flags |= CEPH_I_ODIRECT; |
114 | spin_unlock(lock: &ci->i_ceph_lock); |
115 | /* FIXME: unmap_mapping_range? */ |
116 | filemap_write_and_wait(mapping: inode->i_mapping); |
117 | } |
118 | } |
119 | |
120 | /** |
121 | * ceph_start_io_direct - declare the file is being used for direct i/o |
122 | * @inode: file inode |
123 | * |
124 | * Declare that a direct I/O operation is about to start, and ensure |
125 | * that we block all buffered I/O. |
126 | * On exit, the function ensures that the CEPH_I_ODIRECT flag is set, |
127 | * and holds a shared lock on inode->i_rwsem to ensure that the flag |
128 | * cannot be changed. |
129 | * In practice, this means that direct I/O operations are allowed to |
130 | * execute in parallel, thanks to the shared lock, whereas buffered I/O |
131 | * operations need to wait to grab an exclusive lock in order to clear |
132 | * CEPH_I_ODIRECT. |
133 | * Note that buffered writes and truncates both take a write lock on |
134 | * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. |
135 | */ |
136 | void |
137 | ceph_start_io_direct(struct inode *inode) |
138 | { |
139 | struct ceph_inode_info *ci = ceph_inode(inode); |
140 | |
141 | /* Be an optimist! */ |
142 | down_read(sem: &inode->i_rwsem); |
143 | if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT) |
144 | return; |
145 | up_read(sem: &inode->i_rwsem); |
146 | /* Slow path.... */ |
147 | down_write(sem: &inode->i_rwsem); |
148 | ceph_block_buffered(ci, inode); |
149 | downgrade_write(sem: &inode->i_rwsem); |
150 | } |
151 | |
152 | /** |
153 | * ceph_end_io_direct - declare that the direct i/o operation is done |
154 | * @inode: file inode |
155 | * |
156 | * Declare that a direct I/O operation is done, and release the shared |
157 | * lock on inode->i_rwsem. |
158 | */ |
159 | void |
160 | ceph_end_io_direct(struct inode *inode) |
161 | { |
162 | up_read(sem: &inode->i_rwsem); |
163 | } |
164 | |