1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2018 HUAWEI, Inc. |
4 | * https://www.huawei.com/ |
5 | */ |
6 | #include "internal.h" |
7 | |
8 | struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp) |
9 | { |
10 | struct page *page = *pagepool; |
11 | |
12 | if (page) { |
13 | DBG_BUGON(page_ref_count(page) != 1); |
14 | *pagepool = (struct page *)page_private(page); |
15 | } else { |
16 | page = alloc_page(gfp); |
17 | } |
18 | return page; |
19 | } |
20 | |
21 | void erofs_release_pages(struct page **pagepool) |
22 | { |
23 | while (*pagepool) { |
24 | struct page *page = *pagepool; |
25 | |
26 | *pagepool = (struct page *)page_private(page); |
27 | put_page(page); |
28 | } |
29 | } |
30 | |
31 | #ifdef CONFIG_EROFS_FS_ZIP |
32 | /* global shrink count (for all mounted EROFS instances) */ |
33 | static atomic_long_t erofs_global_shrink_cnt; |
34 | |
35 | static bool erofs_workgroup_get(struct erofs_workgroup *grp) |
36 | { |
37 | if (lockref_get_not_zero(&grp->lockref)) |
38 | return true; |
39 | |
40 | spin_lock(lock: &grp->lockref.lock); |
41 | if (__lockref_is_dead(l: &grp->lockref)) { |
42 | spin_unlock(lock: &grp->lockref.lock); |
43 | return false; |
44 | } |
45 | |
46 | if (!grp->lockref.count++) |
47 | atomic_long_dec(v: &erofs_global_shrink_cnt); |
48 | spin_unlock(lock: &grp->lockref.lock); |
49 | return true; |
50 | } |
51 | |
52 | struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb, |
53 | pgoff_t index) |
54 | { |
55 | struct erofs_sb_info *sbi = EROFS_SB(sb); |
56 | struct erofs_workgroup *grp; |
57 | |
58 | repeat: |
59 | rcu_read_lock(); |
60 | grp = xa_load(&sbi->managed_pslots, index); |
61 | if (grp) { |
62 | if (!erofs_workgroup_get(grp)) { |
63 | /* prefer to relax rcu read side */ |
64 | rcu_read_unlock(); |
65 | goto repeat; |
66 | } |
67 | |
68 | DBG_BUGON(index != grp->index); |
69 | } |
70 | rcu_read_unlock(); |
71 | return grp; |
72 | } |
73 | |
74 | struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb, |
75 | struct erofs_workgroup *grp) |
76 | { |
77 | struct erofs_sb_info *const sbi = EROFS_SB(sb); |
78 | struct erofs_workgroup *pre; |
79 | |
80 | DBG_BUGON(grp->lockref.count < 1); |
81 | repeat: |
82 | xa_lock(&sbi->managed_pslots); |
83 | pre = __xa_cmpxchg(&sbi->managed_pslots, index: grp->index, |
84 | NULL, entry: grp, GFP_KERNEL); |
85 | if (pre) { |
86 | if (xa_is_err(entry: pre)) { |
87 | pre = ERR_PTR(error: xa_err(entry: pre)); |
88 | } else if (!erofs_workgroup_get(grp: pre)) { |
89 | /* try to legitimize the current in-tree one */ |
90 | xa_unlock(&sbi->managed_pslots); |
91 | cond_resched(); |
92 | goto repeat; |
93 | } |
94 | grp = pre; |
95 | } |
96 | xa_unlock(&sbi->managed_pslots); |
97 | return grp; |
98 | } |
99 | |
100 | static void __erofs_workgroup_free(struct erofs_workgroup *grp) |
101 | { |
102 | atomic_long_dec(v: &erofs_global_shrink_cnt); |
103 | erofs_workgroup_free_rcu(grp); |
104 | } |
105 | |
106 | void erofs_workgroup_put(struct erofs_workgroup *grp) |
107 | { |
108 | if (lockref_put_or_lock(&grp->lockref)) |
109 | return; |
110 | |
111 | DBG_BUGON(__lockref_is_dead(&grp->lockref)); |
112 | if (grp->lockref.count == 1) |
113 | atomic_long_inc(v: &erofs_global_shrink_cnt); |
114 | --grp->lockref.count; |
115 | spin_unlock(lock: &grp->lockref.lock); |
116 | } |
117 | |
118 | static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi, |
119 | struct erofs_workgroup *grp) |
120 | { |
121 | int free = false; |
122 | |
123 | spin_lock(lock: &grp->lockref.lock); |
124 | if (grp->lockref.count) |
125 | goto out; |
126 | |
127 | /* |
128 | * Note that all cached pages should be detached before deleted from |
129 | * the XArray. Otherwise some cached pages could be still attached to |
130 | * the orphan old workgroup when the new one is available in the tree. |
131 | */ |
132 | if (erofs_try_to_free_all_cached_folios(sbi, egrp: grp)) |
133 | goto out; |
134 | |
135 | /* |
136 | * It's impossible to fail after the workgroup is freezed, |
137 | * however in order to avoid some race conditions, add a |
138 | * DBG_BUGON to observe this in advance. |
139 | */ |
140 | DBG_BUGON(__xa_erase(&sbi->managed_pslots, grp->index) != grp); |
141 | |
142 | lockref_mark_dead(&grp->lockref); |
143 | free = true; |
144 | out: |
145 | spin_unlock(lock: &grp->lockref.lock); |
146 | if (free) |
147 | __erofs_workgroup_free(grp); |
148 | return free; |
149 | } |
150 | |
151 | static unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi, |
152 | unsigned long nr_shrink) |
153 | { |
154 | struct erofs_workgroup *grp; |
155 | unsigned int freed = 0; |
156 | unsigned long index; |
157 | |
158 | xa_lock(&sbi->managed_pslots); |
159 | xa_for_each(&sbi->managed_pslots, index, grp) { |
160 | /* try to shrink each valid workgroup */ |
161 | if (!erofs_try_to_release_workgroup(sbi, grp)) |
162 | continue; |
163 | xa_unlock(&sbi->managed_pslots); |
164 | |
165 | ++freed; |
166 | if (!--nr_shrink) |
167 | return freed; |
168 | xa_lock(&sbi->managed_pslots); |
169 | } |
170 | xa_unlock(&sbi->managed_pslots); |
171 | return freed; |
172 | } |
173 | |
174 | /* protected by 'erofs_sb_list_lock' */ |
175 | static unsigned int shrinker_run_no; |
176 | |
177 | /* protects the mounted 'erofs_sb_list' */ |
178 | static DEFINE_SPINLOCK(erofs_sb_list_lock); |
179 | static LIST_HEAD(erofs_sb_list); |
180 | |
181 | void erofs_shrinker_register(struct super_block *sb) |
182 | { |
183 | struct erofs_sb_info *sbi = EROFS_SB(sb); |
184 | |
185 | mutex_init(&sbi->umount_mutex); |
186 | |
187 | spin_lock(lock: &erofs_sb_list_lock); |
188 | list_add(new: &sbi->list, head: &erofs_sb_list); |
189 | spin_unlock(lock: &erofs_sb_list_lock); |
190 | } |
191 | |
192 | void erofs_shrinker_unregister(struct super_block *sb) |
193 | { |
194 | struct erofs_sb_info *const sbi = EROFS_SB(sb); |
195 | |
196 | mutex_lock(&sbi->umount_mutex); |
197 | /* clean up all remaining workgroups in memory */ |
198 | erofs_shrink_workstation(sbi, nr_shrink: ~0UL); |
199 | |
200 | spin_lock(lock: &erofs_sb_list_lock); |
201 | list_del(entry: &sbi->list); |
202 | spin_unlock(lock: &erofs_sb_list_lock); |
203 | mutex_unlock(lock: &sbi->umount_mutex); |
204 | } |
205 | |
206 | static unsigned long erofs_shrink_count(struct shrinker *shrink, |
207 | struct shrink_control *sc) |
208 | { |
209 | return atomic_long_read(v: &erofs_global_shrink_cnt); |
210 | } |
211 | |
212 | static unsigned long erofs_shrink_scan(struct shrinker *shrink, |
213 | struct shrink_control *sc) |
214 | { |
215 | struct erofs_sb_info *sbi; |
216 | struct list_head *p; |
217 | |
218 | unsigned long nr = sc->nr_to_scan; |
219 | unsigned int run_no; |
220 | unsigned long freed = 0; |
221 | |
222 | spin_lock(lock: &erofs_sb_list_lock); |
223 | do { |
224 | run_no = ++shrinker_run_no; |
225 | } while (run_no == 0); |
226 | |
227 | /* Iterate over all mounted superblocks and try to shrink them */ |
228 | p = erofs_sb_list.next; |
229 | while (p != &erofs_sb_list) { |
230 | sbi = list_entry(p, struct erofs_sb_info, list); |
231 | |
232 | /* |
233 | * We move the ones we do to the end of the list, so we stop |
234 | * when we see one we have already done. |
235 | */ |
236 | if (sbi->shrinker_run_no == run_no) |
237 | break; |
238 | |
239 | if (!mutex_trylock(lock: &sbi->umount_mutex)) { |
240 | p = p->next; |
241 | continue; |
242 | } |
243 | |
244 | spin_unlock(lock: &erofs_sb_list_lock); |
245 | sbi->shrinker_run_no = run_no; |
246 | |
247 | freed += erofs_shrink_workstation(sbi, nr_shrink: nr - freed); |
248 | |
249 | spin_lock(lock: &erofs_sb_list_lock); |
250 | /* Get the next list element before we move this one */ |
251 | p = p->next; |
252 | |
253 | /* |
254 | * Move this one to the end of the list to provide some |
255 | * fairness. |
256 | */ |
257 | list_move_tail(list: &sbi->list, head: &erofs_sb_list); |
258 | mutex_unlock(lock: &sbi->umount_mutex); |
259 | |
260 | if (freed >= nr) |
261 | break; |
262 | } |
263 | spin_unlock(lock: &erofs_sb_list_lock); |
264 | return freed; |
265 | } |
266 | |
267 | static struct shrinker *erofs_shrinker_info; |
268 | |
269 | int __init erofs_init_shrinker(void) |
270 | { |
271 | erofs_shrinker_info = shrinker_alloc(flags: 0, fmt: "erofs-shrinker" ); |
272 | if (!erofs_shrinker_info) |
273 | return -ENOMEM; |
274 | |
275 | erofs_shrinker_info->count_objects = erofs_shrink_count; |
276 | erofs_shrinker_info->scan_objects = erofs_shrink_scan; |
277 | |
278 | shrinker_register(shrinker: erofs_shrinker_info); |
279 | |
280 | return 0; |
281 | } |
282 | |
283 | void erofs_exit_shrinker(void) |
284 | { |
285 | shrinker_free(shrinker: erofs_shrinker_info); |
286 | } |
287 | #endif /* !CONFIG_EROFS_FS_ZIP */ |
288 | |