1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * zswap.c - zswap driver file |
4 | * |
5 | * zswap is a cache that takes pages that are in the process |
6 | * of being swapped out and attempts to compress and store them in a |
7 | * RAM-based memory pool. This can result in a significant I/O reduction on |
8 | * the swap device and, in the case where decompressing from RAM is faster |
9 | * than reading from the swap device, can also improve workload performance. |
10 | * |
11 | * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com> |
12 | */ |
13 | |
14 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
15 | |
16 | #include <linux/module.h> |
17 | #include <linux/cpu.h> |
18 | #include <linux/highmem.h> |
19 | #include <linux/slab.h> |
20 | #include <linux/spinlock.h> |
21 | #include <linux/types.h> |
22 | #include <linux/atomic.h> |
23 | #include <linux/rbtree.h> |
24 | #include <linux/swap.h> |
25 | #include <linux/crypto.h> |
26 | #include <linux/scatterlist.h> |
27 | #include <linux/mempolicy.h> |
28 | #include <linux/mempool.h> |
29 | #include <linux/zpool.h> |
30 | #include <crypto/acompress.h> |
31 | #include <linux/zswap.h> |
32 | #include <linux/mm_types.h> |
33 | #include <linux/page-flags.h> |
34 | #include <linux/swapops.h> |
35 | #include <linux/writeback.h> |
36 | #include <linux/pagemap.h> |
37 | #include <linux/workqueue.h> |
38 | |
39 | #include "swap.h" |
40 | #include "internal.h" |
41 | |
42 | /********************************* |
43 | * statistics |
44 | **********************************/ |
45 | /* Total bytes used by the compressed storage */ |
46 | u64 zswap_pool_total_size; |
47 | /* The number of compressed pages currently stored in zswap */ |
48 | atomic_t zswap_stored_pages = ATOMIC_INIT(0); |
49 | /* The number of same-value filled pages currently stored in zswap */ |
50 | static atomic_t zswap_same_filled_pages = ATOMIC_INIT(0); |
51 | |
52 | /* |
53 | * The statistics below are not protected from concurrent access for |
54 | * performance reasons so they may not be a 100% accurate. However, |
55 | * they do provide useful information on roughly how many times a |
56 | * certain event is occurring. |
57 | */ |
58 | |
59 | /* Pool limit was hit (see zswap_max_pool_percent) */ |
60 | static u64 zswap_pool_limit_hit; |
61 | /* Pages written back when pool limit was reached */ |
62 | static u64 zswap_written_back_pages; |
63 | /* Store failed due to a reclaim failure after pool limit was reached */ |
64 | static u64 zswap_reject_reclaim_fail; |
65 | /* Store failed due to compression algorithm failure */ |
66 | static u64 zswap_reject_compress_fail; |
67 | /* Compressed page was too big for the allocator to (optimally) store */ |
68 | static u64 zswap_reject_compress_poor; |
69 | /* Store failed because underlying allocator could not get memory */ |
70 | static u64 zswap_reject_alloc_fail; |
71 | /* Store failed because the entry metadata could not be allocated (rare) */ |
72 | static u64 zswap_reject_kmemcache_fail; |
73 | /* Duplicate store was encountered (rare) */ |
74 | static u64 zswap_duplicate_entry; |
75 | |
76 | /* Shrinker work queue */ |
77 | static struct workqueue_struct *shrink_wq; |
78 | /* Pool limit was hit, we need to calm down */ |
79 | static bool zswap_pool_reached_full; |
80 | |
81 | /********************************* |
82 | * tunables |
83 | **********************************/ |
84 | |
85 | #define ZSWAP_PARAM_UNSET "" |
86 | |
87 | static int zswap_setup(void); |
88 | |
89 | /* Enable/disable zswap */ |
90 | static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON); |
91 | static int zswap_enabled_param_set(const char *, |
92 | const struct kernel_param *); |
93 | static const struct kernel_param_ops zswap_enabled_param_ops = { |
94 | .set = zswap_enabled_param_set, |
95 | .get = param_get_bool, |
96 | }; |
97 | module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644); |
98 | |
99 | /* Crypto compressor to use */ |
100 | static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; |
101 | static int zswap_compressor_param_set(const char *, |
102 | const struct kernel_param *); |
103 | static const struct kernel_param_ops zswap_compressor_param_ops = { |
104 | .set = zswap_compressor_param_set, |
105 | .get = param_get_charp, |
106 | .free = param_free_charp, |
107 | }; |
108 | module_param_cb(compressor, &zswap_compressor_param_ops, |
109 | &zswap_compressor, 0644); |
110 | |
111 | /* Compressed storage zpool to use */ |
112 | static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; |
113 | static int zswap_zpool_param_set(const char *, const struct kernel_param *); |
114 | static const struct kernel_param_ops zswap_zpool_param_ops = { |
115 | .set = zswap_zpool_param_set, |
116 | .get = param_get_charp, |
117 | .free = param_free_charp, |
118 | }; |
119 | module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644); |
120 | |
121 | /* The maximum percentage of memory that the compressed pool can occupy */ |
122 | static unsigned int zswap_max_pool_percent = 20; |
123 | module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644); |
124 | |
125 | /* The threshold for accepting new pages after the max_pool_percent was hit */ |
126 | static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */ |
127 | module_param_named(accept_threshold_percent, zswap_accept_thr_percent, |
128 | uint, 0644); |
129 | |
130 | /* |
131 | * Enable/disable handling same-value filled pages (enabled by default). |
132 | * If disabled every page is considered non-same-value filled. |
133 | */ |
134 | static bool zswap_same_filled_pages_enabled = true; |
135 | module_param_named(same_filled_pages_enabled, zswap_same_filled_pages_enabled, |
136 | bool, 0644); |
137 | |
138 | /* Enable/disable handling non-same-value filled pages (enabled by default) */ |
139 | static bool zswap_non_same_filled_pages_enabled = true; |
140 | module_param_named(non_same_filled_pages_enabled, zswap_non_same_filled_pages_enabled, |
141 | bool, 0644); |
142 | |
143 | static bool zswap_exclusive_loads_enabled = IS_ENABLED( |
144 | CONFIG_ZSWAP_EXCLUSIVE_LOADS_DEFAULT_ON); |
145 | module_param_named(exclusive_loads, zswap_exclusive_loads_enabled, bool, 0644); |
146 | |
147 | /* Number of zpools in zswap_pool (empirically determined for scalability) */ |
148 | #define ZSWAP_NR_ZPOOLS 32 |
149 | |
150 | /********************************* |
151 | * data structures |
152 | **********************************/ |
153 | |
154 | struct crypto_acomp_ctx { |
155 | struct crypto_acomp *acomp; |
156 | struct acomp_req *req; |
157 | struct crypto_wait wait; |
158 | u8 *dstmem; |
159 | struct mutex *mutex; |
160 | }; |
161 | |
162 | /* |
163 | * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock. |
164 | * The only case where lru_lock is not acquired while holding tree.lock is |
165 | * when a zswap_entry is taken off the lru for writeback, in that case it |
166 | * needs to be verified that it's still valid in the tree. |
167 | */ |
168 | struct zswap_pool { |
169 | struct zpool *zpools[ZSWAP_NR_ZPOOLS]; |
170 | struct crypto_acomp_ctx __percpu *acomp_ctx; |
171 | struct kref kref; |
172 | struct list_head list; |
173 | struct work_struct release_work; |
174 | struct work_struct shrink_work; |
175 | struct hlist_node node; |
176 | char tfm_name[CRYPTO_MAX_ALG_NAME]; |
177 | struct list_head lru; |
178 | spinlock_t lru_lock; |
179 | }; |
180 | |
181 | /* |
182 | * struct zswap_entry |
183 | * |
184 | * This structure contains the metadata for tracking a single compressed |
185 | * page within zswap. |
186 | * |
187 | * rbnode - links the entry into red-black tree for the appropriate swap type |
188 | * swpentry - associated swap entry, the offset indexes into the red-black tree |
189 | * refcount - the number of outstanding reference to the entry. This is needed |
190 | * to protect against premature freeing of the entry by code |
191 | * concurrent calls to load, invalidate, and writeback. The lock |
192 | * for the zswap_tree structure that contains the entry must |
193 | * be held while changing the refcount. Since the lock must |
194 | * be held, there is no reason to also make refcount atomic. |
195 | * length - the length in bytes of the compressed page data. Needed during |
196 | * decompression. For a same value filled page length is 0, and both |
197 | * pool and lru are invalid and must be ignored. |
198 | * pool - the zswap_pool the entry's data is in |
199 | * handle - zpool allocation handle that stores the compressed page data |
200 | * value - value of the same-value filled pages which have same content |
201 | * objcg - the obj_cgroup that the compressed memory is charged to |
202 | * lru - handle to the pool's lru used to evict pages. |
203 | */ |
204 | struct zswap_entry { |
205 | struct rb_node rbnode; |
206 | swp_entry_t swpentry; |
207 | int refcount; |
208 | unsigned int length; |
209 | struct zswap_pool *pool; |
210 | union { |
211 | unsigned long handle; |
212 | unsigned long value; |
213 | }; |
214 | struct obj_cgroup *objcg; |
215 | struct list_head lru; |
216 | }; |
217 | |
218 | /* |
219 | * The tree lock in the zswap_tree struct protects a few things: |
220 | * - the rbtree |
221 | * - the refcount field of each entry in the tree |
222 | */ |
223 | struct zswap_tree { |
224 | struct rb_root rbroot; |
225 | spinlock_t lock; |
226 | }; |
227 | |
228 | static struct zswap_tree *zswap_trees[MAX_SWAPFILES]; |
229 | |
230 | /* RCU-protected iteration */ |
231 | static LIST_HEAD(zswap_pools); |
232 | /* protects zswap_pools list modification */ |
233 | static DEFINE_SPINLOCK(zswap_pools_lock); |
234 | /* pool counter to provide unique names to zpool */ |
235 | static atomic_t zswap_pools_count = ATOMIC_INIT(0); |
236 | |
237 | enum zswap_init_type { |
238 | ZSWAP_UNINIT, |
239 | ZSWAP_INIT_SUCCEED, |
240 | ZSWAP_INIT_FAILED |
241 | }; |
242 | |
243 | static enum zswap_init_type zswap_init_state; |
244 | |
245 | /* used to ensure the integrity of initialization */ |
246 | static DEFINE_MUTEX(zswap_init_lock); |
247 | |
248 | /* init completed, but couldn't create the initial pool */ |
249 | static bool zswap_has_pool; |
250 | |
251 | /********************************* |
252 | * helpers and fwd declarations |
253 | **********************************/ |
254 | |
255 | #define zswap_pool_debug(msg, p) \ |
256 | pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \ |
257 | zpool_get_type((p)->zpools[0])) |
258 | |
259 | static int zswap_writeback_entry(struct zswap_entry *entry, |
260 | struct zswap_tree *tree); |
261 | static int zswap_pool_get(struct zswap_pool *pool); |
262 | static void zswap_pool_put(struct zswap_pool *pool); |
263 | |
264 | static bool zswap_is_full(void) |
265 | { |
266 | return totalram_pages() * zswap_max_pool_percent / 100 < |
267 | DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); |
268 | } |
269 | |
270 | static bool zswap_can_accept(void) |
271 | { |
272 | return totalram_pages() * zswap_accept_thr_percent / 100 * |
273 | zswap_max_pool_percent / 100 > |
274 | DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); |
275 | } |
276 | |
277 | static void zswap_update_total_size(void) |
278 | { |
279 | struct zswap_pool *pool; |
280 | u64 total = 0; |
281 | int i; |
282 | |
283 | rcu_read_lock(); |
284 | |
285 | list_for_each_entry_rcu(pool, &zswap_pools, list) |
286 | for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) |
287 | total += zpool_get_total_size(pool: pool->zpools[i]); |
288 | |
289 | rcu_read_unlock(); |
290 | |
291 | zswap_pool_total_size = total; |
292 | } |
293 | |
294 | /********************************* |
295 | * zswap entry functions |
296 | **********************************/ |
297 | static struct kmem_cache *zswap_entry_cache; |
298 | |
299 | static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp) |
300 | { |
301 | struct zswap_entry *entry; |
302 | entry = kmem_cache_alloc(cachep: zswap_entry_cache, flags: gfp); |
303 | if (!entry) |
304 | return NULL; |
305 | entry->refcount = 1; |
306 | RB_CLEAR_NODE(&entry->rbnode); |
307 | return entry; |
308 | } |
309 | |
310 | static void zswap_entry_cache_free(struct zswap_entry *entry) |
311 | { |
312 | kmem_cache_free(s: zswap_entry_cache, objp: entry); |
313 | } |
314 | |
315 | /********************************* |
316 | * rbtree functions |
317 | **********************************/ |
318 | static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset) |
319 | { |
320 | struct rb_node *node = root->rb_node; |
321 | struct zswap_entry *entry; |
322 | pgoff_t entry_offset; |
323 | |
324 | while (node) { |
325 | entry = rb_entry(node, struct zswap_entry, rbnode); |
326 | entry_offset = swp_offset(entry: entry->swpentry); |
327 | if (entry_offset > offset) |
328 | node = node->rb_left; |
329 | else if (entry_offset < offset) |
330 | node = node->rb_right; |
331 | else |
332 | return entry; |
333 | } |
334 | return NULL; |
335 | } |
336 | |
337 | /* |
338 | * In the case that a entry with the same offset is found, a pointer to |
339 | * the existing entry is stored in dupentry and the function returns -EEXIST |
340 | */ |
341 | static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry, |
342 | struct zswap_entry **dupentry) |
343 | { |
344 | struct rb_node **link = &root->rb_node, *parent = NULL; |
345 | struct zswap_entry *myentry; |
346 | pgoff_t myentry_offset, entry_offset = swp_offset(entry: entry->swpentry); |
347 | |
348 | while (*link) { |
349 | parent = *link; |
350 | myentry = rb_entry(parent, struct zswap_entry, rbnode); |
351 | myentry_offset = swp_offset(entry: myentry->swpentry); |
352 | if (myentry_offset > entry_offset) |
353 | link = &(*link)->rb_left; |
354 | else if (myentry_offset < entry_offset) |
355 | link = &(*link)->rb_right; |
356 | else { |
357 | *dupentry = myentry; |
358 | return -EEXIST; |
359 | } |
360 | } |
361 | rb_link_node(node: &entry->rbnode, parent, rb_link: link); |
362 | rb_insert_color(&entry->rbnode, root); |
363 | return 0; |
364 | } |
365 | |
366 | static bool zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry) |
367 | { |
368 | if (!RB_EMPTY_NODE(&entry->rbnode)) { |
369 | rb_erase(&entry->rbnode, root); |
370 | RB_CLEAR_NODE(&entry->rbnode); |
371 | return true; |
372 | } |
373 | return false; |
374 | } |
375 | |
376 | static struct zpool *zswap_find_zpool(struct zswap_entry *entry) |
377 | { |
378 | int i = 0; |
379 | |
380 | if (ZSWAP_NR_ZPOOLS > 1) |
381 | i = hash_ptr(ptr: entry, ilog2(ZSWAP_NR_ZPOOLS)); |
382 | |
383 | return entry->pool->zpools[i]; |
384 | } |
385 | |
386 | /* |
387 | * Carries out the common pattern of freeing and entry's zpool allocation, |
388 | * freeing the entry itself, and decrementing the number of stored pages. |
389 | */ |
390 | static void zswap_free_entry(struct zswap_entry *entry) |
391 | { |
392 | if (entry->objcg) { |
393 | obj_cgroup_uncharge_zswap(objcg: entry->objcg, size: entry->length); |
394 | obj_cgroup_put(objcg: entry->objcg); |
395 | } |
396 | if (!entry->length) |
397 | atomic_dec(v: &zswap_same_filled_pages); |
398 | else { |
399 | spin_lock(lock: &entry->pool->lru_lock); |
400 | list_del(entry: &entry->lru); |
401 | spin_unlock(lock: &entry->pool->lru_lock); |
402 | zpool_free(pool: zswap_find_zpool(entry), handle: entry->handle); |
403 | zswap_pool_put(pool: entry->pool); |
404 | } |
405 | zswap_entry_cache_free(entry); |
406 | atomic_dec(v: &zswap_stored_pages); |
407 | zswap_update_total_size(); |
408 | } |
409 | |
410 | /* caller must hold the tree lock */ |
411 | static void zswap_entry_get(struct zswap_entry *entry) |
412 | { |
413 | entry->refcount++; |
414 | } |
415 | |
416 | /* caller must hold the tree lock |
417 | * remove from the tree and free it, if nobody reference the entry |
418 | */ |
419 | static void zswap_entry_put(struct zswap_tree *tree, |
420 | struct zswap_entry *entry) |
421 | { |
422 | int refcount = --entry->refcount; |
423 | |
424 | WARN_ON_ONCE(refcount < 0); |
425 | if (refcount == 0) { |
426 | WARN_ON_ONCE(!RB_EMPTY_NODE(&entry->rbnode)); |
427 | zswap_free_entry(entry); |
428 | } |
429 | } |
430 | |
431 | /* caller must hold the tree lock */ |
432 | static struct zswap_entry *zswap_entry_find_get(struct rb_root *root, |
433 | pgoff_t offset) |
434 | { |
435 | struct zswap_entry *entry; |
436 | |
437 | entry = zswap_rb_search(root, offset); |
438 | if (entry) |
439 | zswap_entry_get(entry); |
440 | |
441 | return entry; |
442 | } |
443 | |
444 | /********************************* |
445 | * per-cpu code |
446 | **********************************/ |
447 | static DEFINE_PER_CPU(u8 *, zswap_dstmem); |
448 | /* |
449 | * If users dynamically change the zpool type and compressor at runtime, i.e. |
450 | * zswap is running, zswap can have more than one zpool on one cpu, but they |
451 | * are sharing dtsmem. So we need this mutex to be per-cpu. |
452 | */ |
453 | static DEFINE_PER_CPU(struct mutex *, zswap_mutex); |
454 | |
455 | static int zswap_dstmem_prepare(unsigned int cpu) |
456 | { |
457 | struct mutex *mutex; |
458 | u8 *dst; |
459 | |
460 | dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); |
461 | if (!dst) |
462 | return -ENOMEM; |
463 | |
464 | mutex = kmalloc_node(size: sizeof(*mutex), GFP_KERNEL, cpu_to_node(cpu)); |
465 | if (!mutex) { |
466 | kfree(objp: dst); |
467 | return -ENOMEM; |
468 | } |
469 | |
470 | mutex_init(mutex); |
471 | per_cpu(zswap_dstmem, cpu) = dst; |
472 | per_cpu(zswap_mutex, cpu) = mutex; |
473 | return 0; |
474 | } |
475 | |
476 | static int zswap_dstmem_dead(unsigned int cpu) |
477 | { |
478 | struct mutex *mutex; |
479 | u8 *dst; |
480 | |
481 | mutex = per_cpu(zswap_mutex, cpu); |
482 | kfree(objp: mutex); |
483 | per_cpu(zswap_mutex, cpu) = NULL; |
484 | |
485 | dst = per_cpu(zswap_dstmem, cpu); |
486 | kfree(objp: dst); |
487 | per_cpu(zswap_dstmem, cpu) = NULL; |
488 | |
489 | return 0; |
490 | } |
491 | |
492 | static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) |
493 | { |
494 | struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); |
495 | struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); |
496 | struct crypto_acomp *acomp; |
497 | struct acomp_req *req; |
498 | |
499 | acomp = crypto_alloc_acomp_node(alg_name: pool->tfm_name, type: 0, mask: 0, cpu_to_node(cpu)); |
500 | if (IS_ERR(ptr: acomp)) { |
501 | pr_err("could not alloc crypto acomp %s : %ld\n" , |
502 | pool->tfm_name, PTR_ERR(acomp)); |
503 | return PTR_ERR(ptr: acomp); |
504 | } |
505 | acomp_ctx->acomp = acomp; |
506 | |
507 | req = acomp_request_alloc(tfm: acomp_ctx->acomp); |
508 | if (!req) { |
509 | pr_err("could not alloc crypto acomp_request %s\n" , |
510 | pool->tfm_name); |
511 | crypto_free_acomp(tfm: acomp_ctx->acomp); |
512 | return -ENOMEM; |
513 | } |
514 | acomp_ctx->req = req; |
515 | |
516 | crypto_init_wait(wait: &acomp_ctx->wait); |
517 | /* |
518 | * if the backend of acomp is async zip, crypto_req_done() will wakeup |
519 | * crypto_wait_req(); if the backend of acomp is scomp, the callback |
520 | * won't be called, crypto_wait_req() will return without blocking. |
521 | */ |
522 | acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, |
523 | cmpl: crypto_req_done, data: &acomp_ctx->wait); |
524 | |
525 | acomp_ctx->mutex = per_cpu(zswap_mutex, cpu); |
526 | acomp_ctx->dstmem = per_cpu(zswap_dstmem, cpu); |
527 | |
528 | return 0; |
529 | } |
530 | |
531 | static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) |
532 | { |
533 | struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); |
534 | struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu); |
535 | |
536 | if (!IS_ERR_OR_NULL(ptr: acomp_ctx)) { |
537 | if (!IS_ERR_OR_NULL(ptr: acomp_ctx->req)) |
538 | acomp_request_free(req: acomp_ctx->req); |
539 | if (!IS_ERR_OR_NULL(ptr: acomp_ctx->acomp)) |
540 | crypto_free_acomp(tfm: acomp_ctx->acomp); |
541 | } |
542 | |
543 | return 0; |
544 | } |
545 | |
546 | /********************************* |
547 | * pool functions |
548 | **********************************/ |
549 | |
550 | static struct zswap_pool *__zswap_pool_current(void) |
551 | { |
552 | struct zswap_pool *pool; |
553 | |
554 | pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list); |
555 | WARN_ONCE(!pool && zswap_has_pool, |
556 | "%s: no page storage pool!\n" , __func__); |
557 | |
558 | return pool; |
559 | } |
560 | |
561 | static struct zswap_pool *zswap_pool_current(void) |
562 | { |
563 | assert_spin_locked(&zswap_pools_lock); |
564 | |
565 | return __zswap_pool_current(); |
566 | } |
567 | |
568 | static struct zswap_pool *zswap_pool_current_get(void) |
569 | { |
570 | struct zswap_pool *pool; |
571 | |
572 | rcu_read_lock(); |
573 | |
574 | pool = __zswap_pool_current(); |
575 | if (!zswap_pool_get(pool)) |
576 | pool = NULL; |
577 | |
578 | rcu_read_unlock(); |
579 | |
580 | return pool; |
581 | } |
582 | |
583 | static struct zswap_pool *zswap_pool_last_get(void) |
584 | { |
585 | struct zswap_pool *pool, *last = NULL; |
586 | |
587 | rcu_read_lock(); |
588 | |
589 | list_for_each_entry_rcu(pool, &zswap_pools, list) |
590 | last = pool; |
591 | WARN_ONCE(!last && zswap_has_pool, |
592 | "%s: no page storage pool!\n" , __func__); |
593 | if (!zswap_pool_get(pool: last)) |
594 | last = NULL; |
595 | |
596 | rcu_read_unlock(); |
597 | |
598 | return last; |
599 | } |
600 | |
601 | /* type and compressor must be null-terminated */ |
602 | static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor) |
603 | { |
604 | struct zswap_pool *pool; |
605 | |
606 | assert_spin_locked(&zswap_pools_lock); |
607 | |
608 | list_for_each_entry_rcu(pool, &zswap_pools, list) { |
609 | if (strcmp(pool->tfm_name, compressor)) |
610 | continue; |
611 | /* all zpools share the same type */ |
612 | if (strcmp(zpool_get_type(pool: pool->zpools[0]), type)) |
613 | continue; |
614 | /* if we can't get it, it's about to be destroyed */ |
615 | if (!zswap_pool_get(pool)) |
616 | continue; |
617 | return pool; |
618 | } |
619 | |
620 | return NULL; |
621 | } |
622 | |
623 | /* |
624 | * If the entry is still valid in the tree, drop the initial ref and remove it |
625 | * from the tree. This function must be called with an additional ref held, |
626 | * otherwise it may race with another invalidation freeing the entry. |
627 | */ |
628 | static void zswap_invalidate_entry(struct zswap_tree *tree, |
629 | struct zswap_entry *entry) |
630 | { |
631 | if (zswap_rb_erase(root: &tree->rbroot, entry)) |
632 | zswap_entry_put(tree, entry); |
633 | } |
634 | |
635 | static int zswap_reclaim_entry(struct zswap_pool *pool) |
636 | { |
637 | struct zswap_entry *entry; |
638 | struct zswap_tree *tree; |
639 | pgoff_t swpoffset; |
640 | int ret; |
641 | |
642 | /* Get an entry off the LRU */ |
643 | spin_lock(lock: &pool->lru_lock); |
644 | if (list_empty(head: &pool->lru)) { |
645 | spin_unlock(lock: &pool->lru_lock); |
646 | return -EINVAL; |
647 | } |
648 | entry = list_last_entry(&pool->lru, struct zswap_entry, lru); |
649 | list_del_init(entry: &entry->lru); |
650 | /* |
651 | * Once the lru lock is dropped, the entry might get freed. The |
652 | * swpoffset is copied to the stack, and entry isn't deref'd again |
653 | * until the entry is verified to still be alive in the tree. |
654 | */ |
655 | swpoffset = swp_offset(entry: entry->swpentry); |
656 | tree = zswap_trees[swp_type(entry: entry->swpentry)]; |
657 | spin_unlock(lock: &pool->lru_lock); |
658 | |
659 | /* Check for invalidate() race */ |
660 | spin_lock(lock: &tree->lock); |
661 | if (entry != zswap_rb_search(root: &tree->rbroot, offset: swpoffset)) { |
662 | ret = -EAGAIN; |
663 | goto unlock; |
664 | } |
665 | /* Hold a reference to prevent a free during writeback */ |
666 | zswap_entry_get(entry); |
667 | spin_unlock(lock: &tree->lock); |
668 | |
669 | ret = zswap_writeback_entry(entry, tree); |
670 | |
671 | spin_lock(lock: &tree->lock); |
672 | if (ret) { |
673 | /* Writeback failed, put entry back on LRU */ |
674 | spin_lock(lock: &pool->lru_lock); |
675 | list_move(list: &entry->lru, head: &pool->lru); |
676 | spin_unlock(lock: &pool->lru_lock); |
677 | goto put_unlock; |
678 | } |
679 | |
680 | /* |
681 | * Writeback started successfully, the page now belongs to the |
682 | * swapcache. Drop the entry from zswap - unless invalidate already |
683 | * took it out while we had the tree->lock released for IO. |
684 | */ |
685 | zswap_invalidate_entry(tree, entry); |
686 | |
687 | put_unlock: |
688 | /* Drop local reference */ |
689 | zswap_entry_put(tree, entry); |
690 | unlock: |
691 | spin_unlock(lock: &tree->lock); |
692 | return ret ? -EAGAIN : 0; |
693 | } |
694 | |
695 | static void shrink_worker(struct work_struct *w) |
696 | { |
697 | struct zswap_pool *pool = container_of(w, typeof(*pool), |
698 | shrink_work); |
699 | int ret, failures = 0; |
700 | |
701 | do { |
702 | ret = zswap_reclaim_entry(pool); |
703 | if (ret) { |
704 | zswap_reject_reclaim_fail++; |
705 | if (ret != -EAGAIN) |
706 | break; |
707 | if (++failures == MAX_RECLAIM_RETRIES) |
708 | break; |
709 | } |
710 | cond_resched(); |
711 | } while (!zswap_can_accept()); |
712 | zswap_pool_put(pool); |
713 | } |
714 | |
715 | static struct zswap_pool *zswap_pool_create(char *type, char *compressor) |
716 | { |
717 | int i; |
718 | struct zswap_pool *pool; |
719 | char name[38]; /* 'zswap' + 32 char (max) num + \0 */ |
720 | gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; |
721 | int ret; |
722 | |
723 | if (!zswap_has_pool) { |
724 | /* if either are unset, pool initialization failed, and we |
725 | * need both params to be set correctly before trying to |
726 | * create a pool. |
727 | */ |
728 | if (!strcmp(type, ZSWAP_PARAM_UNSET)) |
729 | return NULL; |
730 | if (!strcmp(compressor, ZSWAP_PARAM_UNSET)) |
731 | return NULL; |
732 | } |
733 | |
734 | pool = kzalloc(size: sizeof(*pool), GFP_KERNEL); |
735 | if (!pool) |
736 | return NULL; |
737 | |
738 | for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) { |
739 | /* unique name for each pool specifically required by zsmalloc */ |
740 | snprintf(buf: name, size: 38, fmt: "zswap%x" , |
741 | atomic_inc_return(v: &zswap_pools_count)); |
742 | |
743 | pool->zpools[i] = zpool_create_pool(type, name, gfp); |
744 | if (!pool->zpools[i]) { |
745 | pr_err("%s zpool not available\n" , type); |
746 | goto error; |
747 | } |
748 | } |
749 | pr_debug("using %s zpool\n" , zpool_get_type(pool->zpools[0])); |
750 | |
751 | strscpy(p: pool->tfm_name, q: compressor, size: sizeof(pool->tfm_name)); |
752 | |
753 | pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx); |
754 | if (!pool->acomp_ctx) { |
755 | pr_err("percpu alloc failed\n" ); |
756 | goto error; |
757 | } |
758 | |
759 | ret = cpuhp_state_add_instance(state: CPUHP_MM_ZSWP_POOL_PREPARE, |
760 | node: &pool->node); |
761 | if (ret) |
762 | goto error; |
763 | pr_debug("using %s compressor\n" , pool->tfm_name); |
764 | |
765 | /* being the current pool takes 1 ref; this func expects the |
766 | * caller to always add the new pool as the current pool |
767 | */ |
768 | kref_init(kref: &pool->kref); |
769 | INIT_LIST_HEAD(list: &pool->list); |
770 | INIT_LIST_HEAD(list: &pool->lru); |
771 | spin_lock_init(&pool->lru_lock); |
772 | INIT_WORK(&pool->shrink_work, shrink_worker); |
773 | |
774 | zswap_pool_debug("created" , pool); |
775 | |
776 | return pool; |
777 | |
778 | error: |
779 | if (pool->acomp_ctx) |
780 | free_percpu(pdata: pool->acomp_ctx); |
781 | while (i--) |
782 | zpool_destroy_pool(pool: pool->zpools[i]); |
783 | kfree(objp: pool); |
784 | return NULL; |
785 | } |
786 | |
787 | static struct zswap_pool *__zswap_pool_create_fallback(void) |
788 | { |
789 | bool has_comp, has_zpool; |
790 | |
791 | has_comp = crypto_has_acomp(alg_name: zswap_compressor, type: 0, mask: 0); |
792 | if (!has_comp && strcmp(zswap_compressor, |
793 | CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) { |
794 | pr_err("compressor %s not available, using default %s\n" , |
795 | zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT); |
796 | param_free_charp(arg: &zswap_compressor); |
797 | zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT; |
798 | has_comp = crypto_has_acomp(alg_name: zswap_compressor, type: 0, mask: 0); |
799 | } |
800 | if (!has_comp) { |
801 | pr_err("default compressor %s not available\n" , |
802 | zswap_compressor); |
803 | param_free_charp(arg: &zswap_compressor); |
804 | zswap_compressor = ZSWAP_PARAM_UNSET; |
805 | } |
806 | |
807 | has_zpool = zpool_has_pool(type: zswap_zpool_type); |
808 | if (!has_zpool && strcmp(zswap_zpool_type, |
809 | CONFIG_ZSWAP_ZPOOL_DEFAULT)) { |
810 | pr_err("zpool %s not available, using default %s\n" , |
811 | zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT); |
812 | param_free_charp(arg: &zswap_zpool_type); |
813 | zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT; |
814 | has_zpool = zpool_has_pool(type: zswap_zpool_type); |
815 | } |
816 | if (!has_zpool) { |
817 | pr_err("default zpool %s not available\n" , |
818 | zswap_zpool_type); |
819 | param_free_charp(arg: &zswap_zpool_type); |
820 | zswap_zpool_type = ZSWAP_PARAM_UNSET; |
821 | } |
822 | |
823 | if (!has_comp || !has_zpool) |
824 | return NULL; |
825 | |
826 | return zswap_pool_create(type: zswap_zpool_type, compressor: zswap_compressor); |
827 | } |
828 | |
829 | static void zswap_pool_destroy(struct zswap_pool *pool) |
830 | { |
831 | int i; |
832 | |
833 | zswap_pool_debug("destroying" , pool); |
834 | |
835 | cpuhp_state_remove_instance(state: CPUHP_MM_ZSWP_POOL_PREPARE, node: &pool->node); |
836 | free_percpu(pdata: pool->acomp_ctx); |
837 | for (i = 0; i < ZSWAP_NR_ZPOOLS; i++) |
838 | zpool_destroy_pool(pool: pool->zpools[i]); |
839 | kfree(objp: pool); |
840 | } |
841 | |
842 | static int __must_check zswap_pool_get(struct zswap_pool *pool) |
843 | { |
844 | if (!pool) |
845 | return 0; |
846 | |
847 | return kref_get_unless_zero(kref: &pool->kref); |
848 | } |
849 | |
850 | static void __zswap_pool_release(struct work_struct *work) |
851 | { |
852 | struct zswap_pool *pool = container_of(work, typeof(*pool), |
853 | release_work); |
854 | |
855 | synchronize_rcu(); |
856 | |
857 | /* nobody should have been able to get a kref... */ |
858 | WARN_ON(kref_get_unless_zero(&pool->kref)); |
859 | |
860 | /* pool is now off zswap_pools list and has no references. */ |
861 | zswap_pool_destroy(pool); |
862 | } |
863 | |
864 | static void __zswap_pool_empty(struct kref *kref) |
865 | { |
866 | struct zswap_pool *pool; |
867 | |
868 | pool = container_of(kref, typeof(*pool), kref); |
869 | |
870 | spin_lock(lock: &zswap_pools_lock); |
871 | |
872 | WARN_ON(pool == zswap_pool_current()); |
873 | |
874 | list_del_rcu(entry: &pool->list); |
875 | |
876 | INIT_WORK(&pool->release_work, __zswap_pool_release); |
877 | schedule_work(work: &pool->release_work); |
878 | |
879 | spin_unlock(lock: &zswap_pools_lock); |
880 | } |
881 | |
882 | static void zswap_pool_put(struct zswap_pool *pool) |
883 | { |
884 | kref_put(kref: &pool->kref, release: __zswap_pool_empty); |
885 | } |
886 | |
887 | /********************************* |
888 | * param callbacks |
889 | **********************************/ |
890 | |
891 | static bool zswap_pool_changed(const char *s, const struct kernel_param *kp) |
892 | { |
893 | /* no change required */ |
894 | if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool) |
895 | return false; |
896 | return true; |
897 | } |
898 | |
899 | /* val must be a null-terminated string */ |
900 | static int __zswap_param_set(const char *val, const struct kernel_param *kp, |
901 | char *type, char *compressor) |
902 | { |
903 | struct zswap_pool *pool, *put_pool = NULL; |
904 | char *s = strstrip(str: (char *)val); |
905 | int ret = 0; |
906 | bool new_pool = false; |
907 | |
908 | mutex_lock(&zswap_init_lock); |
909 | switch (zswap_init_state) { |
910 | case ZSWAP_UNINIT: |
911 | /* if this is load-time (pre-init) param setting, |
912 | * don't create a pool; that's done during init. |
913 | */ |
914 | ret = param_set_charp(val: s, kp); |
915 | break; |
916 | case ZSWAP_INIT_SUCCEED: |
917 | new_pool = zswap_pool_changed(s, kp); |
918 | break; |
919 | case ZSWAP_INIT_FAILED: |
920 | pr_err("can't set param, initialization failed\n" ); |
921 | ret = -ENODEV; |
922 | } |
923 | mutex_unlock(lock: &zswap_init_lock); |
924 | |
925 | /* no need to create a new pool, return directly */ |
926 | if (!new_pool) |
927 | return ret; |
928 | |
929 | if (!type) { |
930 | if (!zpool_has_pool(type: s)) { |
931 | pr_err("zpool %s not available\n" , s); |
932 | return -ENOENT; |
933 | } |
934 | type = s; |
935 | } else if (!compressor) { |
936 | if (!crypto_has_acomp(alg_name: s, type: 0, mask: 0)) { |
937 | pr_err("compressor %s not available\n" , s); |
938 | return -ENOENT; |
939 | } |
940 | compressor = s; |
941 | } else { |
942 | WARN_ON(1); |
943 | return -EINVAL; |
944 | } |
945 | |
946 | spin_lock(lock: &zswap_pools_lock); |
947 | |
948 | pool = zswap_pool_find_get(type, compressor); |
949 | if (pool) { |
950 | zswap_pool_debug("using existing" , pool); |
951 | WARN_ON(pool == zswap_pool_current()); |
952 | list_del_rcu(entry: &pool->list); |
953 | } |
954 | |
955 | spin_unlock(lock: &zswap_pools_lock); |
956 | |
957 | if (!pool) |
958 | pool = zswap_pool_create(type, compressor); |
959 | |
960 | if (pool) |
961 | ret = param_set_charp(val: s, kp); |
962 | else |
963 | ret = -EINVAL; |
964 | |
965 | spin_lock(lock: &zswap_pools_lock); |
966 | |
967 | if (!ret) { |
968 | put_pool = zswap_pool_current(); |
969 | list_add_rcu(new: &pool->list, head: &zswap_pools); |
970 | zswap_has_pool = true; |
971 | } else if (pool) { |
972 | /* add the possibly pre-existing pool to the end of the pools |
973 | * list; if it's new (and empty) then it'll be removed and |
974 | * destroyed by the put after we drop the lock |
975 | */ |
976 | list_add_tail_rcu(new: &pool->list, head: &zswap_pools); |
977 | put_pool = pool; |
978 | } |
979 | |
980 | spin_unlock(lock: &zswap_pools_lock); |
981 | |
982 | if (!zswap_has_pool && !pool) { |
983 | /* if initial pool creation failed, and this pool creation also |
984 | * failed, maybe both compressor and zpool params were bad. |
985 | * Allow changing this param, so pool creation will succeed |
986 | * when the other param is changed. We already verified this |
987 | * param is ok in the zpool_has_pool() or crypto_has_acomp() |
988 | * checks above. |
989 | */ |
990 | ret = param_set_charp(val: s, kp); |
991 | } |
992 | |
993 | /* drop the ref from either the old current pool, |
994 | * or the new pool we failed to add |
995 | */ |
996 | if (put_pool) |
997 | zswap_pool_put(pool: put_pool); |
998 | |
999 | return ret; |
1000 | } |
1001 | |
1002 | static int zswap_compressor_param_set(const char *val, |
1003 | const struct kernel_param *kp) |
1004 | { |
1005 | return __zswap_param_set(val, kp, type: zswap_zpool_type, NULL); |
1006 | } |
1007 | |
1008 | static int zswap_zpool_param_set(const char *val, |
1009 | const struct kernel_param *kp) |
1010 | { |
1011 | return __zswap_param_set(val, kp, NULL, compressor: zswap_compressor); |
1012 | } |
1013 | |
1014 | static int zswap_enabled_param_set(const char *val, |
1015 | const struct kernel_param *kp) |
1016 | { |
1017 | int ret = -ENODEV; |
1018 | |
1019 | /* if this is load-time (pre-init) param setting, only set param. */ |
1020 | if (system_state != SYSTEM_RUNNING) |
1021 | return param_set_bool(val, kp); |
1022 | |
1023 | mutex_lock(&zswap_init_lock); |
1024 | switch (zswap_init_state) { |
1025 | case ZSWAP_UNINIT: |
1026 | if (zswap_setup()) |
1027 | break; |
1028 | fallthrough; |
1029 | case ZSWAP_INIT_SUCCEED: |
1030 | if (!zswap_has_pool) |
1031 | pr_err("can't enable, no pool configured\n" ); |
1032 | else |
1033 | ret = param_set_bool(val, kp); |
1034 | break; |
1035 | case ZSWAP_INIT_FAILED: |
1036 | pr_err("can't enable, initialization failed\n" ); |
1037 | } |
1038 | mutex_unlock(lock: &zswap_init_lock); |
1039 | |
1040 | return ret; |
1041 | } |
1042 | |
1043 | /********************************* |
1044 | * writeback code |
1045 | **********************************/ |
1046 | /* |
1047 | * Attempts to free an entry by adding a page to the swap cache, |
1048 | * decompressing the entry data into the page, and issuing a |
1049 | * bio write to write the page back to the swap device. |
1050 | * |
1051 | * This can be thought of as a "resumed writeback" of the page |
1052 | * to the swap device. We are basically resuming the same swap |
1053 | * writeback path that was intercepted with the zswap_store() |
1054 | * in the first place. After the page has been decompressed into |
1055 | * the swap cache, the compressed version stored by zswap can be |
1056 | * freed. |
1057 | */ |
1058 | static int zswap_writeback_entry(struct zswap_entry *entry, |
1059 | struct zswap_tree *tree) |
1060 | { |
1061 | swp_entry_t swpentry = entry->swpentry; |
1062 | struct page *page; |
1063 | struct mempolicy *mpol; |
1064 | struct scatterlist input, output; |
1065 | struct crypto_acomp_ctx *acomp_ctx; |
1066 | struct zpool *pool = zswap_find_zpool(entry); |
1067 | bool page_was_allocated; |
1068 | u8 *src, *tmp = NULL; |
1069 | unsigned int dlen; |
1070 | int ret; |
1071 | struct writeback_control wbc = { |
1072 | .sync_mode = WB_SYNC_NONE, |
1073 | }; |
1074 | |
1075 | if (!zpool_can_sleep_mapped(pool)) { |
1076 | tmp = kmalloc(PAGE_SIZE, GFP_KERNEL); |
1077 | if (!tmp) |
1078 | return -ENOMEM; |
1079 | } |
1080 | |
1081 | /* try to allocate swap cache page */ |
1082 | mpol = get_task_policy(current); |
1083 | page = __read_swap_cache_async(entry: swpentry, GFP_KERNEL, mpol, |
1084 | NO_INTERLEAVE_INDEX, new_page_allocated: &page_was_allocated); |
1085 | if (!page) { |
1086 | ret = -ENOMEM; |
1087 | goto fail; |
1088 | } |
1089 | |
1090 | /* Found an existing page, we raced with load/swapin */ |
1091 | if (!page_was_allocated) { |
1092 | put_page(page); |
1093 | ret = -EEXIST; |
1094 | goto fail; |
1095 | } |
1096 | |
1097 | /* |
1098 | * Page is locked, and the swapcache is now secured against |
1099 | * concurrent swapping to and from the slot. Verify that the |
1100 | * swap entry hasn't been invalidated and recycled behind our |
1101 | * backs (our zswap_entry reference doesn't prevent that), to |
1102 | * avoid overwriting a new swap page with old compressed data. |
1103 | */ |
1104 | spin_lock(lock: &tree->lock); |
1105 | if (zswap_rb_search(root: &tree->rbroot, offset: swp_offset(entry: entry->swpentry)) != entry) { |
1106 | spin_unlock(lock: &tree->lock); |
1107 | delete_from_swap_cache(page_folio(page)); |
1108 | ret = -ENOMEM; |
1109 | goto fail; |
1110 | } |
1111 | spin_unlock(lock: &tree->lock); |
1112 | |
1113 | /* decompress */ |
1114 | acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); |
1115 | dlen = PAGE_SIZE; |
1116 | |
1117 | src = zpool_map_handle(pool, handle: entry->handle, mm: ZPOOL_MM_RO); |
1118 | if (!zpool_can_sleep_mapped(pool)) { |
1119 | memcpy(tmp, src, entry->length); |
1120 | src = tmp; |
1121 | zpool_unmap_handle(pool, handle: entry->handle); |
1122 | } |
1123 | |
1124 | mutex_lock(acomp_ctx->mutex); |
1125 | sg_init_one(&input, src, entry->length); |
1126 | sg_init_table(&output, 1); |
1127 | sg_set_page(sg: &output, page, PAGE_SIZE, offset: 0); |
1128 | acomp_request_set_params(req: acomp_ctx->req, src: &input, dst: &output, slen: entry->length, dlen); |
1129 | ret = crypto_wait_req(err: crypto_acomp_decompress(req: acomp_ctx->req), wait: &acomp_ctx->wait); |
1130 | dlen = acomp_ctx->req->dlen; |
1131 | mutex_unlock(lock: acomp_ctx->mutex); |
1132 | |
1133 | if (!zpool_can_sleep_mapped(pool)) |
1134 | kfree(objp: tmp); |
1135 | else |
1136 | zpool_unmap_handle(pool, handle: entry->handle); |
1137 | |
1138 | BUG_ON(ret); |
1139 | BUG_ON(dlen != PAGE_SIZE); |
1140 | |
1141 | /* page is up to date */ |
1142 | SetPageUptodate(page); |
1143 | |
1144 | /* move it to the tail of the inactive list after end_writeback */ |
1145 | SetPageReclaim(page); |
1146 | |
1147 | /* start writeback */ |
1148 | __swap_writepage(page, wbc: &wbc); |
1149 | put_page(page); |
1150 | zswap_written_back_pages++; |
1151 | |
1152 | return ret; |
1153 | |
1154 | fail: |
1155 | if (!zpool_can_sleep_mapped(pool)) |
1156 | kfree(objp: tmp); |
1157 | |
1158 | /* |
1159 | * If we get here because the page is already in swapcache, a |
1160 | * load may be happening concurrently. It is safe and okay to |
1161 | * not free the entry. It is also okay to return !0. |
1162 | */ |
1163 | return ret; |
1164 | } |
1165 | |
1166 | static int zswap_is_page_same_filled(void *ptr, unsigned long *value) |
1167 | { |
1168 | unsigned long *page; |
1169 | unsigned long val; |
1170 | unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1; |
1171 | |
1172 | page = (unsigned long *)ptr; |
1173 | val = page[0]; |
1174 | |
1175 | if (val != page[last_pos]) |
1176 | return 0; |
1177 | |
1178 | for (pos = 1; pos < last_pos; pos++) { |
1179 | if (val != page[pos]) |
1180 | return 0; |
1181 | } |
1182 | |
1183 | *value = val; |
1184 | |
1185 | return 1; |
1186 | } |
1187 | |
1188 | static void zswap_fill_page(void *ptr, unsigned long value) |
1189 | { |
1190 | unsigned long *page; |
1191 | |
1192 | page = (unsigned long *)ptr; |
1193 | memset_l(p: page, v: value, PAGE_SIZE / sizeof(unsigned long)); |
1194 | } |
1195 | |
1196 | bool zswap_store(struct folio *folio) |
1197 | { |
1198 | swp_entry_t swp = folio->swap; |
1199 | int type = swp_type(entry: swp); |
1200 | pgoff_t offset = swp_offset(entry: swp); |
1201 | struct page *page = &folio->page; |
1202 | struct zswap_tree *tree = zswap_trees[type]; |
1203 | struct zswap_entry *entry, *dupentry; |
1204 | struct scatterlist input, output; |
1205 | struct crypto_acomp_ctx *acomp_ctx; |
1206 | struct obj_cgroup *objcg = NULL; |
1207 | struct zswap_pool *pool; |
1208 | struct zpool *zpool; |
1209 | unsigned int dlen = PAGE_SIZE; |
1210 | unsigned long handle, value; |
1211 | char *buf; |
1212 | u8 *src, *dst; |
1213 | gfp_t gfp; |
1214 | int ret; |
1215 | |
1216 | VM_WARN_ON_ONCE(!folio_test_locked(folio)); |
1217 | VM_WARN_ON_ONCE(!folio_test_swapcache(folio)); |
1218 | |
1219 | /* Large folios aren't supported */ |
1220 | if (folio_test_large(folio)) |
1221 | return false; |
1222 | |
1223 | if (!zswap_enabled || !tree) |
1224 | return false; |
1225 | |
1226 | /* |
1227 | * If this is a duplicate, it must be removed before attempting to store |
1228 | * it, otherwise, if the store fails the old page won't be removed from |
1229 | * the tree, and it might be written back overriding the new data. |
1230 | */ |
1231 | spin_lock(lock: &tree->lock); |
1232 | dupentry = zswap_rb_search(root: &tree->rbroot, offset); |
1233 | if (dupentry) { |
1234 | zswap_duplicate_entry++; |
1235 | zswap_invalidate_entry(tree, entry: dupentry); |
1236 | } |
1237 | spin_unlock(lock: &tree->lock); |
1238 | |
1239 | /* |
1240 | * XXX: zswap reclaim does not work with cgroups yet. Without a |
1241 | * cgroup-aware entry LRU, we will push out entries system-wide based on |
1242 | * local cgroup limits. |
1243 | */ |
1244 | objcg = get_obj_cgroup_from_folio(folio); |
1245 | if (objcg && !obj_cgroup_may_zswap(objcg)) |
1246 | goto reject; |
1247 | |
1248 | /* reclaim space if needed */ |
1249 | if (zswap_is_full()) { |
1250 | zswap_pool_limit_hit++; |
1251 | zswap_pool_reached_full = true; |
1252 | goto shrink; |
1253 | } |
1254 | |
1255 | if (zswap_pool_reached_full) { |
1256 | if (!zswap_can_accept()) |
1257 | goto shrink; |
1258 | else |
1259 | zswap_pool_reached_full = false; |
1260 | } |
1261 | |
1262 | /* allocate entry */ |
1263 | entry = zswap_entry_cache_alloc(GFP_KERNEL); |
1264 | if (!entry) { |
1265 | zswap_reject_kmemcache_fail++; |
1266 | goto reject; |
1267 | } |
1268 | |
1269 | if (zswap_same_filled_pages_enabled) { |
1270 | src = kmap_atomic(page); |
1271 | if (zswap_is_page_same_filled(ptr: src, value: &value)) { |
1272 | kunmap_atomic(src); |
1273 | entry->swpentry = swp_entry(type, offset); |
1274 | entry->length = 0; |
1275 | entry->value = value; |
1276 | atomic_inc(v: &zswap_same_filled_pages); |
1277 | goto insert_entry; |
1278 | } |
1279 | kunmap_atomic(src); |
1280 | } |
1281 | |
1282 | if (!zswap_non_same_filled_pages_enabled) |
1283 | goto freepage; |
1284 | |
1285 | /* if entry is successfully added, it keeps the reference */ |
1286 | entry->pool = zswap_pool_current_get(); |
1287 | if (!entry->pool) |
1288 | goto freepage; |
1289 | |
1290 | /* compress */ |
1291 | acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); |
1292 | |
1293 | mutex_lock(acomp_ctx->mutex); |
1294 | |
1295 | dst = acomp_ctx->dstmem; |
1296 | sg_init_table(&input, 1); |
1297 | sg_set_page(sg: &input, page, PAGE_SIZE, offset: 0); |
1298 | |
1299 | /* zswap_dstmem is of size (PAGE_SIZE * 2). Reflect same in sg_list */ |
1300 | sg_init_one(&output, dst, PAGE_SIZE * 2); |
1301 | acomp_request_set_params(req: acomp_ctx->req, src: &input, dst: &output, PAGE_SIZE, dlen); |
1302 | /* |
1303 | * it maybe looks a little bit silly that we send an asynchronous request, |
1304 | * then wait for its completion synchronously. This makes the process look |
1305 | * synchronous in fact. |
1306 | * Theoretically, acomp supports users send multiple acomp requests in one |
1307 | * acomp instance, then get those requests done simultaneously. but in this |
1308 | * case, zswap actually does store and load page by page, there is no |
1309 | * existing method to send the second page before the first page is done |
1310 | * in one thread doing zwap. |
1311 | * but in different threads running on different cpu, we have different |
1312 | * acomp instance, so multiple threads can do (de)compression in parallel. |
1313 | */ |
1314 | ret = crypto_wait_req(err: crypto_acomp_compress(req: acomp_ctx->req), wait: &acomp_ctx->wait); |
1315 | dlen = acomp_ctx->req->dlen; |
1316 | |
1317 | if (ret) { |
1318 | zswap_reject_compress_fail++; |
1319 | goto put_dstmem; |
1320 | } |
1321 | |
1322 | /* store */ |
1323 | zpool = zswap_find_zpool(entry); |
1324 | gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; |
1325 | if (zpool_malloc_support_movable(pool: zpool)) |
1326 | gfp |= __GFP_HIGHMEM | __GFP_MOVABLE; |
1327 | ret = zpool_malloc(pool: zpool, size: dlen, gfp, handle: &handle); |
1328 | if (ret == -ENOSPC) { |
1329 | zswap_reject_compress_poor++; |
1330 | goto put_dstmem; |
1331 | } |
1332 | if (ret) { |
1333 | zswap_reject_alloc_fail++; |
1334 | goto put_dstmem; |
1335 | } |
1336 | buf = zpool_map_handle(pool: zpool, handle, mm: ZPOOL_MM_WO); |
1337 | memcpy(buf, dst, dlen); |
1338 | zpool_unmap_handle(pool: zpool, handle); |
1339 | mutex_unlock(lock: acomp_ctx->mutex); |
1340 | |
1341 | /* populate entry */ |
1342 | entry->swpentry = swp_entry(type, offset); |
1343 | entry->handle = handle; |
1344 | entry->length = dlen; |
1345 | |
1346 | insert_entry: |
1347 | entry->objcg = objcg; |
1348 | if (objcg) { |
1349 | obj_cgroup_charge_zswap(objcg, size: entry->length); |
1350 | /* Account before objcg ref is moved to tree */ |
1351 | count_objcg_event(objcg, idx: ZSWPOUT); |
1352 | } |
1353 | |
1354 | /* map */ |
1355 | spin_lock(lock: &tree->lock); |
1356 | /* |
1357 | * A duplicate entry should have been removed at the beginning of this |
1358 | * function. Since the swap entry should be pinned, if a duplicate is |
1359 | * found again here it means that something went wrong in the swap |
1360 | * cache. |
1361 | */ |
1362 | while (zswap_rb_insert(root: &tree->rbroot, entry, dupentry: &dupentry) == -EEXIST) { |
1363 | WARN_ON(1); |
1364 | zswap_duplicate_entry++; |
1365 | zswap_invalidate_entry(tree, entry: dupentry); |
1366 | } |
1367 | if (entry->length) { |
1368 | spin_lock(lock: &entry->pool->lru_lock); |
1369 | list_add(new: &entry->lru, head: &entry->pool->lru); |
1370 | spin_unlock(lock: &entry->pool->lru_lock); |
1371 | } |
1372 | spin_unlock(lock: &tree->lock); |
1373 | |
1374 | /* update stats */ |
1375 | atomic_inc(v: &zswap_stored_pages); |
1376 | zswap_update_total_size(); |
1377 | count_vm_event(item: ZSWPOUT); |
1378 | |
1379 | return true; |
1380 | |
1381 | put_dstmem: |
1382 | mutex_unlock(lock: acomp_ctx->mutex); |
1383 | zswap_pool_put(pool: entry->pool); |
1384 | freepage: |
1385 | zswap_entry_cache_free(entry); |
1386 | reject: |
1387 | if (objcg) |
1388 | obj_cgroup_put(objcg); |
1389 | return false; |
1390 | |
1391 | shrink: |
1392 | pool = zswap_pool_last_get(); |
1393 | if (pool && !queue_work(wq: shrink_wq, work: &pool->shrink_work)) |
1394 | zswap_pool_put(pool); |
1395 | goto reject; |
1396 | } |
1397 | |
1398 | bool zswap_load(struct folio *folio) |
1399 | { |
1400 | swp_entry_t swp = folio->swap; |
1401 | int type = swp_type(entry: swp); |
1402 | pgoff_t offset = swp_offset(entry: swp); |
1403 | struct page *page = &folio->page; |
1404 | struct zswap_tree *tree = zswap_trees[type]; |
1405 | struct zswap_entry *entry; |
1406 | struct scatterlist input, output; |
1407 | struct crypto_acomp_ctx *acomp_ctx; |
1408 | u8 *src, *dst, *tmp; |
1409 | struct zpool *zpool; |
1410 | unsigned int dlen; |
1411 | bool ret; |
1412 | |
1413 | VM_WARN_ON_ONCE(!folio_test_locked(folio)); |
1414 | |
1415 | /* find */ |
1416 | spin_lock(lock: &tree->lock); |
1417 | entry = zswap_entry_find_get(root: &tree->rbroot, offset); |
1418 | if (!entry) { |
1419 | spin_unlock(lock: &tree->lock); |
1420 | return false; |
1421 | } |
1422 | spin_unlock(lock: &tree->lock); |
1423 | |
1424 | if (!entry->length) { |
1425 | dst = kmap_atomic(page); |
1426 | zswap_fill_page(ptr: dst, value: entry->value); |
1427 | kunmap_atomic(dst); |
1428 | ret = true; |
1429 | goto stats; |
1430 | } |
1431 | |
1432 | zpool = zswap_find_zpool(entry); |
1433 | if (!zpool_can_sleep_mapped(pool: zpool)) { |
1434 | tmp = kmalloc(size: entry->length, GFP_KERNEL); |
1435 | if (!tmp) { |
1436 | ret = false; |
1437 | goto freeentry; |
1438 | } |
1439 | } |
1440 | |
1441 | /* decompress */ |
1442 | dlen = PAGE_SIZE; |
1443 | src = zpool_map_handle(pool: zpool, handle: entry->handle, mm: ZPOOL_MM_RO); |
1444 | |
1445 | if (!zpool_can_sleep_mapped(pool: zpool)) { |
1446 | memcpy(tmp, src, entry->length); |
1447 | src = tmp; |
1448 | zpool_unmap_handle(pool: zpool, handle: entry->handle); |
1449 | } |
1450 | |
1451 | acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx); |
1452 | mutex_lock(acomp_ctx->mutex); |
1453 | sg_init_one(&input, src, entry->length); |
1454 | sg_init_table(&output, 1); |
1455 | sg_set_page(sg: &output, page, PAGE_SIZE, offset: 0); |
1456 | acomp_request_set_params(req: acomp_ctx->req, src: &input, dst: &output, slen: entry->length, dlen); |
1457 | if (crypto_wait_req(err: crypto_acomp_decompress(req: acomp_ctx->req), wait: &acomp_ctx->wait)) |
1458 | WARN_ON(1); |
1459 | mutex_unlock(lock: acomp_ctx->mutex); |
1460 | |
1461 | if (zpool_can_sleep_mapped(pool: zpool)) |
1462 | zpool_unmap_handle(pool: zpool, handle: entry->handle); |
1463 | else |
1464 | kfree(objp: tmp); |
1465 | |
1466 | ret = true; |
1467 | stats: |
1468 | count_vm_event(item: ZSWPIN); |
1469 | if (entry->objcg) |
1470 | count_objcg_event(objcg: entry->objcg, idx: ZSWPIN); |
1471 | freeentry: |
1472 | spin_lock(lock: &tree->lock); |
1473 | if (ret && zswap_exclusive_loads_enabled) { |
1474 | zswap_invalidate_entry(tree, entry); |
1475 | folio_mark_dirty(folio); |
1476 | } else if (entry->length) { |
1477 | spin_lock(lock: &entry->pool->lru_lock); |
1478 | list_move(list: &entry->lru, head: &entry->pool->lru); |
1479 | spin_unlock(lock: &entry->pool->lru_lock); |
1480 | } |
1481 | zswap_entry_put(tree, entry); |
1482 | spin_unlock(lock: &tree->lock); |
1483 | |
1484 | return ret; |
1485 | } |
1486 | |
1487 | void zswap_invalidate(int type, pgoff_t offset) |
1488 | { |
1489 | struct zswap_tree *tree = zswap_trees[type]; |
1490 | struct zswap_entry *entry; |
1491 | |
1492 | /* find */ |
1493 | spin_lock(lock: &tree->lock); |
1494 | entry = zswap_rb_search(root: &tree->rbroot, offset); |
1495 | if (!entry) { |
1496 | /* entry was written back */ |
1497 | spin_unlock(lock: &tree->lock); |
1498 | return; |
1499 | } |
1500 | zswap_invalidate_entry(tree, entry); |
1501 | spin_unlock(lock: &tree->lock); |
1502 | } |
1503 | |
1504 | void zswap_swapon(int type) |
1505 | { |
1506 | struct zswap_tree *tree; |
1507 | |
1508 | tree = kzalloc(size: sizeof(*tree), GFP_KERNEL); |
1509 | if (!tree) { |
1510 | pr_err("alloc failed, zswap disabled for swap type %d\n" , type); |
1511 | return; |
1512 | } |
1513 | |
1514 | tree->rbroot = RB_ROOT; |
1515 | spin_lock_init(&tree->lock); |
1516 | zswap_trees[type] = tree; |
1517 | } |
1518 | |
1519 | void zswap_swapoff(int type) |
1520 | { |
1521 | struct zswap_tree *tree = zswap_trees[type]; |
1522 | struct zswap_entry *entry, *n; |
1523 | |
1524 | if (!tree) |
1525 | return; |
1526 | |
1527 | /* walk the tree and free everything */ |
1528 | spin_lock(lock: &tree->lock); |
1529 | rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode) |
1530 | zswap_free_entry(entry); |
1531 | tree->rbroot = RB_ROOT; |
1532 | spin_unlock(lock: &tree->lock); |
1533 | kfree(objp: tree); |
1534 | zswap_trees[type] = NULL; |
1535 | } |
1536 | |
1537 | /********************************* |
1538 | * debugfs functions |
1539 | **********************************/ |
1540 | #ifdef CONFIG_DEBUG_FS |
1541 | #include <linux/debugfs.h> |
1542 | |
1543 | static struct dentry *zswap_debugfs_root; |
1544 | |
1545 | static int zswap_debugfs_init(void) |
1546 | { |
1547 | if (!debugfs_initialized()) |
1548 | return -ENODEV; |
1549 | |
1550 | zswap_debugfs_root = debugfs_create_dir(name: "zswap" , NULL); |
1551 | |
1552 | debugfs_create_u64(name: "pool_limit_hit" , mode: 0444, |
1553 | parent: zswap_debugfs_root, value: &zswap_pool_limit_hit); |
1554 | debugfs_create_u64(name: "reject_reclaim_fail" , mode: 0444, |
1555 | parent: zswap_debugfs_root, value: &zswap_reject_reclaim_fail); |
1556 | debugfs_create_u64(name: "reject_alloc_fail" , mode: 0444, |
1557 | parent: zswap_debugfs_root, value: &zswap_reject_alloc_fail); |
1558 | debugfs_create_u64(name: "reject_kmemcache_fail" , mode: 0444, |
1559 | parent: zswap_debugfs_root, value: &zswap_reject_kmemcache_fail); |
1560 | debugfs_create_u64(name: "reject_compress_fail" , mode: 0444, |
1561 | parent: zswap_debugfs_root, value: &zswap_reject_compress_fail); |
1562 | debugfs_create_u64(name: "reject_compress_poor" , mode: 0444, |
1563 | parent: zswap_debugfs_root, value: &zswap_reject_compress_poor); |
1564 | debugfs_create_u64(name: "written_back_pages" , mode: 0444, |
1565 | parent: zswap_debugfs_root, value: &zswap_written_back_pages); |
1566 | debugfs_create_u64(name: "duplicate_entry" , mode: 0444, |
1567 | parent: zswap_debugfs_root, value: &zswap_duplicate_entry); |
1568 | debugfs_create_u64(name: "pool_total_size" , mode: 0444, |
1569 | parent: zswap_debugfs_root, value: &zswap_pool_total_size); |
1570 | debugfs_create_atomic_t(name: "stored_pages" , mode: 0444, |
1571 | parent: zswap_debugfs_root, value: &zswap_stored_pages); |
1572 | debugfs_create_atomic_t(name: "same_filled_pages" , mode: 0444, |
1573 | parent: zswap_debugfs_root, value: &zswap_same_filled_pages); |
1574 | |
1575 | return 0; |
1576 | } |
1577 | #else |
1578 | static int zswap_debugfs_init(void) |
1579 | { |
1580 | return 0; |
1581 | } |
1582 | #endif |
1583 | |
1584 | /********************************* |
1585 | * module init and exit |
1586 | **********************************/ |
1587 | static int zswap_setup(void) |
1588 | { |
1589 | struct zswap_pool *pool; |
1590 | int ret; |
1591 | |
1592 | zswap_entry_cache = KMEM_CACHE(zswap_entry, 0); |
1593 | if (!zswap_entry_cache) { |
1594 | pr_err("entry cache creation failed\n" ); |
1595 | goto cache_fail; |
1596 | } |
1597 | |
1598 | ret = cpuhp_setup_state(state: CPUHP_MM_ZSWP_MEM_PREPARE, name: "mm/zswap:prepare" , |
1599 | startup: zswap_dstmem_prepare, teardown: zswap_dstmem_dead); |
1600 | if (ret) { |
1601 | pr_err("dstmem alloc failed\n" ); |
1602 | goto dstmem_fail; |
1603 | } |
1604 | |
1605 | ret = cpuhp_setup_state_multi(state: CPUHP_MM_ZSWP_POOL_PREPARE, |
1606 | name: "mm/zswap_pool:prepare" , |
1607 | startup: zswap_cpu_comp_prepare, |
1608 | teardown: zswap_cpu_comp_dead); |
1609 | if (ret) |
1610 | goto hp_fail; |
1611 | |
1612 | pool = __zswap_pool_create_fallback(); |
1613 | if (pool) { |
1614 | pr_info("loaded using pool %s/%s\n" , pool->tfm_name, |
1615 | zpool_get_type(pool->zpools[0])); |
1616 | list_add(new: &pool->list, head: &zswap_pools); |
1617 | zswap_has_pool = true; |
1618 | } else { |
1619 | pr_err("pool creation failed\n" ); |
1620 | zswap_enabled = false; |
1621 | } |
1622 | |
1623 | shrink_wq = create_workqueue("zswap-shrink" ); |
1624 | if (!shrink_wq) |
1625 | goto fallback_fail; |
1626 | |
1627 | if (zswap_debugfs_init()) |
1628 | pr_warn("debugfs initialization failed\n" ); |
1629 | zswap_init_state = ZSWAP_INIT_SUCCEED; |
1630 | return 0; |
1631 | |
1632 | fallback_fail: |
1633 | if (pool) |
1634 | zswap_pool_destroy(pool); |
1635 | hp_fail: |
1636 | cpuhp_remove_state(state: CPUHP_MM_ZSWP_MEM_PREPARE); |
1637 | dstmem_fail: |
1638 | kmem_cache_destroy(s: zswap_entry_cache); |
1639 | cache_fail: |
1640 | /* if built-in, we aren't unloaded on failure; don't allow use */ |
1641 | zswap_init_state = ZSWAP_INIT_FAILED; |
1642 | zswap_enabled = false; |
1643 | return -ENOMEM; |
1644 | } |
1645 | |
1646 | static int __init zswap_init(void) |
1647 | { |
1648 | if (!zswap_enabled) |
1649 | return 0; |
1650 | return zswap_setup(); |
1651 | } |
1652 | /* must be late so crypto has time to come up */ |
1653 | late_initcall(zswap_init); |
1654 | |
1655 | MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>" ); |
1656 | MODULE_DESCRIPTION("Compressed cache for swap pages" ); |
1657 | |