1	/*
2	* Compressed RAM block device
3	*
4	* Copyright (C) 2008, 2009, 2010 Nitin Gupta
5	* 2012, 2013 Minchan Kim
6	*
7	* This code is released using a dual license strategy: BSD/GPL
8	* You can choose the licence that better fits your requirements.
9	*
10	* Released under the terms of 3-clause BSD License
11	* Released under the terms of GNU General Public License Version 2.0
12	*
13	*/
14
15	#define KMSG_COMPONENT "zram"
16	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
17
18	#include <linux/module.h>
19	#include <linux/kernel.h>
20	#include <linux/bio.h>
21	#include <linux/bitops.h>
22	#include <linux/blkdev.h>
23	#include <linux/buffer_head.h>
24	#include <linux/device.h>
25	#include <linux/highmem.h>
26	#include <linux/slab.h>
27	#include <linux/backing-dev.h>
28	#include <linux/string.h>
29	#include <linux/vmalloc.h>
30	#include <linux/err.h>
31	#include <linux/idr.h>
32	#include <linux/sysfs.h>
33	#include <linux/debugfs.h>
34	#include <linux/cpuhotplug.h>
35	#include <linux/part_stat.h>
36	#include <linux/kernel_read_file.h>
37
38	#include "zram_drv.h"
39
40	static DEFINE_IDR(zram_index_idr);
41	/* idr index must be protected */
42	static DEFINE_MUTEX(zram_index_mutex);
43
44	static int zram_major;
45	static const char *default_compressor = CONFIG_ZRAM_DEF_COMP;
46
47	#define ZRAM_MAX_ALGO_NAME_SZ 128
48
49	/* Module params (documentation at end) */
50	static unsigned int num_devices = 1;
51	/*
52	* Pages that compress to sizes equals or greater than this are stored
53	* uncompressed in memory.
54	*/
55	static size_t huge_class_size;
56
57	static const struct block_device_operations zram_devops;
58
59	static void zram_free_page(struct zram *zram, size_t index);
60	static int zram_read_from_zspool(struct zram *zram, struct page *page,
61	u32 index);
62
63	#define slot_dep_map(zram, index) (&(zram)->table[(index)].dep_map)
64
65	static void zram_slot_lock_init(struct zram *zram, u32 index)
66	{
67	static struct lock_class_key __key;
68
69	lockdep_init_map(slot_dep_map(zram, index), name: "zram->table[index].lock",
70	key: &__key, subclass: 0);
71	}
72
73	/*
74	* entry locking rules:
75	*
76	* 1) Lock is exclusive
77	*
78	* 2) lock() function can sleep waiting for the lock
79	*
80	* 3) Lock owner can sleep
81	*
82	* 4) Use TRY lock variant when in atomic context
83	* - must check return value and handle locking failers
84	*/
85	static __must_check bool zram_slot_trylock(struct zram *zram, u32 index)
86	{
87	unsigned long *lock = &zram->table[index].flags;
88
89	if (!test_and_set_bit_lock(nr: ZRAM_ENTRY_LOCK, addr: lock)) {
90	mutex_acquire(slot_dep_map(zram, index), 0, 1, _RET_IP_);
91	lock_acquired(slot_dep_map(zram, index), _RET_IP_);
92	return true;
93	}
94
95	return false;
96	}
97
98	static void zram_slot_lock(struct zram *zram, u32 index)
99	{
100	unsigned long *lock = &zram->table[index].flags;
101
102	mutex_acquire(slot_dep_map(zram, index), 0, 0, _RET_IP_);
103	wait_on_bit_lock(word: lock, bit: ZRAM_ENTRY_LOCK, TASK_UNINTERRUPTIBLE);
104	lock_acquired(slot_dep_map(zram, index), _RET_IP_);
105	}
106
107	static void zram_slot_unlock(struct zram *zram, u32 index)
108	{
109	unsigned long *lock = &zram->table[index].flags;
110
111	mutex_release(slot_dep_map(zram, index), _RET_IP_);
112	clear_and_wake_up_bit(bit: ZRAM_ENTRY_LOCK, word: lock);
113	}
114
115	static inline bool init_done(struct zram *zram)
116	{
117	return zram->disksize;
118	}
119
120	static inline struct zram *dev_to_zram(struct device *dev)
121	{
122	return (struct zram *)dev_to_disk(dev)->private_data;
123	}
124
125	static unsigned long zram_get_handle(struct zram *zram, u32 index)
126	{
127	return zram->table[index].handle;
128	}
129
130	static void zram_set_handle(struct zram *zram, u32 index, unsigned long handle)
131	{
132	zram->table[index].handle = handle;
133	}
134
135	static bool zram_test_flag(struct zram *zram, u32 index,
136	enum zram_pageflags flag)
137	{
138	return zram->table[index].flags & BIT(flag);
139	}
140
141	static void zram_set_flag(struct zram *zram, u32 index,
142	enum zram_pageflags flag)
143	{
144	zram->table[index].flags |= BIT(flag);
145	}
146
147	static void zram_clear_flag(struct zram *zram, u32 index,
148	enum zram_pageflags flag)
149	{
150	zram->table[index].flags &= ~BIT(flag);
151	}
152
153	static size_t zram_get_obj_size(struct zram *zram, u32 index)
154	{
155	return zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1);
156	}
157
158	static void zram_set_obj_size(struct zram *zram,
159	u32 index, size_t size)
160	{
161	unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT;
162
163	zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size;
164	}
165
166	static inline bool zram_allocated(struct zram *zram, u32 index)
167	{
168	return zram_get_obj_size(zram, index) ||
169	zram_test_flag(zram, index, flag: ZRAM_SAME) ||
170	zram_test_flag(zram, index, flag: ZRAM_WB);
171	}
172
173	static inline void update_used_max(struct zram *zram, const unsigned long pages)
174	{
175	unsigned long cur_max = atomic_long_read(v: &zram->stats.max_used_pages);
176
177	do {
178	if (cur_max >= pages)
179	return;
180	} while (!atomic_long_try_cmpxchg(v: &zram->stats.max_used_pages,
181	old: &cur_max, new: pages));
182	}
183
184	static bool zram_can_store_page(struct zram *zram)
185	{
186	unsigned long alloced_pages;
187
188	alloced_pages = zs_get_total_pages(pool: zram->mem_pool);
189	update_used_max(zram, pages: alloced_pages);
190
191	return !zram->limit_pages || alloced_pages <= zram->limit_pages;
192	}
193
194	#if PAGE_SIZE != 4096
195	static inline bool is_partial_io(struct bio_vec *bvec)
196	{
197	return bvec->bv_len != PAGE_SIZE;
198	}
199	#define ZRAM_PARTIAL_IO 1
200	#else
201	static inline bool is_partial_io(struct bio_vec *bvec)
202	{
203	return false;
204	}
205	#endif
206
207	static inline void zram_set_priority(struct zram *zram, u32 index, u32 prio)
208	{
209	prio &= ZRAM_COMP_PRIORITY_MASK;
210	/*
211	* Clear previous priority value first, in case if we recompress
212	* further an already recompressed page
213	*/
214	zram->table[index].flags &= ~(ZRAM_COMP_PRIORITY_MASK <<
215	ZRAM_COMP_PRIORITY_BIT1);
216	zram->table[index].flags |= (prio << ZRAM_COMP_PRIORITY_BIT1);
217	}
218
219	static inline u32 zram_get_priority(struct zram *zram, u32 index)
220	{
221	u32 prio = zram->table[index].flags >> ZRAM_COMP_PRIORITY_BIT1;
222
223	return prio & ZRAM_COMP_PRIORITY_MASK;
224	}
225
226	static void zram_accessed(struct zram *zram, u32 index)
227	{
228	zram_clear_flag(zram, index, flag: ZRAM_IDLE);
229	zram_clear_flag(zram, index, flag: ZRAM_PP_SLOT);
230	#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
231	zram->table[index].ac_time = ktime_get_boottime();
232	#endif
233	}
234
235	#if defined CONFIG_ZRAM_WRITEBACK || defined CONFIG_ZRAM_MULTI_COMP
236	struct zram_pp_slot {
237	unsigned long index;
238	struct list_head entry;
239	};
240
241	/*
242	* A post-processing bucket is, essentially, a size class, this defines
243	* the range (in bytes) of pp-slots sizes in particular bucket.
244	*/
245	#define PP_BUCKET_SIZE_RANGE 64
246	#define NUM_PP_BUCKETS ((PAGE_SIZE / PP_BUCKET_SIZE_RANGE) + 1)
247
248	struct zram_pp_ctl {
249	struct list_head pp_buckets[NUM_PP_BUCKETS];
250	};
251
252	static struct zram_pp_ctl *init_pp_ctl(void)
253	{
254	struct zram_pp_ctl *ctl;
255	u32 idx;
256
257	ctl = kmalloc(sizeof(*ctl), GFP_KERNEL);
258	if (!ctl)
259	return NULL;
260
261	for (idx = 0; idx < NUM_PP_BUCKETS; idx++)
262	INIT_LIST_HEAD(list: &ctl->pp_buckets[idx]);
263	return ctl;
264	}
265
266	static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
267	{
268	list_del_init(entry: &pps->entry);
269
270	zram_slot_lock(zram, index: pps->index);
271	zram_clear_flag(zram, index: pps->index, flag: ZRAM_PP_SLOT);
272	zram_slot_unlock(zram, index: pps->index);
273
274	kfree(objp: pps);
275	}
276
277	static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl)
278	{
279	u32 idx;
280
281	if (!ctl)
282	return;
283
284	for (idx = 0; idx < NUM_PP_BUCKETS; idx++) {
285	while (!list_empty(head: &ctl->pp_buckets[idx])) {
286	struct zram_pp_slot *pps;
287
288	pps = list_first_entry(&ctl->pp_buckets[idx],
289	struct zram_pp_slot,
290	entry);
291	release_pp_slot(zram, pps);
292	}
293	}
294
295	kfree(objp: ctl);
296	}
297
298	static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl,
299	u32 index)
300	{
301	struct zram_pp_slot *pps;
302	u32 bid;
303
304	pps = kmalloc(sizeof(*pps), GFP_NOIO | __GFP_NOWARN);
305	if (!pps)
306	return false;
307
308	INIT_LIST_HEAD(list: &pps->entry);
309	pps->index = index;
310
311	bid = zram_get_obj_size(zram, index: pps->index) / PP_BUCKET_SIZE_RANGE;
312	list_add(new: &pps->entry, head: &ctl->pp_buckets[bid]);
313
314	zram_set_flag(zram, index: pps->index, flag: ZRAM_PP_SLOT);
315	return true;
316	}
317
318	static struct zram_pp_slot *select_pp_slot(struct zram_pp_ctl *ctl)
319	{
320	struct zram_pp_slot *pps = NULL;
321	s32 idx = NUM_PP_BUCKETS - 1;
322
323	/* The higher the bucket id the more optimal slot post-processing is */
324	while (idx >= 0) {
325	pps = list_first_entry_or_null(&ctl->pp_buckets[idx],
326	struct zram_pp_slot,
327	entry);
328	if (pps)
329	break;
330
331	idx--;
332	}
333	return pps;
334	}
335	#endif
336
337	static inline void zram_fill_page(void *ptr, unsigned long len,
338	unsigned long value)
339	{
340	WARN_ON_ONCE(!IS_ALIGNED(len, sizeof(unsigned long)));
341	memset_l(p: ptr, v: value, n: len / sizeof(unsigned long));
342	}
343
344	static bool page_same_filled(void *ptr, unsigned long *element)
345	{
346	unsigned long *page;
347	unsigned long val;
348	unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1;
349
350	page = (unsigned long *)ptr;
351	val = page[0];
352
353	if (val != page[last_pos])
354	return false;
355
356	for (pos = 1; pos < last_pos; pos++) {
357	if (val != page[pos])
358	return false;
359	}
360
361	*element = val;
362
363	return true;
364	}
365
366	static ssize_t initstate_show(struct device *dev,
367	struct device_attribute *attr, char *buf)
368	{
369	u32 val;
370	struct zram *zram = dev_to_zram(dev);
371
372	down_read(sem: &zram->init_lock);
373	val = init_done(zram);
374	up_read(sem: &zram->init_lock);
375
376	return scnprintf(buf, PAGE_SIZE, fmt: "%u\n", val);
377	}
378
379	static ssize_t disksize_show(struct device *dev,
380	struct device_attribute *attr, char *buf)
381	{
382	struct zram *zram = dev_to_zram(dev);
383
384	return scnprintf(buf, PAGE_SIZE, fmt: "%llu\n", zram->disksize);
385	}
386
387	static ssize_t mem_limit_store(struct device *dev,
388	struct device_attribute *attr, const char *buf, size_t len)
389	{
390	u64 limit;
391	char *tmp;
392	struct zram *zram = dev_to_zram(dev);
393
394	limit = memparse(ptr: buf, retptr: &tmp);
395	if (buf == tmp) /* no chars parsed, invalid input */
396	return -EINVAL;
397
398	down_write(sem: &zram->init_lock);
399	zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
400	up_write(sem: &zram->init_lock);
401
402	return len;
403	}
404
405	static ssize_t mem_used_max_store(struct device *dev,
406	struct device_attribute *attr, const char *buf, size_t len)
407	{
408	int err;
409	unsigned long val;
410	struct zram *zram = dev_to_zram(dev);
411
412	err = kstrtoul(s: buf, base: 10, res: &val);
413	if (err || val != 0)
414	return -EINVAL;
415
416	down_read(sem: &zram->init_lock);
417	if (init_done(zram)) {
418	atomic_long_set(v: &zram->stats.max_used_pages,
419	i: zs_get_total_pages(pool: zram->mem_pool));
420	}
421	up_read(sem: &zram->init_lock);
422
423	return len;
424	}
425
426	/*
427	* Mark all pages which are older than or equal to cutoff as IDLE.
428	* Callers should hold the zram init lock in read mode
429	*/
430	static void mark_idle(struct zram *zram, ktime_t cutoff)
431	{
432	int is_idle = 1;
433	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
434	int index;
435
436	for (index = 0; index < nr_pages; index++) {
437	/*
438	* Do not mark ZRAM_SAME slots as ZRAM_IDLE, because no
439	* post-processing (recompress, writeback) happens to the
440	* ZRAM_SAME slot.
441	*
442	* And ZRAM_WB slots simply cannot be ZRAM_IDLE.
443	*/
444	zram_slot_lock(zram, index);
445	if (!zram_allocated(zram, index) ||
446	zram_test_flag(zram, index, flag: ZRAM_WB) ||
447	zram_test_flag(zram, index, flag: ZRAM_SAME)) {
448	zram_slot_unlock(zram, index);
449	continue;
450	}
451
452	#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
453	is_idle = !cutoff ||
454	ktime_after(cmp1: cutoff, cmp2: zram->table[index].ac_time);
455	#endif
456	if (is_idle)
457	zram_set_flag(zram, index, flag: ZRAM_IDLE);
458	else
459	zram_clear_flag(zram, index, flag: ZRAM_IDLE);
460	zram_slot_unlock(zram, index);
461	}
462	}
463
464	static ssize_t idle_store(struct device *dev,
465	struct device_attribute *attr, const char *buf, size_t len)
466	{
467	struct zram *zram = dev_to_zram(dev);
468	ktime_t cutoff_time = 0;
469	ssize_t rv = -EINVAL;
470
471	if (!sysfs_streq(s1: buf, s2: "all")) {
472	/*
473	* If it did not parse as 'all' try to treat it as an integer
474	* when we have memory tracking enabled.
475	*/
476	u64 age_sec;
477
478	if (IS_ENABLED(CONFIG_ZRAM_TRACK_ENTRY_ACTIME) && !kstrtoull(s: buf, base: 0, res: &age_sec))
479	cutoff_time = ktime_sub(ktime_get_boottime(),
480	ns_to_ktime(age_sec * NSEC_PER_SEC));
481	else
482	goto out;
483	}
484
485	down_read(sem: &zram->init_lock);
486	if (!init_done(zram))
487	goto out_unlock;
488
489	/*
490	* A cutoff_time of 0 marks everything as idle, this is the
491	* "all" behavior.
492	*/
493	mark_idle(zram, cutoff: cutoff_time);
494	rv = len;
495
496	out_unlock:
497	up_read(sem: &zram->init_lock);
498	out:
499	return rv;
500	}
501
502	#ifdef CONFIG_ZRAM_WRITEBACK
503	static ssize_t writeback_limit_enable_store(struct device *dev,
504	struct device_attribute *attr, const char *buf, size_t len)
505	{
506	struct zram *zram = dev_to_zram(dev);
507	u64 val;
508	ssize_t ret = -EINVAL;
509
510	if (kstrtoull(s: buf, base: 10, res: &val))
511	return ret;
512
513	down_read(sem: &zram->init_lock);
514	spin_lock(lock: &zram->wb_limit_lock);
515	zram->wb_limit_enable = val;
516	spin_unlock(lock: &zram->wb_limit_lock);
517	up_read(sem: &zram->init_lock);
518	ret = len;
519
520	return ret;
521	}
522
523	static ssize_t writeback_limit_enable_show(struct device *dev,
524	struct device_attribute *attr, char *buf)
525	{
526	bool val;
527	struct zram *zram = dev_to_zram(dev);
528
529	down_read(sem: &zram->init_lock);
530	spin_lock(lock: &zram->wb_limit_lock);
531	val = zram->wb_limit_enable;
532	spin_unlock(lock: &zram->wb_limit_lock);
533	up_read(sem: &zram->init_lock);
534
535	return scnprintf(buf, PAGE_SIZE, fmt: "%d\n", val);
536	}
537
538	static ssize_t writeback_limit_store(struct device *dev,
539	struct device_attribute *attr, const char *buf, size_t len)
540	{
541	struct zram *zram = dev_to_zram(dev);
542	u64 val;
543	ssize_t ret = -EINVAL;
544
545	if (kstrtoull(s: buf, base: 10, res: &val))
546	return ret;
547
548	down_read(sem: &zram->init_lock);
549	spin_lock(lock: &zram->wb_limit_lock);
550	zram->bd_wb_limit = val;
551	spin_unlock(lock: &zram->wb_limit_lock);
552	up_read(sem: &zram->init_lock);
553	ret = len;
554
555	return ret;
556	}
557
558	static ssize_t writeback_limit_show(struct device *dev,
559	struct device_attribute *attr, char *buf)
560	{
561	u64 val;
562	struct zram *zram = dev_to_zram(dev);
563
564	down_read(sem: &zram->init_lock);
565	spin_lock(lock: &zram->wb_limit_lock);
566	val = zram->bd_wb_limit;
567	spin_unlock(lock: &zram->wb_limit_lock);
568	up_read(sem: &zram->init_lock);
569
570	return scnprintf(buf, PAGE_SIZE, fmt: "%llu\n", val);
571	}
572
573	static void reset_bdev(struct zram *zram)
574	{
575	if (!zram->backing_dev)
576	return;
577
578	/* hope filp_close flush all of IO */
579	filp_close(zram->backing_dev, NULL);
580	zram->backing_dev = NULL;
581	zram->bdev = NULL;
582	zram->disk->fops = &zram_devops;
583	kvfree(addr: zram->bitmap);
584	zram->bitmap = NULL;
585	}
586
587	static ssize_t backing_dev_show(struct device *dev,
588	struct device_attribute *attr, char *buf)
589	{
590	struct file *file;
591	struct zram *zram = dev_to_zram(dev);
592	char *p;
593	ssize_t ret;
594
595	down_read(sem: &zram->init_lock);
596	file = zram->backing_dev;
597	if (!file) {
598	memcpy(buf, "none\n", 5);
599	up_read(sem: &zram->init_lock);
600	return 5;
601	}
602
603	p = file_path(file, buf, PAGE_SIZE - 1);
604	if (IS_ERR(ptr: p)) {
605	ret = PTR_ERR(ptr: p);
606	goto out;
607	}
608
609	ret = strlen(p);
610	memmove(buf, p, ret);
611	buf[ret++] = '\n';
612	out:
613	up_read(sem: &zram->init_lock);
614	return ret;
615	}
616
617	static ssize_t backing_dev_store(struct device *dev,
618	struct device_attribute *attr, const char *buf, size_t len)
619	{
620	char *file_name;
621	size_t sz;
622	struct file *backing_dev = NULL;
623	struct inode *inode;
624	unsigned int bitmap_sz;
625	unsigned long nr_pages, *bitmap = NULL;
626	int err;
627	struct zram *zram = dev_to_zram(dev);
628
629	file_name = kmalloc(PATH_MAX, GFP_KERNEL);
630	if (!file_name)
631	return -ENOMEM;
632
633	down_write(sem: &zram->init_lock);
634	if (init_done(zram)) {
635	pr_info("Can't setup backing device for initialized device\n");
636	err = -EBUSY;
637	goto out;
638	}
639
640	strscpy(file_name, buf, PATH_MAX);
641	/* ignore trailing newline */
642	sz = strlen(file_name);
643	if (sz > 0 && file_name[sz - 1] == '\n')
644	file_name[sz - 1] = 0x00;
645
646	backing_dev = filp_open(file_name, O_RDWR | O_LARGEFILE | O_EXCL, 0);
647	if (IS_ERR(ptr: backing_dev)) {
648	err = PTR_ERR(ptr: backing_dev);
649	backing_dev = NULL;
650	goto out;
651	}
652
653	inode = backing_dev->f_mapping->host;
654
655	/* Support only block device in this moment */
656	if (!S_ISBLK(inode->i_mode)) {
657	err = -ENOTBLK;
658	goto out;
659	}
660
661	nr_pages = i_size_read(inode) >> PAGE_SHIFT;
662	/* Refuse to use zero sized device (also prevents self reference) */
663	if (!nr_pages) {
664	err = -EINVAL;
665	goto out;
666	}
667
668	bitmap_sz = BITS_TO_LONGS(nr_pages) * sizeof(long);
669	bitmap = kvzalloc(bitmap_sz, GFP_KERNEL);
670	if (!bitmap) {
671	err = -ENOMEM;
672	goto out;
673	}
674
675	reset_bdev(zram);
676
677	zram->bdev = I_BDEV(inode);
678	zram->backing_dev = backing_dev;
679	zram->bitmap = bitmap;
680	zram->nr_pages = nr_pages;
681	up_write(sem: &zram->init_lock);
682
683	pr_info("setup backing device %s\n", file_name);
684	kfree(objp: file_name);
685
686	return len;
687	out:
688	kvfree(addr: bitmap);
689
690	if (backing_dev)
691	filp_close(backing_dev, NULL);
692
693	up_write(sem: &zram->init_lock);
694
695	kfree(objp: file_name);
696
697	return err;
698	}
699
700	static unsigned long alloc_block_bdev(struct zram *zram)
701	{
702	unsigned long blk_idx = 1;
703	retry:
704	/* skip 0 bit to confuse zram.handle = 0 */
705	blk_idx = find_next_zero_bit(addr: zram->bitmap, size: zram->nr_pages, offset: blk_idx);
706	if (blk_idx == zram->nr_pages)
707	return 0;
708
709	if (test_and_set_bit(nr: blk_idx, addr: zram->bitmap))
710	goto retry;
711
712	atomic64_inc(v: &zram->stats.bd_count);
713	return blk_idx;
714	}
715
716	static void free_block_bdev(struct zram *zram, unsigned long blk_idx)
717	{
718	int was_set;
719
720	was_set = test_and_clear_bit(nr: blk_idx, addr: zram->bitmap);
721	WARN_ON_ONCE(!was_set);
722	atomic64_dec(v: &zram->stats.bd_count);
723	}
724
725	static void read_from_bdev_async(struct zram *zram, struct page *page,
726	unsigned long entry, struct bio *parent)
727	{
728	struct bio *bio;
729
730	bio = bio_alloc(bdev: zram->bdev, nr_vecs: 1, opf: parent->bi_opf, GFP_NOIO);
731	bio->bi_iter.bi_sector = entry * (PAGE_SIZE >> 9);
732	__bio_add_page(bio, page, PAGE_SIZE, off: 0);
733	bio_chain(bio, parent);
734	submit_bio(bio);
735	}
736
737	static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl)
738	{
739	unsigned long blk_idx = 0;
740	struct page *page = NULL;
741	struct zram_pp_slot *pps;
742	struct bio_vec bio_vec;
743	struct bio bio;
744	int ret = 0, err;
745	u32 index;
746
747	page = alloc_page(GFP_KERNEL);
748	if (!page)
749	return -ENOMEM;
750
751	while ((pps = select_pp_slot(ctl))) {
752	spin_lock(lock: &zram->wb_limit_lock);
753	if (zram->wb_limit_enable && !zram->bd_wb_limit) {
754	spin_unlock(lock: &zram->wb_limit_lock);
755	ret = -EIO;
756	break;
757	}
758	spin_unlock(lock: &zram->wb_limit_lock);
759
760	if (!blk_idx) {
761	blk_idx = alloc_block_bdev(zram);
762	if (!blk_idx) {
763	ret = -ENOSPC;
764	break;
765	}
766	}
767
768	index = pps->index;
769	zram_slot_lock(zram, index);
770	/*
771	* scan_slots() sets ZRAM_PP_SLOT and relases slot lock, so
772	* slots can change in the meantime. If slots are accessed or
773	* freed they lose ZRAM_PP_SLOT flag and hence we don't
774	* post-process them.
775	*/
776	if (!zram_test_flag(zram, index, flag: ZRAM_PP_SLOT))
777	goto next;
778	if (zram_read_from_zspool(zram, page, index))
779	goto next;
780	zram_slot_unlock(zram, index);
781
782	bio_init(bio: &bio, bdev: zram->bdev, table: &bio_vec, max_vecs: 1,
783	opf: REQ_OP_WRITE | REQ_SYNC);
784	bio.bi_iter.bi_sector = blk_idx * (PAGE_SIZE >> 9);
785	__bio_add_page(bio: &bio, page, PAGE_SIZE, off: 0);
786
787	/*
788	* XXX: A single page IO would be inefficient for write
789	* but it would be not bad as starter.
790	*/
791	err = submit_bio_wait(bio: &bio);
792	if (err) {
793	release_pp_slot(zram, pps);
794	/*
795	* BIO errors are not fatal, we continue and simply
796	* attempt to writeback the remaining objects (pages).
797	* At the same time we need to signal user-space that
798	* some writes (at least one, but also could be all of
799	* them) were not successful and we do so by returning
800	* the most recent BIO error.
801	*/
802	ret = err;
803	continue;
804	}
805
806	atomic64_inc(v: &zram->stats.bd_writes);
807	zram_slot_lock(zram, index);
808	/*
809	* Same as above, we release slot lock during writeback so
810	* slot can change under us: slot_free() or slot_free() and
811	* reallocation (zram_write_page()). In both cases slot loses
812	* ZRAM_PP_SLOT flag. No concurrent post-processing can set
813	* ZRAM_PP_SLOT on such slots until current post-processing
814	* finishes.
815	*/
816	if (!zram_test_flag(zram, index, flag: ZRAM_PP_SLOT))
817	goto next;
818
819	zram_free_page(zram, index);
820	zram_set_flag(zram, index, flag: ZRAM_WB);
821	zram_set_handle(zram, index, handle: blk_idx);
822	blk_idx = 0;
823	atomic64_inc(v: &zram->stats.pages_stored);
824	spin_lock(lock: &zram->wb_limit_lock);
825	if (zram->wb_limit_enable && zram->bd_wb_limit > 0)
826	zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);
827	spin_unlock(lock: &zram->wb_limit_lock);
828	next:
829	zram_slot_unlock(zram, index);
830	release_pp_slot(zram, pps);
831
832	cond_resched();
833	}
834
835	if (blk_idx)
836	free_block_bdev(zram, blk_idx);
837	if (page)
838	__free_page(page);
839
840	return ret;
841	}
842
843	#define PAGE_WRITEBACK 0
844	#define HUGE_WRITEBACK (1 << 0)
845	#define IDLE_WRITEBACK (1 << 1)
846	#define INCOMPRESSIBLE_WRITEBACK (1 << 2)
847
848	static int parse_page_index(char *val, unsigned long nr_pages,
849	unsigned long *lo, unsigned long *hi)
850	{
851	int ret;
852
853	ret = kstrtoul(s: val, base: 10, res: lo);
854	if (ret)
855	return ret;
856	if (*lo >= nr_pages)
857	return -ERANGE;
858	*hi = *lo + 1;
859	return 0;
860	}
861
862	static int parse_page_indexes(char *val, unsigned long nr_pages,
863	unsigned long *lo, unsigned long *hi)
864	{
865	char *delim;
866	int ret;
867
868	delim = strchr(val, '-');
869	if (!delim)
870	return -EINVAL;
871
872	*delim = 0x00;
873	ret = kstrtoul(s: val, base: 10, res: lo);
874	if (ret)
875	return ret;
876	if (*lo >= nr_pages)
877	return -ERANGE;
878
879	ret = kstrtoul(s: delim + 1, base: 10, res: hi);
880	if (ret)
881	return ret;
882	if (*hi >= nr_pages || *lo > *hi)
883	return -ERANGE;
884	*hi += 1;
885	return 0;
886	}
887
888	static int parse_mode(char *val, u32 *mode)
889	{
890	*mode = 0;
891
892	if (!strcmp(val, "idle"))
893	*mode = IDLE_WRITEBACK;
894	if (!strcmp(val, "huge"))
895	*mode = HUGE_WRITEBACK;
896	if (!strcmp(val, "huge_idle"))
897	*mode = IDLE_WRITEBACK | HUGE_WRITEBACK;
898	if (!strcmp(val, "incompressible"))
899	*mode = INCOMPRESSIBLE_WRITEBACK;
900
901	if (*mode == 0)
902	return -EINVAL;
903	return 0;
904	}
905
906	static int scan_slots_for_writeback(struct zram *zram, u32 mode,
907	unsigned long lo, unsigned long hi,
908	struct zram_pp_ctl *ctl)
909	{
910	u32 index = lo;
911
912	while (index < hi) {
913	bool ok = true;
914
915	zram_slot_lock(zram, index);
916	if (!zram_allocated(zram, index))
917	goto next;
918
919	if (zram_test_flag(zram, index, flag: ZRAM_WB) ||
920	zram_test_flag(zram, index, flag: ZRAM_SAME))
921	goto next;
922
923	if (mode & IDLE_WRITEBACK &&
924	!zram_test_flag(zram, index, flag: ZRAM_IDLE))
925	goto next;
926	if (mode & HUGE_WRITEBACK &&
927	!zram_test_flag(zram, index, flag: ZRAM_HUGE))
928	goto next;
929	if (mode & INCOMPRESSIBLE_WRITEBACK &&
930	!zram_test_flag(zram, index, flag: ZRAM_INCOMPRESSIBLE))
931	goto next;
932
933	ok = place_pp_slot(zram, ctl, index);
934	next:
935	zram_slot_unlock(zram, index);
936	if (!ok)
937	break;
938	index++;
939	}
940
941	return 0;
942	}
943
944	static ssize_t writeback_store(struct device *dev,
945	struct device_attribute *attr,
946	const char *buf, size_t len)
947	{
948	struct zram *zram = dev_to_zram(dev);
949	u64 nr_pages = zram->disksize >> PAGE_SHIFT;
950	unsigned long lo = 0, hi = nr_pages;
951	struct zram_pp_ctl *ctl = NULL;
952	char *args, *param, *val;
953	ssize_t ret = len;
954	int err, mode = 0;
955
956	down_read(sem: &zram->init_lock);
957	if (!init_done(zram)) {
958	up_read(sem: &zram->init_lock);
959	return -EINVAL;
960	}
961
962	/* Do not permit concurrent post-processing actions. */
963	if (atomic_xchg(v: &zram->pp_in_progress, new: 1)) {
964	up_read(sem: &zram->init_lock);
965	return -EAGAIN;
966	}
967
968	if (!zram->backing_dev) {
969	ret = -ENODEV;
970	goto release_init_lock;
971	}
972
973	ctl = init_pp_ctl();
974	if (!ctl) {
975	ret = -ENOMEM;
976	goto release_init_lock;
977	}
978
979	args = skip_spaces(buf);
980	while (*args) {
981	args = next_arg(args, param: &param, val: &val);
982
983	/*
984	* Workaround to support the old writeback interface.
985	*
986	* The old writeback interface has a minor inconsistency and
987	* requires key=value only for page_index parameter, while the
988	* writeback mode is a valueless parameter.
989	*
990	* This is not the case anymore and now all parameters are
991	* required to have values, however, we need to support the
992	* legacy writeback interface format so we check if we can
993	* recognize a valueless parameter as the (legacy) writeback
994	* mode.
995	*/
996	if (!val || !*val) {
997	err = parse_mode(val: param, mode: &mode);
998	if (err) {
999	ret = err;
1000	goto release_init_lock;
1001	}
1002
1003	scan_slots_for_writeback(zram, mode, lo, hi, ctl);
1004	break;
1005	}
1006
1007	if (!strcmp(param, "type")) {
1008	err = parse_mode(val, mode: &mode);
1009	if (err) {
1010	ret = err;
1011	goto release_init_lock;
1012	}
1013
1014	scan_slots_for_writeback(zram, mode, lo, hi, ctl);
1015	break;
1016	}
1017
1018	if (!strcmp(param, "page_index")) {
1019	err = parse_page_index(val, nr_pages, lo: &lo, hi: &hi);
1020	if (err) {
1021	ret = err;
1022	goto release_init_lock;
1023	}
1024
1025	scan_slots_for_writeback(zram, mode, lo, hi, ctl);
1026	continue;
1027	}
1028
1029	if (!strcmp(param, "page_indexes")) {
1030	err = parse_page_indexes(val, nr_pages, lo: &lo, hi: &hi);
1031	if (err) {
1032	ret = err;
1033	goto release_init_lock;
1034	}
1035
1036	scan_slots_for_writeback(zram, mode, lo, hi, ctl);
1037	continue;
1038	}
1039	}
1040
1041	err = zram_writeback_slots(zram, ctl);
1042	if (err)
1043	ret = err;
1044
1045	release_init_lock:
1046	release_pp_ctl(zram, ctl);
1047	atomic_set(v: &zram->pp_in_progress, i: 0);
1048	up_read(sem: &zram->init_lock);
1049
1050	return ret;
1051	}
1052
1053	struct zram_work {
1054	struct work_struct work;
1055	struct zram *zram;
1056	unsigned long entry;
1057	struct page *page;
1058	int error;
1059	};
1060
1061	static void zram_sync_read(struct work_struct *work)
1062	{
1063	struct zram_work *zw = container_of(work, struct zram_work, work);
1064	struct bio_vec bv;
1065	struct bio bio;
1066
1067	bio_init(bio: &bio, bdev: zw->zram->bdev, table: &bv, max_vecs: 1, opf: REQ_OP_READ);
1068	bio.bi_iter.bi_sector = zw->entry * (PAGE_SIZE >> 9);
1069	__bio_add_page(bio: &bio, page: zw->page, PAGE_SIZE, off: 0);
1070	zw->error = submit_bio_wait(bio: &bio);
1071	}
1072
1073	/*
1074	* Block layer want one ->submit_bio to be active at a time, so if we use
1075	* chained IO with parent IO in same context, it's a deadlock. To avoid that,
1076	* use a worker thread context.
1077	*/
1078	static int read_from_bdev_sync(struct zram *zram, struct page *page,
1079	unsigned long entry)
1080	{
1081	struct zram_work work;
1082
1083	work.page = page;
1084	work.zram = zram;
1085	work.entry = entry;
1086
1087	INIT_WORK_ONSTACK(&work.work, zram_sync_read);
1088	queue_work(wq: system_unbound_wq, work: &work.work);
1089	flush_work(work: &work.work);
1090	destroy_work_on_stack(work: &work.work);
1091
1092	return work.error;
1093	}
1094
1095	static int read_from_bdev(struct zram *zram, struct page *page,
1096	unsigned long entry, struct bio *parent)
1097	{
1098	atomic64_inc(v: &zram->stats.bd_reads);
1099	if (!parent) {
1100	if (WARN_ON_ONCE(!IS_ENABLED(ZRAM_PARTIAL_IO)))
1101	return -EIO;
1102	return read_from_bdev_sync(zram, page, entry);
1103	}
1104	read_from_bdev_async(zram, page, entry, parent);
1105	return 0;
1106	}
1107	#else
1108	static inline void reset_bdev(struct zram *zram) {};
1109	static int read_from_bdev(struct zram *zram, struct page *page,
1110	unsigned long entry, struct bio *parent)
1111	{
1112	return -EIO;
1113	}
1114
1115	static void free_block_bdev(struct zram *zram, unsigned long blk_idx) {};
1116	#endif
1117
1118	#ifdef CONFIG_ZRAM_MEMORY_TRACKING
1119
1120	static struct dentry *zram_debugfs_root;
1121
1122	static void zram_debugfs_create(void)
1123	{
1124	zram_debugfs_root = debugfs_create_dir(name: "zram", NULL);
1125	}
1126
1127	static void zram_debugfs_destroy(void)
1128	{
1129	debugfs_remove_recursive(dentry: zram_debugfs_root);
1130	}
1131
1132	static ssize_t read_block_state(struct file *file, char __user *buf,
1133	size_t count, loff_t *ppos)
1134	{
1135	char *kbuf;
1136	ssize_t index, written = 0;
1137	struct zram *zram = file->private_data;
1138	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
1139	struct timespec64 ts;
1140
1141	kbuf = kvmalloc(count, GFP_KERNEL);
1142	if (!kbuf)
1143	return -ENOMEM;
1144
1145	down_read(sem: &zram->init_lock);
1146	if (!init_done(zram)) {
1147	up_read(sem: &zram->init_lock);
1148	kvfree(addr: kbuf);
1149	return -EINVAL;
1150	}
1151
1152	for (index = *ppos; index < nr_pages; index++) {
1153	int copied;
1154
1155	zram_slot_lock(zram, index);
1156	if (!zram_allocated(zram, index))
1157	goto next;
1158
1159	ts = ktime_to_timespec64(zram->table[index].ac_time);
1160	copied = snprintf(buf: kbuf + written, size: count,
1161	fmt: "%12zd %12lld.%06lu %c%c%c%c%c%c\n",
1162	index, (s64)ts.tv_sec,
1163	ts.tv_nsec / NSEC_PER_USEC,
1164	zram_test_flag(zram, index, flag: ZRAM_SAME) ? 's' : '.',
1165	zram_test_flag(zram, index, flag: ZRAM_WB) ? 'w' : '.',
1166	zram_test_flag(zram, index, flag: ZRAM_HUGE) ? 'h' : '.',
1167	zram_test_flag(zram, index, flag: ZRAM_IDLE) ? 'i' : '.',
1168	zram_get_priority(zram, index) ? 'r' : '.',
1169	zram_test_flag(zram, index,
1170	flag: ZRAM_INCOMPRESSIBLE) ? 'n' : '.');
1171
1172	if (count <= copied) {
1173	zram_slot_unlock(zram, index);
1174	break;
1175	}
1176	written += copied;
1177	count -= copied;
1178	next:
1179	zram_slot_unlock(zram, index);
1180	*ppos += 1;
1181	}
1182
1183	up_read(sem: &zram->init_lock);
1184	if (copy_to_user(to: buf, from: kbuf, n: written))
1185	written = -EFAULT;
1186	kvfree(addr: kbuf);
1187
1188	return written;
1189	}
1190
1191	static const struct file_operations proc_zram_block_state_op = {
1192	.open = simple_open,
1193	.read = read_block_state,
1194	.llseek = default_llseek,
1195	};
1196
1197	static void zram_debugfs_register(struct zram *zram)
1198	{
1199	if (!zram_debugfs_root)
1200	return;
1201
1202	zram->debugfs_dir = debugfs_create_dir(name: zram->disk->disk_name,
1203	parent: zram_debugfs_root);
1204	debugfs_create_file("block_state", 0400, zram->debugfs_dir,
1205	zram, &proc_zram_block_state_op);
1206	}
1207
1208	static void zram_debugfs_unregister(struct zram *zram)
1209	{
1210	debugfs_remove_recursive(dentry: zram->debugfs_dir);
1211	}
1212	#else
1213	static void zram_debugfs_create(void) {};
1214	static void zram_debugfs_destroy(void) {};
1215	static void zram_debugfs_register(struct zram *zram) {};
1216	static void zram_debugfs_unregister(struct zram *zram) {};
1217	#endif
1218
1219	static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg)
1220	{
1221	/* Do not free statically defined compression algorithms */
1222	if (zram->comp_algs[prio] != default_compressor)
1223	kfree(objp: zram->comp_algs[prio]);
1224
1225	zram->comp_algs[prio] = alg;
1226	}
1227
1228	static ssize_t __comp_algorithm_show(struct zram *zram, u32 prio, char *buf)
1229	{
1230	ssize_t sz;
1231
1232	down_read(sem: &zram->init_lock);
1233	sz = zcomp_available_show(comp: zram->comp_algs[prio], buf);
1234	up_read(sem: &zram->init_lock);
1235
1236	return sz;
1237	}
1238
1239	static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf)
1240	{
1241	char *compressor;
1242	size_t sz;
1243
1244	sz = strlen(buf);
1245	if (sz >= ZRAM_MAX_ALGO_NAME_SZ)
1246	return -E2BIG;
1247
1248	compressor = kstrdup(s: buf, GFP_KERNEL);
1249	if (!compressor)
1250	return -ENOMEM;
1251
1252	/* ignore trailing newline */
1253	if (sz > 0 && compressor[sz - 1] == '\n')
1254	compressor[sz - 1] = 0x00;
1255
1256	if (!zcomp_available_algorithm(comp: compressor)) {
1257	kfree(objp: compressor);
1258	return -EINVAL;
1259	}
1260
1261	down_write(sem: &zram->init_lock);
1262	if (init_done(zram)) {
1263	up_write(sem: &zram->init_lock);
1264	kfree(objp: compressor);
1265	pr_info("Can't change algorithm for initialized device\n");
1266	return -EBUSY;
1267	}
1268
1269	comp_algorithm_set(zram, prio, alg: compressor);
1270	up_write(sem: &zram->init_lock);
1271	return 0;
1272	}
1273
1274	static void comp_params_reset(struct zram *zram, u32 prio)
1275	{
1276	struct zcomp_params *params = &zram->params[prio];
1277
1278	vfree(addr: params->dict);
1279	params->level = ZCOMP_PARAM_NOT_SET;
1280	params->deflate.winbits = ZCOMP_PARAM_NOT_SET;
1281	params->dict_sz = 0;
1282	params->dict = NULL;
1283	}
1284
1285	static int comp_params_store(struct zram *zram, u32 prio, s32 level,
1286	const char *dict_path,
1287	struct deflate_params *deflate_params)
1288	{
1289	ssize_t sz = 0;
1290
1291	comp_params_reset(zram, prio);
1292
1293	if (dict_path) {
1294	sz = kernel_read_file_from_path(path: dict_path, offset: 0,
1295	buf: &zram->params[prio].dict,
1296	INT_MAX,
1297	NULL,
1298	id: READING_POLICY);
1299	if (sz < 0)
1300	return -EINVAL;
1301	}
1302
1303	zram->params[prio].dict_sz = sz;
1304	zram->params[prio].level = level;
1305	zram->params[prio].deflate.winbits = deflate_params->winbits;
1306	return 0;
1307	}
1308
1309	static ssize_t algorithm_params_store(struct device *dev,
1310	struct device_attribute *attr,
1311	const char *buf,
1312	size_t len)
1313	{
1314	s32 prio = ZRAM_PRIMARY_COMP, level = ZCOMP_PARAM_NOT_SET;
1315	char *args, *param, *val, *algo = NULL, *dict_path = NULL;
1316	struct deflate_params deflate_params;
1317	struct zram *zram = dev_to_zram(dev);
1318	int ret;
1319
1320	deflate_params.winbits = ZCOMP_PARAM_NOT_SET;
1321
1322	args = skip_spaces(buf);
1323	while (*args) {
1324	args = next_arg(args, param: &param, val: &val);
1325
1326	if (!val || !*val)
1327	return -EINVAL;
1328
1329	if (!strcmp(param, "priority")) {
1330	ret = kstrtoint(s: val, base: 10, res: &prio);
1331	if (ret)
1332	return ret;
1333	continue;
1334	}
1335
1336	if (!strcmp(param, "level")) {
1337	ret = kstrtoint(s: val, base: 10, res: &level);
1338	if (ret)
1339	return ret;
1340	continue;
1341	}
1342
1343	if (!strcmp(param, "algo")) {
1344	algo = val;
1345	continue;
1346	}
1347
1348	if (!strcmp(param, "dict")) {
1349	dict_path = val;
1350	continue;
1351	}
1352
1353	if (!strcmp(param, "deflate.winbits")) {
1354	ret = kstrtoint(s: val, base: 10, res: &deflate_params.winbits);
1355	if (ret)
1356	return ret;
1357	continue;
1358	}
1359	}
1360
1361	/* Lookup priority by algorithm name */
1362	if (algo) {
1363	s32 p;
1364
1365	prio = -EINVAL;
1366	for (p = ZRAM_PRIMARY_COMP; p < ZRAM_MAX_COMPS; p++) {
1367	if (!zram->comp_algs[p])
1368	continue;
1369
1370	if (!strcmp(zram->comp_algs[p], algo)) {
1371	prio = p;
1372	break;
1373	}
1374	}
1375	}
1376
1377	if (prio < ZRAM_PRIMARY_COMP || prio >= ZRAM_MAX_COMPS)
1378	return -EINVAL;
1379
1380	ret = comp_params_store(zram, prio, level, dict_path, deflate_params: &deflate_params);
1381	return ret ? ret : len;
1382	}
1383
1384	static ssize_t comp_algorithm_show(struct device *dev,
1385	struct device_attribute *attr,
1386	char *buf)
1387	{
1388	struct zram *zram = dev_to_zram(dev);
1389
1390	return __comp_algorithm_show(zram, ZRAM_PRIMARY_COMP, buf);
1391	}
1392
1393	static ssize_t comp_algorithm_store(struct device *dev,
1394	struct device_attribute *attr,
1395	const char *buf,
1396	size_t len)
1397	{
1398	struct zram *zram = dev_to_zram(dev);
1399	int ret;
1400
1401	ret = __comp_algorithm_store(zram, ZRAM_PRIMARY_COMP, buf);
1402	return ret ? ret : len;
1403	}
1404
1405	#ifdef CONFIG_ZRAM_MULTI_COMP
1406	static ssize_t recomp_algorithm_show(struct device *dev,
1407	struct device_attribute *attr,
1408	char *buf)
1409	{
1410	struct zram *zram = dev_to_zram(dev);
1411	ssize_t sz = 0;
1412	u32 prio;
1413
1414	for (prio = ZRAM_SECONDARY_COMP; prio < ZRAM_MAX_COMPS; prio++) {
1415	if (!zram->comp_algs[prio])
1416	continue;
1417
1418	sz += scnprintf(buf: buf + sz, PAGE_SIZE - sz - 2, fmt: "#%d: ", prio);
1419	sz += __comp_algorithm_show(zram, prio, buf: buf + sz);
1420	}
1421
1422	return sz;
1423	}
1424
1425	static ssize_t recomp_algorithm_store(struct device *dev,
1426	struct device_attribute *attr,
1427	const char *buf,
1428	size_t len)
1429	{
1430	struct zram *zram = dev_to_zram(dev);
1431	int prio = ZRAM_SECONDARY_COMP;
1432	char *args, *param, *val;
1433	char *alg = NULL;
1434	int ret;
1435
1436	args = skip_spaces(buf);
1437	while (*args) {
1438	args = next_arg(args, param: &param, val: &val);
1439
1440	if (!val || !*val)
1441	return -EINVAL;
1442
1443	if (!strcmp(param, "algo")) {
1444	alg = val;
1445	continue;
1446	}
1447
1448	if (!strcmp(param, "priority")) {
1449	ret = kstrtoint(s: val, base: 10, res: &prio);
1450	if (ret)
1451	return ret;
1452	continue;
1453	}
1454	}
1455
1456	if (!alg)
1457	return -EINVAL;
1458
1459	if (prio < ZRAM_SECONDARY_COMP || prio >= ZRAM_MAX_COMPS)
1460	return -EINVAL;
1461
1462	ret = __comp_algorithm_store(zram, prio, buf: alg);
1463	return ret ? ret : len;
1464	}
1465	#endif
1466
1467	static ssize_t compact_store(struct device *dev,
1468	struct device_attribute *attr, const char *buf, size_t len)
1469	{
1470	struct zram *zram = dev_to_zram(dev);
1471
1472	down_read(sem: &zram->init_lock);
1473	if (!init_done(zram)) {
1474	up_read(sem: &zram->init_lock);
1475	return -EINVAL;
1476	}
1477
1478	zs_compact(pool: zram->mem_pool);
1479	up_read(sem: &zram->init_lock);
1480
1481	return len;
1482	}
1483
1484	static ssize_t io_stat_show(struct device *dev,
1485	struct device_attribute *attr, char *buf)
1486	{
1487	struct zram *zram = dev_to_zram(dev);
1488	ssize_t ret;
1489
1490	down_read(sem: &zram->init_lock);
1491	ret = scnprintf(buf, PAGE_SIZE,
1492	fmt: "%8llu %8llu 0 %8llu\n",
1493	(u64)atomic64_read(v: &zram->stats.failed_reads),
1494	(u64)atomic64_read(v: &zram->stats.failed_writes),
1495	(u64)atomic64_read(v: &zram->stats.notify_free));
1496	up_read(sem: &zram->init_lock);
1497
1498	return ret;
1499	}
1500
1501	static ssize_t mm_stat_show(struct device *dev,
1502	struct device_attribute *attr, char *buf)
1503	{
1504	struct zram *zram = dev_to_zram(dev);
1505	struct zs_pool_stats pool_stats;
1506	u64 orig_size, mem_used = 0;
1507	long max_used;
1508	ssize_t ret;
1509
1510	memset(&pool_stats, 0x00, sizeof(struct zs_pool_stats));
1511
1512	down_read(sem: &zram->init_lock);
1513	if (init_done(zram)) {
1514	mem_used = zs_get_total_pages(pool: zram->mem_pool);
1515	zs_pool_stats(pool: zram->mem_pool, stats: &pool_stats);
1516	}
1517
1518	orig_size = atomic64_read(v: &zram->stats.pages_stored);
1519	max_used = atomic_long_read(v: &zram->stats.max_used_pages);
1520
1521	ret = scnprintf(buf, PAGE_SIZE,
1522	fmt: "%8llu %8llu %8llu %8lu %8ld %8llu %8lu %8llu %8llu\n",
1523	orig_size << PAGE_SHIFT,
1524	(u64)atomic64_read(v: &zram->stats.compr_data_size),
1525	mem_used << PAGE_SHIFT,
1526	zram->limit_pages << PAGE_SHIFT,
1527	max_used << PAGE_SHIFT,
1528	(u64)atomic64_read(v: &zram->stats.same_pages),
1529	atomic_long_read(v: &pool_stats.pages_compacted),
1530	(u64)atomic64_read(v: &zram->stats.huge_pages),
1531	(u64)atomic64_read(v: &zram->stats.huge_pages_since));
1532	up_read(sem: &zram->init_lock);
1533
1534	return ret;
1535	}
1536
1537	#ifdef CONFIG_ZRAM_WRITEBACK
1538	#define FOUR_K(x) ((x) * (1 << (PAGE_SHIFT - 12)))
1539	static ssize_t bd_stat_show(struct device *dev,
1540	struct device_attribute *attr, char *buf)
1541	{
1542	struct zram *zram = dev_to_zram(dev);
1543	ssize_t ret;
1544
1545	down_read(sem: &zram->init_lock);
1546	ret = scnprintf(buf, PAGE_SIZE,
1547	fmt: "%8llu %8llu %8llu\n",
1548	FOUR_K((u64)atomic64_read(&zram->stats.bd_count)),
1549	FOUR_K((u64)atomic64_read(&zram->stats.bd_reads)),
1550	FOUR_K((u64)atomic64_read(&zram->stats.bd_writes)));
1551	up_read(sem: &zram->init_lock);
1552
1553	return ret;
1554	}
1555	#endif
1556
1557	static ssize_t debug_stat_show(struct device *dev,
1558	struct device_attribute *attr, char *buf)
1559	{
1560	int version = 1;
1561	struct zram *zram = dev_to_zram(dev);
1562	ssize_t ret;
1563
1564	down_read(sem: &zram->init_lock);
1565	ret = scnprintf(buf, PAGE_SIZE,
1566	fmt: "version: %d\n0 %8llu\n",
1567	version,
1568	(u64)atomic64_read(v: &zram->stats.miss_free));
1569	up_read(sem: &zram->init_lock);
1570
1571	return ret;
1572	}
1573
1574	static DEVICE_ATTR_RO(io_stat);
1575	static DEVICE_ATTR_RO(mm_stat);
1576	#ifdef CONFIG_ZRAM_WRITEBACK
1577	static DEVICE_ATTR_RO(bd_stat);
1578	#endif
1579	static DEVICE_ATTR_RO(debug_stat);
1580
1581	static void zram_meta_free(struct zram *zram, u64 disksize)
1582	{
1583	size_t num_pages = disksize >> PAGE_SHIFT;
1584	size_t index;
1585
1586	if (!zram->table)
1587	return;
1588
1589	/* Free all pages that are still in this zram device */
1590	for (index = 0; index < num_pages; index++)
1591	zram_free_page(zram, index);
1592
1593	zs_destroy_pool(pool: zram->mem_pool);
1594	vfree(addr: zram->table);
1595	zram->table = NULL;
1596	}
1597
1598	static bool zram_meta_alloc(struct zram *zram, u64 disksize)
1599	{
1600	size_t num_pages, index;
1601
1602	num_pages = disksize >> PAGE_SHIFT;
1603	zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table)));
1604	if (!zram->table)
1605	return false;
1606
1607	zram->mem_pool = zs_create_pool(name: zram->disk->disk_name);
1608	if (!zram->mem_pool) {
1609	vfree(addr: zram->table);
1610	zram->table = NULL;
1611	return false;
1612	}
1613
1614	if (!huge_class_size)
1615	huge_class_size = zs_huge_class_size(pool: zram->mem_pool);
1616
1617	for (index = 0; index < num_pages; index++)
1618	zram_slot_lock_init(zram, index);
1619
1620	return true;
1621	}
1622
1623	static void zram_free_page(struct zram *zram, size_t index)
1624	{
1625	unsigned long handle;
1626
1627	#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
1628	zram->table[index].ac_time = 0;
1629	#endif
1630
1631	zram_clear_flag(zram, index, flag: ZRAM_IDLE);
1632	zram_clear_flag(zram, index, flag: ZRAM_INCOMPRESSIBLE);
1633	zram_clear_flag(zram, index, flag: ZRAM_PP_SLOT);
1634	zram_set_priority(zram, index, prio: 0);
1635
1636	if (zram_test_flag(zram, index, flag: ZRAM_HUGE)) {
1637	zram_clear_flag(zram, index, flag: ZRAM_HUGE);
1638	atomic64_dec(v: &zram->stats.huge_pages);
1639	}
1640
1641	if (zram_test_flag(zram, index, flag: ZRAM_WB)) {
1642	zram_clear_flag(zram, index, flag: ZRAM_WB);
1643	free_block_bdev(zram, blk_idx: zram_get_handle(zram, index));
1644	goto out;
1645	}
1646
1647	/*
1648	* No memory is allocated for same element filled pages.
1649	* Simply clear same page flag.
1650	*/
1651	if (zram_test_flag(zram, index, flag: ZRAM_SAME)) {
1652	zram_clear_flag(zram, index, flag: ZRAM_SAME);
1653	atomic64_dec(v: &zram->stats.same_pages);
1654	goto out;
1655	}
1656
1657	handle = zram_get_handle(zram, index);
1658	if (!handle)
1659	return;
1660
1661	zs_free(pool: zram->mem_pool, obj: handle);
1662
1663	atomic64_sub(i: zram_get_obj_size(zram, index),
1664	v: &zram->stats.compr_data_size);
1665	out:
1666	atomic64_dec(v: &zram->stats.pages_stored);
1667	zram_set_handle(zram, index, handle: 0);
1668	zram_set_obj_size(zram, index, size: 0);
1669	}
1670
1671	static int read_same_filled_page(struct zram *zram, struct page *page,
1672	u32 index)
1673	{
1674	void *mem;
1675
1676	mem = kmap_local_page(page);
1677	zram_fill_page(ptr: mem, PAGE_SIZE, value: zram_get_handle(zram, index));
1678	kunmap_local(mem);
1679	return 0;
1680	}
1681
1682	static int read_incompressible_page(struct zram *zram, struct page *page,
1683	u32 index)
1684	{
1685	unsigned long handle;
1686	void *src, *dst;
1687
1688	handle = zram_get_handle(zram, index);
1689	src = zs_obj_read_begin(pool: zram->mem_pool, handle, NULL);
1690	dst = kmap_local_page(page);
1691	copy_page(to: dst, from: src);
1692	kunmap_local(dst);
1693	zs_obj_read_end(pool: zram->mem_pool, handle, handle_mem: src);
1694
1695	return 0;
1696	}
1697
1698	static int read_compressed_page(struct zram *zram, struct page *page, u32 index)
1699	{
1700	struct zcomp_strm *zstrm;
1701	unsigned long handle;
1702	unsigned int size;
1703	void *src, *dst;
1704	int ret, prio;
1705
1706	handle = zram_get_handle(zram, index);
1707	size = zram_get_obj_size(zram, index);
1708	prio = zram_get_priority(zram, index);
1709
1710	zstrm = zcomp_stream_get(comp: zram->comps[prio]);
1711	src = zs_obj_read_begin(pool: zram->mem_pool, handle, local_copy: zstrm->local_copy);
1712	dst = kmap_local_page(page);
1713	ret = zcomp_decompress(comp: zram->comps[prio], zstrm, src, src_len: size, dst);
1714	kunmap_local(dst);
1715	zs_obj_read_end(pool: zram->mem_pool, handle, handle_mem: src);
1716	zcomp_stream_put(zstrm);
1717
1718	return ret;
1719	}
1720
1721	/*
1722	* Reads (decompresses if needed) a page from zspool (zsmalloc).
1723	* Corresponding ZRAM slot should be locked.
1724	*/
1725	static int zram_read_from_zspool(struct zram *zram, struct page *page,
1726	u32 index)
1727	{
1728	if (zram_test_flag(zram, index, flag: ZRAM_SAME) ||
1729	!zram_get_handle(zram, index))
1730	return read_same_filled_page(zram, page, index);
1731
1732	if (!zram_test_flag(zram, index, flag: ZRAM_HUGE))
1733	return read_compressed_page(zram, page, index);
1734	else
1735	return read_incompressible_page(zram, page, index);
1736	}
1737
1738	static int zram_read_page(struct zram *zram, struct page *page, u32 index,
1739	struct bio *parent)
1740	{
1741	int ret;
1742
1743	zram_slot_lock(zram, index);
1744	if (!zram_test_flag(zram, index, flag: ZRAM_WB)) {
1745	/* Slot should be locked through out the function call */
1746	ret = zram_read_from_zspool(zram, page, index);
1747	zram_slot_unlock(zram, index);
1748	} else {
1749	/*
1750	* The slot should be unlocked before reading from the backing
1751	* device.
1752	*/
1753	zram_slot_unlock(zram, index);
1754
1755	ret = read_from_bdev(zram, page, entry: zram_get_handle(zram, index),
1756	parent);
1757	}
1758
1759	/* Should NEVER happen. Return bio error if it does. */
1760	if (WARN_ON(ret < 0))
1761	pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
1762
1763	return ret;
1764	}
1765
1766	/*
1767	* Use a temporary buffer to decompress the page, as the decompressor
1768	* always expects a full page for the output.
1769	*/
1770	static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec,
1771	u32 index, int offset)
1772	{
1773	struct page *page = alloc_page(GFP_NOIO);
1774	int ret;
1775
1776	if (!page)
1777	return -ENOMEM;
1778	ret = zram_read_page(zram, page, index, NULL);
1779	if (likely(!ret))
1780	memcpy_to_bvec(bvec, page_address(page) + offset);
1781	__free_page(page);
1782	return ret;
1783	}
1784
1785	static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
1786	u32 index, int offset, struct bio *bio)
1787	{
1788	if (is_partial_io(bvec))
1789	return zram_bvec_read_partial(zram, bvec, index, offset);
1790	return zram_read_page(zram, page: bvec->bv_page, index, parent: bio);
1791	}
1792
1793	static int write_same_filled_page(struct zram *zram, unsigned long fill,
1794	u32 index)
1795	{
1796	zram_slot_lock(zram, index);
1797	zram_set_flag(zram, index, flag: ZRAM_SAME);
1798	zram_set_handle(zram, index, handle: fill);
1799	zram_slot_unlock(zram, index);
1800
1801	atomic64_inc(v: &zram->stats.same_pages);
1802	atomic64_inc(v: &zram->stats.pages_stored);
1803
1804	return 0;
1805	}
1806
1807	static int write_incompressible_page(struct zram *zram, struct page *page,
1808	u32 index)
1809	{
1810	unsigned long handle;
1811	void *src;
1812
1813	/*
1814	* This function is called from preemptible context so we don't need
1815	* to do optimistic and fallback to pessimistic handle allocation,
1816	* like we do for compressible pages.
1817	*/
1818	handle = zs_malloc(pool: zram->mem_pool, PAGE_SIZE,
1819	GFP_NOIO | __GFP_NOWARN |
1820	__GFP_HIGHMEM | __GFP_MOVABLE, nid: page_to_nid(page));
1821	if (IS_ERR_VALUE(handle))
1822	return PTR_ERR(ptr: (void *)handle);
1823
1824	if (!zram_can_store_page(zram)) {
1825	zs_free(pool: zram->mem_pool, obj: handle);
1826	return -ENOMEM;
1827	}
1828
1829	src = kmap_local_page(page);
1830	zs_obj_write(pool: zram->mem_pool, handle, handle_mem: src, PAGE_SIZE);
1831	kunmap_local(src);
1832
1833	zram_slot_lock(zram, index);
1834	zram_set_flag(zram, index, flag: ZRAM_HUGE);
1835	zram_set_handle(zram, index, handle);
1836	zram_set_obj_size(zram, index, PAGE_SIZE);
1837	zram_slot_unlock(zram, index);
1838
1839	atomic64_add(PAGE_SIZE, v: &zram->stats.compr_data_size);
1840	atomic64_inc(v: &zram->stats.huge_pages);
1841	atomic64_inc(v: &zram->stats.huge_pages_since);
1842	atomic64_inc(v: &zram->stats.pages_stored);
1843
1844	return 0;
1845	}
1846
1847	static int zram_write_page(struct zram *zram, struct page *page, u32 index)
1848	{
1849	int ret = 0;
1850	unsigned long handle;
1851	unsigned int comp_len;
1852	void *mem;
1853	struct zcomp_strm *zstrm;
1854	unsigned long element;
1855	bool same_filled;
1856
1857	/* First, free memory allocated to this slot (if any) */
1858	zram_slot_lock(zram, index);
1859	zram_free_page(zram, index);
1860	zram_slot_unlock(zram, index);
1861
1862	mem = kmap_local_page(page);
1863	same_filled = page_same_filled(ptr: mem, element: &element);
1864	kunmap_local(mem);
1865	if (same_filled)
1866	return write_same_filled_page(zram, fill: element, index);
1867
1868	zstrm = zcomp_stream_get(comp: zram->comps[ZRAM_PRIMARY_COMP]);
1869	mem = kmap_local_page(page);
1870	ret = zcomp_compress(comp: zram->comps[ZRAM_PRIMARY_COMP], zstrm,
1871	src: mem, dst_len: &comp_len);
1872	kunmap_local(mem);
1873
1874	if (unlikely(ret)) {
1875	zcomp_stream_put(zstrm);
1876	pr_err("Compression failed! err=%d\n", ret);
1877	return ret;
1878	}
1879
1880	if (comp_len >= huge_class_size) {
1881	zcomp_stream_put(zstrm);
1882	return write_incompressible_page(zram, page, index);
1883	}
1884
1885	handle = zs_malloc(pool: zram->mem_pool, size: comp_len,
1886	GFP_NOIO | __GFP_NOWARN |
1887	__GFP_HIGHMEM | __GFP_MOVABLE, nid: page_to_nid(page));
1888	if (IS_ERR_VALUE(handle)) {
1889	zcomp_stream_put(zstrm);
1890	return PTR_ERR(ptr: (void *)handle);
1891	}
1892
1893	if (!zram_can_store_page(zram)) {
1894	zcomp_stream_put(zstrm);
1895	zs_free(pool: zram->mem_pool, obj: handle);
1896	return -ENOMEM;
1897	}
1898
1899	zs_obj_write(pool: zram->mem_pool, handle, handle_mem: zstrm->buffer, mem_len: comp_len);
1900	zcomp_stream_put(zstrm);
1901
1902	zram_slot_lock(zram, index);
1903	zram_set_handle(zram, index, handle);
1904	zram_set_obj_size(zram, index, size: comp_len);
1905	zram_slot_unlock(zram, index);
1906
1907	/* Update stats */
1908	atomic64_inc(v: &zram->stats.pages_stored);
1909	atomic64_add(i: comp_len, v: &zram->stats.compr_data_size);
1910
1911	return ret;
1912	}
1913
1914	/*
1915	* This is a partial IO. Read the full page before writing the changes.
1916	*/
1917	static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec,
1918	u32 index, int offset, struct bio *bio)
1919	{
1920	struct page *page = alloc_page(GFP_NOIO);
1921	int ret;
1922
1923	if (!page)
1924	return -ENOMEM;
1925
1926	ret = zram_read_page(zram, page, index, parent: bio);
1927	if (!ret) {
1928	memcpy_from_bvec(page_address(page) + offset, bvec);
1929	ret = zram_write_page(zram, page, index);
1930	}
1931	__free_page(page);
1932	return ret;
1933	}
1934
1935	static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
1936	u32 index, int offset, struct bio *bio)
1937	{
1938	if (is_partial_io(bvec))
1939	return zram_bvec_write_partial(zram, bvec, index, offset, bio);
1940	return zram_write_page(zram, page: bvec->bv_page, index);
1941	}
1942
1943	#ifdef CONFIG_ZRAM_MULTI_COMP
1944	#define RECOMPRESS_IDLE (1 << 0)
1945	#define RECOMPRESS_HUGE (1 << 1)
1946
1947	static int scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio_max,
1948	struct zram_pp_ctl *ctl)
1949	{
1950	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
1951	unsigned long index;
1952
1953	for (index = 0; index < nr_pages; index++) {
1954	bool ok = true;
1955
1956	zram_slot_lock(zram, index);
1957	if (!zram_allocated(zram, index))
1958	goto next;
1959
1960	if (mode & RECOMPRESS_IDLE &&
1961	!zram_test_flag(zram, index, flag: ZRAM_IDLE))
1962	goto next;
1963
1964	if (mode & RECOMPRESS_HUGE &&
1965	!zram_test_flag(zram, index, flag: ZRAM_HUGE))
1966	goto next;
1967
1968	if (zram_test_flag(zram, index, flag: ZRAM_WB) ||
1969	zram_test_flag(zram, index, flag: ZRAM_SAME) ||
1970	zram_test_flag(zram, index, flag: ZRAM_INCOMPRESSIBLE))
1971	goto next;
1972
1973	/* Already compressed with same of higher priority */
1974	if (zram_get_priority(zram, index) + 1 >= prio_max)
1975	goto next;
1976
1977	ok = place_pp_slot(zram, ctl, index);
1978	next:
1979	zram_slot_unlock(zram, index);
1980	if (!ok)
1981	break;
1982	}
1983
1984	return 0;
1985	}
1986
1987	/*
1988	* This function will decompress (unless it's ZRAM_HUGE) the page and then
1989	* attempt to compress it using provided compression algorithm priority
1990	* (which is potentially more effective).
1991	*
1992	* Corresponding ZRAM slot should be locked.
1993	*/
1994	static int recompress_slot(struct zram *zram, u32 index, struct page *page,
1995	u64 *num_recomp_pages, u32 threshold, u32 prio,
1996	u32 prio_max)
1997	{
1998	struct zcomp_strm *zstrm = NULL;
1999	unsigned long handle_old;
2000	unsigned long handle_new;
2001	unsigned int comp_len_old;
2002	unsigned int comp_len_new;
2003	unsigned int class_index_old;
2004	unsigned int class_index_new;
2005	void *src;
2006	int ret = 0;
2007
2008	handle_old = zram_get_handle(zram, index);
2009	if (!handle_old)
2010	return -EINVAL;
2011
2012	comp_len_old = zram_get_obj_size(zram, index);
2013	/*
2014	* Do not recompress objects that are already "small enough".
2015	*/
2016	if (comp_len_old < threshold)
2017	return 0;
2018
2019	ret = zram_read_from_zspool(zram, page, index);
2020	if (ret)
2021	return ret;
2022
2023	/*
2024	* We touched this entry so mark it as non-IDLE. This makes sure that
2025	* we don't preserve IDLE flag and don't incorrectly pick this entry
2026	* for different post-processing type (e.g. writeback).
2027	*/
2028	zram_clear_flag(zram, index, flag: ZRAM_IDLE);
2029
2030	class_index_old = zs_lookup_class_index(pool: zram->mem_pool, size: comp_len_old);
2031
2032	prio = max(prio, zram_get_priority(zram, index) + 1);
2033	/*
2034	* Recompression slots scan should not select slots that are
2035	* already compressed with a higher priority algorithm, but
2036	* just in case
2037	*/
2038	if (prio >= prio_max)
2039	return 0;
2040
2041	/*
2042	* Iterate the secondary comp algorithms list (in order of priority)
2043	* and try to recompress the page.
2044	*/
2045	for (; prio < prio_max; prio++) {
2046	if (!zram->comps[prio])
2047	continue;
2048
2049	zstrm = zcomp_stream_get(comp: zram->comps[prio]);
2050	src = kmap_local_page(page);
2051	ret = zcomp_compress(comp: zram->comps[prio], zstrm,
2052	src, dst_len: &comp_len_new);
2053	kunmap_local(src);
2054
2055	if (ret) {
2056	zcomp_stream_put(zstrm);
2057	zstrm = NULL;
2058	break;
2059	}
2060
2061	class_index_new = zs_lookup_class_index(pool: zram->mem_pool,
2062	size: comp_len_new);
2063
2064	/* Continue until we make progress */
2065	if (class_index_new >= class_index_old ||
2066	(threshold && comp_len_new >= threshold)) {
2067	zcomp_stream_put(zstrm);
2068	zstrm = NULL;
2069	continue;
2070	}
2071
2072	/* Recompression was successful so break out */
2073	break;
2074	}
2075
2076	/*
2077	* Decrement the limit (if set) on pages we can recompress, even
2078	* when current recompression was unsuccessful or did not compress
2079	* the page below the threshold, because we still spent resources
2080	* on it.
2081	*/
2082	if (*num_recomp_pages)
2083	*num_recomp_pages -= 1;
2084
2085	/* Compression error */
2086	if (ret)
2087	return ret;
2088
2089	if (!zstrm) {
2090	/*
2091	* Secondary algorithms failed to re-compress the page
2092	* in a way that would save memory.
2093	*
2094	* Mark the object incompressible if the max-priority
2095	* algorithm couldn't re-compress it.
2096	*/
2097	if (prio < zram->num_active_comps)
2098	return 0;
2099	zram_set_flag(zram, index, flag: ZRAM_INCOMPRESSIBLE);
2100	return 0;
2101	}
2102
2103	/*
2104	* We are holding per-CPU stream mutex and entry lock so better
2105	* avoid direct reclaim. Allocation error is not fatal since
2106	* we still have the old object in the mem_pool.
2107	*
2108	* XXX: technically, the node we really want here is the node that holds
2109	* the original compressed data. But that would require us to modify
2110	* zsmalloc API to return this information. For now, we will make do with
2111	* the node of the page allocated for recompression.
2112	*/
2113	handle_new = zs_malloc(pool: zram->mem_pool, size: comp_len_new,
2114	GFP_NOIO | __GFP_NOWARN |
2115	__GFP_HIGHMEM | __GFP_MOVABLE, nid: page_to_nid(page));
2116	if (IS_ERR_VALUE(handle_new)) {
2117	zcomp_stream_put(zstrm);
2118	return PTR_ERR(ptr: (void *)handle_new);
2119	}
2120
2121	zs_obj_write(pool: zram->mem_pool, handle: handle_new, handle_mem: zstrm->buffer, mem_len: comp_len_new);
2122	zcomp_stream_put(zstrm);
2123
2124	zram_free_page(zram, index);
2125	zram_set_handle(zram, index, handle: handle_new);
2126	zram_set_obj_size(zram, index, size: comp_len_new);
2127	zram_set_priority(zram, index, prio);
2128
2129	atomic64_add(i: comp_len_new, v: &zram->stats.compr_data_size);
2130	atomic64_inc(v: &zram->stats.pages_stored);
2131
2132	return 0;
2133	}
2134
2135	static ssize_t recompress_store(struct device *dev,
2136	struct device_attribute *attr,
2137	const char *buf, size_t len)
2138	{
2139	struct zram *zram = dev_to_zram(dev);
2140	char *args, *param, *val, *algo = NULL;
2141	u64 num_recomp_pages = ULLONG_MAX;
2142	struct zram_pp_ctl *ctl = NULL;
2143	struct zram_pp_slot *pps;
2144	u32 mode = 0, threshold = 0;
2145	u32 prio, prio_max;
2146	struct page *page = NULL;
2147	ssize_t ret;
2148
2149	prio = ZRAM_SECONDARY_COMP;
2150	prio_max = zram->num_active_comps;
2151
2152	args = skip_spaces(buf);
2153	while (*args) {
2154	args = next_arg(args, param: &param, val: &val);
2155
2156	if (!val || !*val)
2157	return -EINVAL;
2158
2159	if (!strcmp(param, "type")) {
2160	if (!strcmp(val, "idle"))
2161	mode = RECOMPRESS_IDLE;
2162	if (!strcmp(val, "huge"))
2163	mode = RECOMPRESS_HUGE;
2164	if (!strcmp(val, "huge_idle"))
2165	mode = RECOMPRESS_IDLE | RECOMPRESS_HUGE;
2166	continue;
2167	}
2168
2169	if (!strcmp(param, "max_pages")) {
2170	/*
2171	* Limit the number of entries (pages) we attempt to
2172	* recompress.
2173	*/
2174	ret = kstrtoull(s: val, base: 10, res: &num_recomp_pages);
2175	if (ret)
2176	return ret;
2177	continue;
2178	}
2179
2180	if (!strcmp(param, "threshold")) {
2181	/*
2182	* We will re-compress only idle objects equal or
2183	* greater in size than watermark.
2184	*/
2185	ret = kstrtouint(s: val, base: 10, res: &threshold);
2186	if (ret)
2187	return ret;
2188	continue;
2189	}
2190
2191	if (!strcmp(param, "algo")) {
2192	algo = val;
2193	continue;
2194	}
2195
2196	if (!strcmp(param, "priority")) {
2197	ret = kstrtouint(s: val, base: 10, res: &prio);
2198	if (ret)
2199	return ret;
2200
2201	if (prio == ZRAM_PRIMARY_COMP)
2202	prio = ZRAM_SECONDARY_COMP;
2203
2204	prio_max = prio + 1;
2205	continue;
2206	}
2207	}
2208
2209	if (threshold >= huge_class_size)
2210	return -EINVAL;
2211
2212	down_read(sem: &zram->init_lock);
2213	if (!init_done(zram)) {
2214	ret = -EINVAL;
2215	goto release_init_lock;
2216	}
2217
2218	/* Do not permit concurrent post-processing actions. */
2219	if (atomic_xchg(v: &zram->pp_in_progress, new: 1)) {
2220	up_read(sem: &zram->init_lock);
2221	return -EAGAIN;
2222	}
2223
2224	if (algo) {
2225	bool found = false;
2226
2227	for (; prio < ZRAM_MAX_COMPS; prio++) {
2228	if (!zram->comp_algs[prio])
2229	continue;
2230
2231	if (!strcmp(zram->comp_algs[prio], algo)) {
2232	prio_max = prio + 1;
2233	found = true;
2234	break;
2235	}
2236	}
2237
2238	if (!found) {
2239	ret = -EINVAL;
2240	goto release_init_lock;
2241	}
2242	}
2243
2244	prio_max = min(prio_max, (u32)zram->num_active_comps);
2245	if (prio >= prio_max) {
2246	ret = -EINVAL;
2247	goto release_init_lock;
2248	}
2249
2250	page = alloc_page(GFP_KERNEL);
2251	if (!page) {
2252	ret = -ENOMEM;
2253	goto release_init_lock;
2254	}
2255
2256	ctl = init_pp_ctl();
2257	if (!ctl) {
2258	ret = -ENOMEM;
2259	goto release_init_lock;
2260	}
2261
2262	scan_slots_for_recompress(zram, mode, prio_max, ctl);
2263
2264	ret = len;
2265	while ((pps = select_pp_slot(ctl))) {
2266	int err = 0;
2267
2268	if (!num_recomp_pages)
2269	break;
2270
2271	zram_slot_lock(zram, index: pps->index);
2272	if (!zram_test_flag(zram, index: pps->index, flag: ZRAM_PP_SLOT))
2273	goto next;
2274
2275	err = recompress_slot(zram, index: pps->index, page,
2276	num_recomp_pages: &num_recomp_pages, threshold,
2277	prio, prio_max);
2278	next:
2279	zram_slot_unlock(zram, index: pps->index);
2280	release_pp_slot(zram, pps);
2281
2282	if (err) {
2283	ret = err;
2284	break;
2285	}
2286
2287	cond_resched();
2288	}
2289
2290	release_init_lock:
2291	if (page)
2292	__free_page(page);
2293	release_pp_ctl(zram, ctl);
2294	atomic_set(v: &zram->pp_in_progress, i: 0);
2295	up_read(sem: &zram->init_lock);
2296	return ret;
2297	}
2298	#endif
2299
2300	static void zram_bio_discard(struct zram *zram, struct bio *bio)
2301	{
2302	size_t n = bio->bi_iter.bi_size;
2303	u32 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
2304	u32 offset = (bio->bi_iter.bi_sector & (SECTORS_PER_PAGE - 1)) <<
2305	SECTOR_SHIFT;
2306
2307	/*
2308	* zram manages data in physical block size units. Because logical block
2309	* size isn't identical with physical block size on some arch, we
2310	* could get a discard request pointing to a specific offset within a
2311	* certain physical block. Although we can handle this request by
2312	* reading that physiclal block and decompressing and partially zeroing
2313	* and re-compressing and then re-storing it, this isn't reasonable
2314	* because our intent with a discard request is to save memory. So
2315	* skipping this logical block is appropriate here.
2316	*/
2317	if (offset) {
2318	if (n <= (PAGE_SIZE - offset))
2319	return;
2320
2321	n -= (PAGE_SIZE - offset);
2322	index++;
2323	}
2324
2325	while (n >= PAGE_SIZE) {
2326	zram_slot_lock(zram, index);
2327	zram_free_page(zram, index);
2328	zram_slot_unlock(zram, index);
2329	atomic64_inc(v: &zram->stats.notify_free);
2330	index++;
2331	n -= PAGE_SIZE;
2332	}
2333
2334	bio_endio(bio);
2335	}
2336
2337	static void zram_bio_read(struct zram *zram, struct bio *bio)
2338	{
2339	unsigned long start_time = bio_start_io_acct(bio);
2340	struct bvec_iter iter = bio->bi_iter;
2341
2342	do {
2343	u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
2344	u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) <<
2345	SECTOR_SHIFT;
2346	struct bio_vec bv = bio_iter_iovec(bio, iter);
2347
2348	bv.bv_len = min_t(u32, bv.bv_len, PAGE_SIZE - offset);
2349
2350	if (zram_bvec_read(zram, bvec: &bv, index, offset, bio) < 0) {
2351	atomic64_inc(v: &zram->stats.failed_reads);
2352	bio->bi_status = BLK_STS_IOERR;
2353	break;
2354	}
2355	flush_dcache_page(page: bv.bv_page);
2356
2357	zram_slot_lock(zram, index);
2358	zram_accessed(zram, index);
2359	zram_slot_unlock(zram, index);
2360
2361	bio_advance_iter_single(bio, iter: &iter, bytes: bv.bv_len);
2362	} while (iter.bi_size);
2363
2364	bio_end_io_acct(bio, start_time);
2365	bio_endio(bio);
2366	}
2367
2368	static void zram_bio_write(struct zram *zram, struct bio *bio)
2369	{
2370	unsigned long start_time = bio_start_io_acct(bio);
2371	struct bvec_iter iter = bio->bi_iter;
2372
2373	do {
2374	u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
2375	u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) <<
2376	SECTOR_SHIFT;
2377	struct bio_vec bv = bio_iter_iovec(bio, iter);
2378
2379	bv.bv_len = min_t(u32, bv.bv_len, PAGE_SIZE - offset);
2380
2381	if (zram_bvec_write(zram, bvec: &bv, index, offset, bio) < 0) {
2382	atomic64_inc(v: &zram->stats.failed_writes);
2383	bio->bi_status = BLK_STS_IOERR;
2384	break;
2385	}
2386
2387	zram_slot_lock(zram, index);
2388	zram_accessed(zram, index);
2389	zram_slot_unlock(zram, index);
2390
2391	bio_advance_iter_single(bio, iter: &iter, bytes: bv.bv_len);
2392	} while (iter.bi_size);
2393
2394	bio_end_io_acct(bio, start_time);
2395	bio_endio(bio);
2396	}
2397
2398	/*
2399	* Handler function for all zram I/O requests.
2400	*/
2401	static void zram_submit_bio(struct bio *bio)
2402	{
2403	struct zram *zram = bio->bi_bdev->bd_disk->private_data;
2404
2405	switch (bio_op(bio)) {
2406	case REQ_OP_READ:
2407	zram_bio_read(zram, bio);
2408	break;
2409	case REQ_OP_WRITE:
2410	zram_bio_write(zram, bio);
2411	break;
2412	case REQ_OP_DISCARD:
2413	case REQ_OP_WRITE_ZEROES:
2414	zram_bio_discard(zram, bio);
2415	break;
2416	default:
2417	WARN_ON_ONCE(1);
2418	bio_endio(bio);
2419	}
2420	}
2421
2422	static void zram_slot_free_notify(struct block_device *bdev,
2423	unsigned long index)
2424	{
2425	struct zram *zram;
2426
2427	zram = bdev->bd_disk->private_data;
2428
2429	atomic64_inc(v: &zram->stats.notify_free);
2430	if (!zram_slot_trylock(zram, index)) {
2431	atomic64_inc(v: &zram->stats.miss_free);
2432	return;
2433	}
2434
2435	zram_free_page(zram, index);
2436	zram_slot_unlock(zram, index);
2437	}
2438
2439	static void zram_comp_params_reset(struct zram *zram)
2440	{
2441	u32 prio;
2442
2443	for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++) {
2444	comp_params_reset(zram, prio);
2445	}
2446	}
2447
2448	static void zram_destroy_comps(struct zram *zram)
2449	{
2450	u32 prio;
2451
2452	for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++) {
2453	struct zcomp *comp = zram->comps[prio];
2454
2455	zram->comps[prio] = NULL;
2456	if (!comp)
2457	continue;
2458	zcomp_destroy(comp);
2459	zram->num_active_comps--;
2460	}
2461
2462	for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++) {
2463	/* Do not free statically defined compression algorithms */
2464	if (zram->comp_algs[prio] != default_compressor)
2465	kfree(objp: zram->comp_algs[prio]);
2466	zram->comp_algs[prio] = NULL;
2467	}
2468
2469	zram_comp_params_reset(zram);
2470	}
2471
2472	static void zram_reset_device(struct zram *zram)
2473	{
2474	down_write(sem: &zram->init_lock);
2475
2476	zram->limit_pages = 0;
2477
2478	set_capacity_and_notify(disk: zram->disk, size: 0);
2479	part_stat_set_all(part: zram->disk->part0, value: 0);
2480
2481	/* I/O operation under all of CPU are done so let's free */
2482	zram_meta_free(zram, disksize: zram->disksize);
2483	zram->disksize = 0;
2484	zram_destroy_comps(zram);
2485	memset(&zram->stats, 0, sizeof(zram->stats));
2486	atomic_set(v: &zram->pp_in_progress, i: 0);
2487	reset_bdev(zram);
2488
2489	comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, alg: default_compressor);
2490	up_write(sem: &zram->init_lock);
2491	}
2492
2493	static ssize_t disksize_store(struct device *dev,
2494	struct device_attribute *attr, const char *buf, size_t len)
2495	{
2496	u64 disksize;
2497	struct zcomp *comp;
2498	struct zram *zram = dev_to_zram(dev);
2499	int err;
2500	u32 prio;
2501
2502	disksize = memparse(ptr: buf, NULL);
2503	if (!disksize)
2504	return -EINVAL;
2505
2506	down_write(sem: &zram->init_lock);
2507	if (init_done(zram)) {
2508	pr_info("Cannot change disksize for initialized device\n");
2509	err = -EBUSY;
2510	goto out_unlock;
2511	}
2512
2513	disksize = PAGE_ALIGN(disksize);
2514	if (!zram_meta_alloc(zram, disksize)) {
2515	err = -ENOMEM;
2516	goto out_unlock;
2517	}
2518
2519	for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++) {
2520	if (!zram->comp_algs[prio])
2521	continue;
2522
2523	comp = zcomp_create(alg: zram->comp_algs[prio],
2524	params: &zram->params[prio]);
2525	if (IS_ERR(ptr: comp)) {
2526	pr_err("Cannot initialise %s compressing backend\n",
2527	zram->comp_algs[prio]);
2528	err = PTR_ERR(ptr: comp);
2529	goto out_free_comps;
2530	}
2531
2532	zram->comps[prio] = comp;
2533	zram->num_active_comps++;
2534	}
2535	zram->disksize = disksize;
2536	set_capacity_and_notify(disk: zram->disk, size: zram->disksize >> SECTOR_SHIFT);
2537	up_write(sem: &zram->init_lock);
2538
2539	return len;
2540
2541	out_free_comps:
2542	zram_destroy_comps(zram);
2543	zram_meta_free(zram, disksize);
2544	out_unlock:
2545	up_write(sem: &zram->init_lock);
2546	return err;
2547	}
2548
2549	static ssize_t reset_store(struct device *dev,
2550	struct device_attribute *attr, const char *buf, size_t len)
2551	{
2552	int ret;
2553	unsigned short do_reset;
2554	struct zram *zram;
2555	struct gendisk *disk;
2556
2557	ret = kstrtou16(s: buf, base: 10, res: &do_reset);
2558	if (ret)
2559	return ret;
2560
2561	if (!do_reset)
2562	return -EINVAL;
2563
2564	zram = dev_to_zram(dev);
2565	disk = zram->disk;
2566
2567	mutex_lock(&disk->open_mutex);
2568	/* Do not reset an active device or claimed device */
2569	if (disk_openers(disk) || zram->claim) {
2570	mutex_unlock(lock: &disk->open_mutex);
2571	return -EBUSY;
2572	}
2573
2574	/* From now on, anyone can't open /dev/zram[0-9] */
2575	zram->claim = true;
2576	mutex_unlock(lock: &disk->open_mutex);
2577
2578	/* Make sure all the pending I/O are finished */
2579	sync_blockdev(bdev: disk->part0);
2580	zram_reset_device(zram);
2581
2582	mutex_lock(&disk->open_mutex);
2583	zram->claim = false;
2584	mutex_unlock(lock: &disk->open_mutex);
2585
2586	return len;
2587	}
2588
2589	static int zram_open(struct gendisk *disk, blk_mode_t mode)
2590	{
2591	struct zram *zram = disk->private_data;
2592
2593	WARN_ON(!mutex_is_locked(&disk->open_mutex));
2594
2595	/* zram was claimed to reset so open request fails */
2596	if (zram->claim)
2597	return -EBUSY;
2598	return 0;
2599	}
2600
2601	static const struct block_device_operations zram_devops = {
2602	.open = zram_open,
2603	.submit_bio = zram_submit_bio,
2604	.swap_slot_free_notify = zram_slot_free_notify,
2605	.owner = THIS_MODULE
2606	};
2607
2608	static DEVICE_ATTR_WO(compact);
2609	static DEVICE_ATTR_RW(disksize);
2610	static DEVICE_ATTR_RO(initstate);
2611	static DEVICE_ATTR_WO(reset);
2612	static DEVICE_ATTR_WO(mem_limit);
2613	static DEVICE_ATTR_WO(mem_used_max);
2614	static DEVICE_ATTR_WO(idle);
2615	static DEVICE_ATTR_RW(comp_algorithm);
2616	#ifdef CONFIG_ZRAM_WRITEBACK
2617	static DEVICE_ATTR_RW(backing_dev);
2618	static DEVICE_ATTR_WO(writeback);
2619	static DEVICE_ATTR_RW(writeback_limit);
2620	static DEVICE_ATTR_RW(writeback_limit_enable);
2621	#endif
2622	#ifdef CONFIG_ZRAM_MULTI_COMP
2623	static DEVICE_ATTR_RW(recomp_algorithm);
2624	static DEVICE_ATTR_WO(recompress);
2625	#endif
2626	static DEVICE_ATTR_WO(algorithm_params);
2627
2628	static struct attribute *zram_disk_attrs[] = {
2629	&dev_attr_disksize.attr,
2630	&dev_attr_initstate.attr,
2631	&dev_attr_reset.attr,
2632	&dev_attr_compact.attr,
2633	&dev_attr_mem_limit.attr,
2634	&dev_attr_mem_used_max.attr,
2635	&dev_attr_idle.attr,
2636	&dev_attr_comp_algorithm.attr,
2637	#ifdef CONFIG_ZRAM_WRITEBACK
2638	&dev_attr_backing_dev.attr,
2639	&dev_attr_writeback.attr,
2640	&dev_attr_writeback_limit.attr,
2641	&dev_attr_writeback_limit_enable.attr,
2642	#endif
2643	&dev_attr_io_stat.attr,
2644	&dev_attr_mm_stat.attr,
2645	#ifdef CONFIG_ZRAM_WRITEBACK
2646	&dev_attr_bd_stat.attr,
2647	#endif
2648	&dev_attr_debug_stat.attr,
2649	#ifdef CONFIG_ZRAM_MULTI_COMP
2650	&dev_attr_recomp_algorithm.attr,
2651	&dev_attr_recompress.attr,
2652	#endif
2653	&dev_attr_algorithm_params.attr,
2654	NULL,
2655	};
2656
2657	ATTRIBUTE_GROUPS(zram_disk);
2658
2659	/*
2660	* Allocate and initialize new zram device. the function returns
2661	* '>= 0' device_id upon success, and negative value otherwise.
2662	*/
2663	static int zram_add(void)
2664	{
2665	struct queue_limits lim = {
2666	.logical_block_size = ZRAM_LOGICAL_BLOCK_SIZE,
2667	/*
2668	* To ensure that we always get PAGE_SIZE aligned and
2669	* n*PAGE_SIZED sized I/O requests.
2670	*/
2671	.physical_block_size = PAGE_SIZE,
2672	.io_min = PAGE_SIZE,
2673	.io_opt = PAGE_SIZE,
2674	.max_hw_discard_sectors = UINT_MAX,
2675	/*
2676	* zram_bio_discard() will clear all logical blocks if logical
2677	* block size is identical with physical block size(PAGE_SIZE).
2678	* But if it is different, we will skip discarding some parts of
2679	* logical blocks in the part of the request range which isn't
2680	* aligned to physical block size. So we can't ensure that all
2681	* discarded logical blocks are zeroed.
2682	*/
2683	#if ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE
2684	.max_write_zeroes_sectors = UINT_MAX,
2685	#endif
2686	.features = BLK_FEAT_STABLE_WRITES |
2687	BLK_FEAT_SYNCHRONOUS,
2688	};
2689	struct zram *zram;
2690	int ret, device_id;
2691
2692	zram = kzalloc(sizeof(struct zram), GFP_KERNEL);
2693	if (!zram)
2694	return -ENOMEM;
2695
2696	ret = idr_alloc(&zram_index_idr, ptr: zram, start: 0, end: 0, GFP_KERNEL);
2697	if (ret < 0)
2698	goto out_free_dev;
2699	device_id = ret;
2700
2701	init_rwsem(&zram->init_lock);
2702	#ifdef CONFIG_ZRAM_WRITEBACK
2703	spin_lock_init(&zram->wb_limit_lock);
2704	#endif
2705
2706	/* gendisk structure */
2707	zram->disk = blk_alloc_disk(&lim, NUMA_NO_NODE);
2708	if (IS_ERR(ptr: zram->disk)) {
2709	pr_err("Error allocating disk structure for device %d\n",
2710	device_id);
2711	ret = PTR_ERR(ptr: zram->disk);
2712	goto out_free_idr;
2713	}
2714
2715	zram->disk->major = zram_major;
2716	zram->disk->first_minor = device_id;
2717	zram->disk->minors = 1;
2718	zram->disk->flags |= GENHD_FL_NO_PART;
2719	zram->disk->fops = &zram_devops;
2720	zram->disk->private_data = zram;
2721	snprintf(buf: zram->disk->disk_name, size: 16, fmt: "zram%d", device_id);
2722	atomic_set(v: &zram->pp_in_progress, i: 0);
2723	zram_comp_params_reset(zram);
2724	comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, alg: default_compressor);
2725
2726	/* Actual capacity set using sysfs (/sys/block/zram<id>/disksize */
2727	set_capacity(disk: zram->disk, size: 0);
2728	ret = device_add_disk(NULL, disk: zram->disk, groups: zram_disk_groups);
2729	if (ret)
2730	goto out_cleanup_disk;
2731
2732	zram_debugfs_register(zram);
2733	pr_info("Added device: %s\n", zram->disk->disk_name);
2734	return device_id;
2735
2736	out_cleanup_disk:
2737	put_disk(disk: zram->disk);
2738	out_free_idr:
2739	idr_remove(&zram_index_idr, id: device_id);
2740	out_free_dev:
2741	kfree(objp: zram);
2742	return ret;
2743	}
2744
2745	static int zram_remove(struct zram *zram)
2746	{
2747	bool claimed;
2748
2749	mutex_lock(&zram->disk->open_mutex);
2750	if (disk_openers(disk: zram->disk)) {
2751	mutex_unlock(lock: &zram->disk->open_mutex);
2752	return -EBUSY;
2753	}
2754
2755	claimed = zram->claim;
2756	if (!claimed)
2757	zram->claim = true;
2758	mutex_unlock(lock: &zram->disk->open_mutex);
2759
2760	zram_debugfs_unregister(zram);
2761
2762	if (claimed) {
2763	/*
2764	* If we were claimed by reset_store(), del_gendisk() will
2765	* wait until reset_store() is done, so nothing need to do.
2766	*/
2767	;
2768	} else {
2769	/* Make sure all the pending I/O are finished */
2770	sync_blockdev(bdev: zram->disk->part0);
2771	zram_reset_device(zram);
2772	}
2773
2774	pr_info("Removed device: %s\n", zram->disk->disk_name);
2775
2776	del_gendisk(gp: zram->disk);
2777
2778	/* del_gendisk drains pending reset_store */
2779	WARN_ON_ONCE(claimed && zram->claim);
2780
2781	/*
2782	* disksize_store() may be called in between zram_reset_device()
2783	* and del_gendisk(), so run the last reset to avoid leaking
2784	* anything allocated with disksize_store()
2785	*/
2786	zram_reset_device(zram);
2787
2788	put_disk(disk: zram->disk);
2789	kfree(objp: zram);
2790	return 0;
2791	}
2792
2793	/* zram-control sysfs attributes */
2794
2795	/*
2796	* NOTE: hot_add attribute is not the usual read-only sysfs attribute. In a
2797	* sense that reading from this file does alter the state of your system -- it
2798	* creates a new un-initialized zram device and returns back this device's
2799	* device_id (or an error code if it fails to create a new device).
2800	*/
2801	static ssize_t hot_add_show(const struct class *class,
2802	const struct class_attribute *attr,
2803	char *buf)
2804	{
2805	int ret;
2806
2807	mutex_lock(&zram_index_mutex);
2808	ret = zram_add();
2809	mutex_unlock(lock: &zram_index_mutex);
2810
2811	if (ret < 0)
2812	return ret;
2813	return scnprintf(buf, PAGE_SIZE, fmt: "%d\n", ret);
2814	}
2815	/* This attribute must be set to 0400, so CLASS_ATTR_RO() can not be used */
2816	static struct class_attribute class_attr_hot_add =
2817	__ATTR(hot_add, 0400, hot_add_show, NULL);
2818
2819	static ssize_t hot_remove_store(const struct class *class,
2820	const struct class_attribute *attr,
2821	const char *buf,
2822	size_t count)
2823	{
2824	struct zram *zram;
2825	int ret, dev_id;
2826
2827	/* dev_id is gendisk->first_minor, which is `int' */
2828	ret = kstrtoint(s: buf, base: 10, res: &dev_id);
2829	if (ret)
2830	return ret;
2831	if (dev_id < 0)
2832	return -EINVAL;
2833
2834	mutex_lock(&zram_index_mutex);
2835
2836	zram = idr_find(&zram_index_idr, id: dev_id);
2837	if (zram) {
2838	ret = zram_remove(zram);
2839	if (!ret)
2840	idr_remove(&zram_index_idr, id: dev_id);
2841	} else {
2842	ret = -ENODEV;
2843	}
2844
2845	mutex_unlock(lock: &zram_index_mutex);
2846	return ret ? ret : count;
2847	}
2848	static CLASS_ATTR_WO(hot_remove);
2849
2850	static struct attribute *zram_control_class_attrs[] = {
2851	&class_attr_hot_add.attr,
2852	&class_attr_hot_remove.attr,
2853	NULL,
2854	};
2855	ATTRIBUTE_GROUPS(zram_control_class);
2856
2857	static struct class zram_control_class = {
2858	.name = "zram-control",
2859	.class_groups = zram_control_class_groups,
2860	};
2861
2862	static int zram_remove_cb(int id, void *ptr, void *data)
2863	{
2864	WARN_ON_ONCE(zram_remove(ptr));
2865	return 0;
2866	}
2867
2868	static void destroy_devices(void)
2869	{
2870	class_unregister(class: &zram_control_class);
2871	idr_for_each(&zram_index_idr, fn: &zram_remove_cb, NULL);
2872	zram_debugfs_destroy();
2873	idr_destroy(&zram_index_idr);
2874	unregister_blkdev(major: zram_major, name: "zram");
2875	cpuhp_remove_multi_state(state: CPUHP_ZCOMP_PREPARE);
2876	}
2877
2878	static int __init zram_init(void)
2879	{
2880	struct zram_table_entry zram_te;
2881	int ret;
2882
2883	BUILD_BUG_ON(__NR_ZRAM_PAGEFLAGS > sizeof(zram_te.flags) * 8);
2884
2885	ret = cpuhp_setup_state_multi(state: CPUHP_ZCOMP_PREPARE, name: "block/zram:prepare",
2886	startup: zcomp_cpu_up_prepare, teardown: zcomp_cpu_dead);
2887	if (ret < 0)
2888	return ret;
2889
2890	ret = class_register(class: &zram_control_class);
2891	if (ret) {
2892	pr_err("Unable to register zram-control class\n");
2893	cpuhp_remove_multi_state(state: CPUHP_ZCOMP_PREPARE);
2894	return ret;
2895	}
2896
2897	zram_debugfs_create();
2898	zram_major = register_blkdev(0, "zram");
2899	if (zram_major <= 0) {
2900	pr_err("Unable to get major number\n");
2901	class_unregister(class: &zram_control_class);
2902	cpuhp_remove_multi_state(state: CPUHP_ZCOMP_PREPARE);
2903	return -EBUSY;
2904	}
2905
2906	while (num_devices != 0) {
2907	mutex_lock(&zram_index_mutex);
2908	ret = zram_add();
2909	mutex_unlock(lock: &zram_index_mutex);
2910	if (ret < 0)
2911	goto out_error;
2912	num_devices--;
2913	}
2914
2915	return 0;
2916
2917	out_error:
2918	destroy_devices();
2919	return ret;
2920	}
2921
2922	static void __exit zram_exit(void)
2923	{
2924	destroy_devices();
2925	}
2926
2927	module_init(zram_init);
2928	module_exit(zram_exit);
2929
2930	module_param(num_devices, uint, 0);
2931	MODULE_PARM_DESC(num_devices, "Number of pre-created zram devices");
2932
2933	MODULE_LICENSE("Dual BSD/GPL");
2934	MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
2935	MODULE_DESCRIPTION("Compressed RAM Block Device");
2936