1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | #include <linux/zlib.h> |
3 | #include "compress.h" |
4 | |
5 | struct z_erofs_deflate { |
6 | struct z_erofs_deflate *next; |
7 | struct z_stream_s z; |
8 | u8 bounce[PAGE_SIZE]; |
9 | }; |
10 | |
11 | static DEFINE_SPINLOCK(z_erofs_deflate_lock); |
12 | static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms; |
13 | static struct z_erofs_deflate *z_erofs_deflate_head; |
14 | static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq); |
15 | |
16 | module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444); |
17 | |
18 | void z_erofs_deflate_exit(void) |
19 | { |
20 | /* there should be no running fs instance */ |
21 | while (z_erofs_deflate_avail_strms) { |
22 | struct z_erofs_deflate *strm; |
23 | |
24 | spin_lock(lock: &z_erofs_deflate_lock); |
25 | strm = z_erofs_deflate_head; |
26 | if (!strm) { |
27 | spin_unlock(lock: &z_erofs_deflate_lock); |
28 | continue; |
29 | } |
30 | z_erofs_deflate_head = NULL; |
31 | spin_unlock(lock: &z_erofs_deflate_lock); |
32 | |
33 | while (strm) { |
34 | struct z_erofs_deflate *n = strm->next; |
35 | |
36 | vfree(addr: strm->z.workspace); |
37 | kfree(objp: strm); |
38 | --z_erofs_deflate_avail_strms; |
39 | strm = n; |
40 | } |
41 | } |
42 | } |
43 | |
44 | int __init z_erofs_deflate_init(void) |
45 | { |
46 | /* by default, use # of possible CPUs instead */ |
47 | if (!z_erofs_deflate_nstrms) |
48 | z_erofs_deflate_nstrms = num_possible_cpus(); |
49 | |
50 | for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms; |
51 | ++z_erofs_deflate_avail_strms) { |
52 | struct z_erofs_deflate *strm; |
53 | |
54 | strm = kzalloc(size: sizeof(*strm), GFP_KERNEL); |
55 | if (!strm) |
56 | goto out_failed; |
57 | |
58 | /* XXX: in-kernel zlib cannot shrink windowbits currently */ |
59 | strm->z.workspace = vmalloc(size: zlib_inflate_workspacesize()); |
60 | if (!strm->z.workspace) { |
61 | kfree(objp: strm); |
62 | goto out_failed; |
63 | } |
64 | |
65 | spin_lock(lock: &z_erofs_deflate_lock); |
66 | strm->next = z_erofs_deflate_head; |
67 | z_erofs_deflate_head = strm; |
68 | spin_unlock(lock: &z_erofs_deflate_lock); |
69 | } |
70 | return 0; |
71 | |
72 | out_failed: |
73 | erofs_err(NULL, "failed to allocate zlib workspace" ); |
74 | z_erofs_deflate_exit(); |
75 | return -ENOMEM; |
76 | } |
77 | |
78 | int z_erofs_load_deflate_config(struct super_block *sb, |
79 | struct erofs_super_block *dsb, void *data, int size) |
80 | { |
81 | struct z_erofs_deflate_cfgs *dfl = data; |
82 | |
83 | if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) { |
84 | erofs_err(sb, "invalid deflate cfgs, size=%u" , size); |
85 | return -EINVAL; |
86 | } |
87 | |
88 | if (dfl->windowbits > MAX_WBITS) { |
89 | erofs_err(sb, "unsupported windowbits %u" , dfl->windowbits); |
90 | return -EOPNOTSUPP; |
91 | } |
92 | |
93 | erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!" ); |
94 | return 0; |
95 | } |
96 | |
97 | int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, |
98 | struct page **pgpl) |
99 | { |
100 | const unsigned int nrpages_out = |
101 | PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; |
102 | const unsigned int nrpages_in = |
103 | PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; |
104 | struct super_block *sb = rq->sb; |
105 | unsigned int insz, outsz, pofs; |
106 | struct z_erofs_deflate *strm; |
107 | u8 *kin, *kout = NULL; |
108 | bool bounced = false; |
109 | int no = -1, ni = 0, j = 0, zerr, err; |
110 | |
111 | /* 1. get the exact DEFLATE compressed size */ |
112 | kin = kmap_local_page(page: *rq->in); |
113 | err = z_erofs_fixup_insize(rq, padbuf: kin + rq->pageofs_in, |
114 | min_t(unsigned int, rq->inputsize, |
115 | sb->s_blocksize - rq->pageofs_in)); |
116 | if (err) { |
117 | kunmap_local(kin); |
118 | return err; |
119 | } |
120 | |
121 | /* 2. get an available DEFLATE context */ |
122 | again: |
123 | spin_lock(lock: &z_erofs_deflate_lock); |
124 | strm = z_erofs_deflate_head; |
125 | if (!strm) { |
126 | spin_unlock(lock: &z_erofs_deflate_lock); |
127 | wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head)); |
128 | goto again; |
129 | } |
130 | z_erofs_deflate_head = strm->next; |
131 | spin_unlock(lock: &z_erofs_deflate_lock); |
132 | |
133 | /* 3. multi-call decompress */ |
134 | insz = rq->inputsize; |
135 | outsz = rq->outputsize; |
136 | zerr = zlib_inflateInit2(strm: &strm->z, windowBits: -MAX_WBITS); |
137 | if (zerr != Z_OK) { |
138 | err = -EIO; |
139 | goto failed_zinit; |
140 | } |
141 | |
142 | pofs = rq->pageofs_out; |
143 | strm->z.avail_in = min_t(u32, insz, PAGE_SIZE - rq->pageofs_in); |
144 | insz -= strm->z.avail_in; |
145 | strm->z.next_in = kin + rq->pageofs_in; |
146 | strm->z.avail_out = 0; |
147 | |
148 | while (1) { |
149 | if (!strm->z.avail_out) { |
150 | if (++no >= nrpages_out || !outsz) { |
151 | erofs_err(sb, "insufficient space for decompressed data" ); |
152 | err = -EFSCORRUPTED; |
153 | break; |
154 | } |
155 | |
156 | if (kout) |
157 | kunmap_local(kout); |
158 | strm->z.avail_out = min_t(u32, outsz, PAGE_SIZE - pofs); |
159 | outsz -= strm->z.avail_out; |
160 | if (!rq->out[no]) { |
161 | rq->out[no] = erofs_allocpage(pagepool: pgpl, gfp: rq->gfp); |
162 | if (!rq->out[no]) { |
163 | kout = NULL; |
164 | err = -ENOMEM; |
165 | break; |
166 | } |
167 | set_page_private(page: rq->out[no], |
168 | Z_EROFS_SHORTLIVED_PAGE); |
169 | } |
170 | kout = kmap_local_page(page: rq->out[no]); |
171 | strm->z.next_out = kout + pofs; |
172 | pofs = 0; |
173 | } |
174 | |
175 | if (!strm->z.avail_in && insz) { |
176 | if (++ni >= nrpages_in) { |
177 | erofs_err(sb, "invalid compressed data" ); |
178 | err = -EFSCORRUPTED; |
179 | break; |
180 | } |
181 | |
182 | if (kout) { /* unlike kmap(), take care of the orders */ |
183 | j = strm->z.next_out - kout; |
184 | kunmap_local(kout); |
185 | } |
186 | kunmap_local(kin); |
187 | strm->z.avail_in = min_t(u32, insz, PAGE_SIZE); |
188 | insz -= strm->z.avail_in; |
189 | kin = kmap_local_page(page: rq->in[ni]); |
190 | strm->z.next_in = kin; |
191 | bounced = false; |
192 | if (kout) { |
193 | kout = kmap_local_page(page: rq->out[no]); |
194 | strm->z.next_out = kout + j; |
195 | } |
196 | } |
197 | |
198 | /* |
199 | * Handle overlapping: Use bounced buffer if the compressed |
200 | * data is under processing; Or use short-lived pages from the |
201 | * on-stack pagepool where pages share among the same request |
202 | * and not _all_ inplace I/O pages are needed to be doubled. |
203 | */ |
204 | if (!bounced && rq->out[no] == rq->in[ni]) { |
205 | memcpy(strm->bounce, strm->z.next_in, strm->z.avail_in); |
206 | strm->z.next_in = strm->bounce; |
207 | bounced = true; |
208 | } |
209 | |
210 | for (j = ni + 1; j < nrpages_in; ++j) { |
211 | struct page *tmppage; |
212 | |
213 | if (rq->out[no] != rq->in[j]) |
214 | continue; |
215 | tmppage = erofs_allocpage(pagepool: pgpl, gfp: rq->gfp); |
216 | if (!tmppage) { |
217 | err = -ENOMEM; |
218 | goto failed; |
219 | } |
220 | set_page_private(page: tmppage, Z_EROFS_SHORTLIVED_PAGE); |
221 | copy_highpage(to: tmppage, from: rq->in[j]); |
222 | rq->in[j] = tmppage; |
223 | } |
224 | |
225 | zerr = zlib_inflate(strm: &strm->z, Z_SYNC_FLUSH); |
226 | if (zerr != Z_OK || !(outsz + strm->z.avail_out)) { |
227 | if (zerr == Z_OK && rq->partial_decoding) |
228 | break; |
229 | if (zerr == Z_STREAM_END && !outsz) |
230 | break; |
231 | erofs_err(sb, "failed to decompress %d in[%u] out[%u]" , |
232 | zerr, rq->inputsize, rq->outputsize); |
233 | err = -EFSCORRUPTED; |
234 | break; |
235 | } |
236 | } |
237 | failed: |
238 | if (zlib_inflateEnd(strm: &strm->z) != Z_OK && !err) |
239 | err = -EIO; |
240 | if (kout) |
241 | kunmap_local(kout); |
242 | failed_zinit: |
243 | kunmap_local(kin); |
244 | /* 4. push back DEFLATE stream context to the global list */ |
245 | spin_lock(lock: &z_erofs_deflate_lock); |
246 | strm->next = z_erofs_deflate_head; |
247 | z_erofs_deflate_head = strm; |
248 | spin_unlock(lock: &z_erofs_deflate_lock); |
249 | wake_up(&z_erofs_deflate_wq); |
250 | return err; |
251 | } |
252 | |