1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | #include <linux/xz.h> |
3 | #include "compress.h" |
4 | |
5 | struct z_erofs_lzma { |
6 | struct z_erofs_lzma *next; |
7 | struct xz_dec_microlzma *state; |
8 | struct xz_buf buf; |
9 | u8 bounce[PAGE_SIZE]; |
10 | }; |
11 | |
12 | /* considering the LZMA performance, no need to use a lockless list for now */ |
13 | static DEFINE_SPINLOCK(z_erofs_lzma_lock); |
14 | static unsigned int z_erofs_lzma_max_dictsize; |
15 | static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; |
16 | static struct z_erofs_lzma *z_erofs_lzma_head; |
17 | static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); |
18 | |
19 | module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); |
20 | |
21 | void z_erofs_lzma_exit(void) |
22 | { |
23 | /* there should be no running fs instance */ |
24 | while (z_erofs_lzma_avail_strms) { |
25 | struct z_erofs_lzma *strm; |
26 | |
27 | spin_lock(lock: &z_erofs_lzma_lock); |
28 | strm = z_erofs_lzma_head; |
29 | if (!strm) { |
30 | spin_unlock(lock: &z_erofs_lzma_lock); |
31 | DBG_BUGON(1); |
32 | return; |
33 | } |
34 | z_erofs_lzma_head = NULL; |
35 | spin_unlock(lock: &z_erofs_lzma_lock); |
36 | |
37 | while (strm) { |
38 | struct z_erofs_lzma *n = strm->next; |
39 | |
40 | if (strm->state) |
41 | xz_dec_microlzma_end(s: strm->state); |
42 | kfree(objp: strm); |
43 | --z_erofs_lzma_avail_strms; |
44 | strm = n; |
45 | } |
46 | } |
47 | } |
48 | |
49 | int __init z_erofs_lzma_init(void) |
50 | { |
51 | unsigned int i; |
52 | |
53 | /* by default, use # of possible CPUs instead */ |
54 | if (!z_erofs_lzma_nstrms) |
55 | z_erofs_lzma_nstrms = num_possible_cpus(); |
56 | |
57 | for (i = 0; i < z_erofs_lzma_nstrms; ++i) { |
58 | struct z_erofs_lzma *strm = kzalloc(size: sizeof(*strm), GFP_KERNEL); |
59 | |
60 | if (!strm) { |
61 | z_erofs_lzma_exit(); |
62 | return -ENOMEM; |
63 | } |
64 | spin_lock(lock: &z_erofs_lzma_lock); |
65 | strm->next = z_erofs_lzma_head; |
66 | z_erofs_lzma_head = strm; |
67 | spin_unlock(lock: &z_erofs_lzma_lock); |
68 | ++z_erofs_lzma_avail_strms; |
69 | } |
70 | return 0; |
71 | } |
72 | |
73 | int z_erofs_load_lzma_config(struct super_block *sb, |
74 | struct erofs_super_block *dsb, void *data, int size) |
75 | { |
76 | static DEFINE_MUTEX(lzma_resize_mutex); |
77 | struct z_erofs_lzma_cfgs *lzma = data; |
78 | unsigned int dict_size, i; |
79 | struct z_erofs_lzma *strm, *head = NULL; |
80 | int err; |
81 | |
82 | if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { |
83 | erofs_err(sb, "invalid lzma cfgs, size=%u" , size); |
84 | return -EINVAL; |
85 | } |
86 | if (lzma->format) { |
87 | erofs_err(sb, "unidentified lzma format %x, please check kernel version" , |
88 | le16_to_cpu(lzma->format)); |
89 | return -EINVAL; |
90 | } |
91 | dict_size = le32_to_cpu(lzma->dict_size); |
92 | if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { |
93 | erofs_err(sb, "unsupported lzma dictionary size %u" , |
94 | dict_size); |
95 | return -EINVAL; |
96 | } |
97 | |
98 | /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ |
99 | mutex_lock(&lzma_resize_mutex); |
100 | |
101 | if (z_erofs_lzma_max_dictsize >= dict_size) { |
102 | mutex_unlock(lock: &lzma_resize_mutex); |
103 | return 0; |
104 | } |
105 | |
106 | /* 1. collect/isolate all streams for the following check */ |
107 | for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { |
108 | struct z_erofs_lzma *last; |
109 | |
110 | again: |
111 | spin_lock(lock: &z_erofs_lzma_lock); |
112 | strm = z_erofs_lzma_head; |
113 | if (!strm) { |
114 | spin_unlock(lock: &z_erofs_lzma_lock); |
115 | wait_event(z_erofs_lzma_wq, |
116 | READ_ONCE(z_erofs_lzma_head)); |
117 | goto again; |
118 | } |
119 | z_erofs_lzma_head = NULL; |
120 | spin_unlock(lock: &z_erofs_lzma_lock); |
121 | |
122 | for (last = strm; last->next; last = last->next) |
123 | ++i; |
124 | last->next = head; |
125 | head = strm; |
126 | } |
127 | |
128 | err = 0; |
129 | /* 2. walk each isolated stream and grow max dict_size if needed */ |
130 | for (strm = head; strm; strm = strm->next) { |
131 | if (strm->state) |
132 | xz_dec_microlzma_end(s: strm->state); |
133 | strm->state = xz_dec_microlzma_alloc(mode: XZ_PREALLOC, dict_size); |
134 | if (!strm->state) |
135 | err = -ENOMEM; |
136 | } |
137 | |
138 | /* 3. push back all to the global list and update max dict_size */ |
139 | spin_lock(lock: &z_erofs_lzma_lock); |
140 | DBG_BUGON(z_erofs_lzma_head); |
141 | z_erofs_lzma_head = head; |
142 | spin_unlock(lock: &z_erofs_lzma_lock); |
143 | wake_up_all(&z_erofs_lzma_wq); |
144 | |
145 | z_erofs_lzma_max_dictsize = dict_size; |
146 | mutex_unlock(lock: &lzma_resize_mutex); |
147 | return err; |
148 | } |
149 | |
150 | int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, |
151 | struct page **pgpl) |
152 | { |
153 | const unsigned int nrpages_out = |
154 | PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; |
155 | const unsigned int nrpages_in = |
156 | PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; |
157 | unsigned int inlen, outlen, pageofs; |
158 | struct z_erofs_lzma *strm; |
159 | u8 *kin; |
160 | bool bounced = false; |
161 | int no, ni, j, err = 0; |
162 | |
163 | /* 1. get the exact LZMA compressed size */ |
164 | kin = kmap(page: *rq->in); |
165 | err = z_erofs_fixup_insize(rq, padbuf: kin + rq->pageofs_in, |
166 | min_t(unsigned int, rq->inputsize, |
167 | rq->sb->s_blocksize - rq->pageofs_in)); |
168 | if (err) { |
169 | kunmap(page: *rq->in); |
170 | return err; |
171 | } |
172 | |
173 | /* 2. get an available lzma context */ |
174 | again: |
175 | spin_lock(lock: &z_erofs_lzma_lock); |
176 | strm = z_erofs_lzma_head; |
177 | if (!strm) { |
178 | spin_unlock(lock: &z_erofs_lzma_lock); |
179 | wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); |
180 | goto again; |
181 | } |
182 | z_erofs_lzma_head = strm->next; |
183 | spin_unlock(lock: &z_erofs_lzma_lock); |
184 | |
185 | /* 3. multi-call decompress */ |
186 | inlen = rq->inputsize; |
187 | outlen = rq->outputsize; |
188 | xz_dec_microlzma_reset(s: strm->state, comp_size: inlen, uncomp_size: outlen, |
189 | uncomp_size_is_exact: !rq->partial_decoding); |
190 | pageofs = rq->pageofs_out; |
191 | strm->buf.in = kin + rq->pageofs_in; |
192 | strm->buf.in_pos = 0; |
193 | strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in); |
194 | inlen -= strm->buf.in_size; |
195 | strm->buf.out = NULL; |
196 | strm->buf.out_pos = 0; |
197 | strm->buf.out_size = 0; |
198 | |
199 | for (ni = 0, no = -1;;) { |
200 | enum xz_ret xz_err; |
201 | |
202 | if (strm->buf.out_pos == strm->buf.out_size) { |
203 | if (strm->buf.out) { |
204 | kunmap(page: rq->out[no]); |
205 | strm->buf.out = NULL; |
206 | } |
207 | |
208 | if (++no >= nrpages_out || !outlen) { |
209 | erofs_err(rq->sb, "decompressed buf out of bound" ); |
210 | err = -EFSCORRUPTED; |
211 | break; |
212 | } |
213 | strm->buf.out_pos = 0; |
214 | strm->buf.out_size = min_t(u32, outlen, |
215 | PAGE_SIZE - pageofs); |
216 | outlen -= strm->buf.out_size; |
217 | if (!rq->out[no] && rq->fillgaps) { /* deduped */ |
218 | rq->out[no] = erofs_allocpage(pagepool: pgpl, gfp: rq->gfp); |
219 | if (!rq->out[no]) { |
220 | err = -ENOMEM; |
221 | break; |
222 | } |
223 | set_page_private(page: rq->out[no], |
224 | Z_EROFS_SHORTLIVED_PAGE); |
225 | } |
226 | if (rq->out[no]) |
227 | strm->buf.out = kmap(page: rq->out[no]) + pageofs; |
228 | pageofs = 0; |
229 | } else if (strm->buf.in_pos == strm->buf.in_size) { |
230 | kunmap(page: rq->in[ni]); |
231 | |
232 | if (++ni >= nrpages_in || !inlen) { |
233 | erofs_err(rq->sb, "compressed buf out of bound" ); |
234 | err = -EFSCORRUPTED; |
235 | break; |
236 | } |
237 | strm->buf.in_pos = 0; |
238 | strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE); |
239 | inlen -= strm->buf.in_size; |
240 | kin = kmap(page: rq->in[ni]); |
241 | strm->buf.in = kin; |
242 | bounced = false; |
243 | } |
244 | |
245 | /* |
246 | * Handle overlapping: Use bounced buffer if the compressed |
247 | * data is under processing; Otherwise, Use short-lived pages |
248 | * from the on-stack pagepool where pages share with the same |
249 | * request. |
250 | */ |
251 | if (!bounced && rq->out[no] == rq->in[ni]) { |
252 | memcpy(strm->bounce, strm->buf.in, strm->buf.in_size); |
253 | strm->buf.in = strm->bounce; |
254 | bounced = true; |
255 | } |
256 | for (j = ni + 1; j < nrpages_in; ++j) { |
257 | struct page *tmppage; |
258 | |
259 | if (rq->out[no] != rq->in[j]) |
260 | continue; |
261 | tmppage = erofs_allocpage(pagepool: pgpl, gfp: rq->gfp); |
262 | if (!tmppage) { |
263 | err = -ENOMEM; |
264 | goto failed; |
265 | } |
266 | set_page_private(page: tmppage, Z_EROFS_SHORTLIVED_PAGE); |
267 | copy_highpage(to: tmppage, from: rq->in[j]); |
268 | rq->in[j] = tmppage; |
269 | } |
270 | xz_err = xz_dec_microlzma_run(s: strm->state, b: &strm->buf); |
271 | DBG_BUGON(strm->buf.out_pos > strm->buf.out_size); |
272 | DBG_BUGON(strm->buf.in_pos > strm->buf.in_size); |
273 | |
274 | if (xz_err != XZ_OK) { |
275 | if (xz_err == XZ_STREAM_END && !outlen) |
276 | break; |
277 | erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]" , |
278 | xz_err, rq->inputsize, rq->outputsize); |
279 | err = -EFSCORRUPTED; |
280 | break; |
281 | } |
282 | } |
283 | failed: |
284 | if (no < nrpages_out && strm->buf.out) |
285 | kunmap(page: rq->out[no]); |
286 | if (ni < nrpages_in) |
287 | kunmap(page: rq->in[ni]); |
288 | /* 4. push back LZMA stream context to the global list */ |
289 | spin_lock(lock: &z_erofs_lzma_lock); |
290 | strm->next = z_erofs_lzma_head; |
291 | z_erofs_lzma_head = strm; |
292 | spin_unlock(lock: &z_erofs_lzma_lock); |
293 | wake_up(&z_erofs_lzma_wq); |
294 | return err; |
295 | } |
296 | |