1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/xz.h>
3#include "compress.h"
4
5struct z_erofs_lzma {
6 struct z_erofs_lzma *next;
7 struct xz_dec_microlzma *state;
8 struct xz_buf buf;
9 u8 bounce[PAGE_SIZE];
10};
11
12/* considering the LZMA performance, no need to use a lockless list for now */
13static DEFINE_SPINLOCK(z_erofs_lzma_lock);
14static unsigned int z_erofs_lzma_max_dictsize;
15static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
16static struct z_erofs_lzma *z_erofs_lzma_head;
17static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
18
19module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
20
21void z_erofs_lzma_exit(void)
22{
23 /* there should be no running fs instance */
24 while (z_erofs_lzma_avail_strms) {
25 struct z_erofs_lzma *strm;
26
27 spin_lock(lock: &z_erofs_lzma_lock);
28 strm = z_erofs_lzma_head;
29 if (!strm) {
30 spin_unlock(lock: &z_erofs_lzma_lock);
31 DBG_BUGON(1);
32 return;
33 }
34 z_erofs_lzma_head = NULL;
35 spin_unlock(lock: &z_erofs_lzma_lock);
36
37 while (strm) {
38 struct z_erofs_lzma *n = strm->next;
39
40 if (strm->state)
41 xz_dec_microlzma_end(s: strm->state);
42 kfree(objp: strm);
43 --z_erofs_lzma_avail_strms;
44 strm = n;
45 }
46 }
47}
48
49int __init z_erofs_lzma_init(void)
50{
51 unsigned int i;
52
53 /* by default, use # of possible CPUs instead */
54 if (!z_erofs_lzma_nstrms)
55 z_erofs_lzma_nstrms = num_possible_cpus();
56
57 for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
58 struct z_erofs_lzma *strm = kzalloc(size: sizeof(*strm), GFP_KERNEL);
59
60 if (!strm) {
61 z_erofs_lzma_exit();
62 return -ENOMEM;
63 }
64 spin_lock(lock: &z_erofs_lzma_lock);
65 strm->next = z_erofs_lzma_head;
66 z_erofs_lzma_head = strm;
67 spin_unlock(lock: &z_erofs_lzma_lock);
68 ++z_erofs_lzma_avail_strms;
69 }
70 return 0;
71}
72
73int z_erofs_load_lzma_config(struct super_block *sb,
74 struct erofs_super_block *dsb, void *data, int size)
75{
76 static DEFINE_MUTEX(lzma_resize_mutex);
77 struct z_erofs_lzma_cfgs *lzma = data;
78 unsigned int dict_size, i;
79 struct z_erofs_lzma *strm, *head = NULL;
80 int err;
81
82 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
83 erofs_err(sb, "invalid lzma cfgs, size=%u", size);
84 return -EINVAL;
85 }
86 if (lzma->format) {
87 erofs_err(sb, "unidentified lzma format %x, please check kernel version",
88 le16_to_cpu(lzma->format));
89 return -EINVAL;
90 }
91 dict_size = le32_to_cpu(lzma->dict_size);
92 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
93 erofs_err(sb, "unsupported lzma dictionary size %u",
94 dict_size);
95 return -EINVAL;
96 }
97
98 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
99 mutex_lock(&lzma_resize_mutex);
100
101 if (z_erofs_lzma_max_dictsize >= dict_size) {
102 mutex_unlock(lock: &lzma_resize_mutex);
103 return 0;
104 }
105
106 /* 1. collect/isolate all streams for the following check */
107 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
108 struct z_erofs_lzma *last;
109
110again:
111 spin_lock(lock: &z_erofs_lzma_lock);
112 strm = z_erofs_lzma_head;
113 if (!strm) {
114 spin_unlock(lock: &z_erofs_lzma_lock);
115 wait_event(z_erofs_lzma_wq,
116 READ_ONCE(z_erofs_lzma_head));
117 goto again;
118 }
119 z_erofs_lzma_head = NULL;
120 spin_unlock(lock: &z_erofs_lzma_lock);
121
122 for (last = strm; last->next; last = last->next)
123 ++i;
124 last->next = head;
125 head = strm;
126 }
127
128 err = 0;
129 /* 2. walk each isolated stream and grow max dict_size if needed */
130 for (strm = head; strm; strm = strm->next) {
131 if (strm->state)
132 xz_dec_microlzma_end(s: strm->state);
133 strm->state = xz_dec_microlzma_alloc(mode: XZ_PREALLOC, dict_size);
134 if (!strm->state)
135 err = -ENOMEM;
136 }
137
138 /* 3. push back all to the global list and update max dict_size */
139 spin_lock(lock: &z_erofs_lzma_lock);
140 DBG_BUGON(z_erofs_lzma_head);
141 z_erofs_lzma_head = head;
142 spin_unlock(lock: &z_erofs_lzma_lock);
143 wake_up_all(&z_erofs_lzma_wq);
144
145 z_erofs_lzma_max_dictsize = dict_size;
146 mutex_unlock(lock: &lzma_resize_mutex);
147 return err;
148}
149
150int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
151 struct page **pgpl)
152{
153 const unsigned int nrpages_out =
154 PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
155 const unsigned int nrpages_in =
156 PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
157 unsigned int inlen, outlen, pageofs;
158 struct z_erofs_lzma *strm;
159 u8 *kin;
160 bool bounced = false;
161 int no, ni, j, err = 0;
162
163 /* 1. get the exact LZMA compressed size */
164 kin = kmap(page: *rq->in);
165 err = z_erofs_fixup_insize(rq, padbuf: kin + rq->pageofs_in,
166 min_t(unsigned int, rq->inputsize,
167 rq->sb->s_blocksize - rq->pageofs_in));
168 if (err) {
169 kunmap(page: *rq->in);
170 return err;
171 }
172
173 /* 2. get an available lzma context */
174again:
175 spin_lock(lock: &z_erofs_lzma_lock);
176 strm = z_erofs_lzma_head;
177 if (!strm) {
178 spin_unlock(lock: &z_erofs_lzma_lock);
179 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
180 goto again;
181 }
182 z_erofs_lzma_head = strm->next;
183 spin_unlock(lock: &z_erofs_lzma_lock);
184
185 /* 3. multi-call decompress */
186 inlen = rq->inputsize;
187 outlen = rq->outputsize;
188 xz_dec_microlzma_reset(s: strm->state, comp_size: inlen, uncomp_size: outlen,
189 uncomp_size_is_exact: !rq->partial_decoding);
190 pageofs = rq->pageofs_out;
191 strm->buf.in = kin + rq->pageofs_in;
192 strm->buf.in_pos = 0;
193 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
194 inlen -= strm->buf.in_size;
195 strm->buf.out = NULL;
196 strm->buf.out_pos = 0;
197 strm->buf.out_size = 0;
198
199 for (ni = 0, no = -1;;) {
200 enum xz_ret xz_err;
201
202 if (strm->buf.out_pos == strm->buf.out_size) {
203 if (strm->buf.out) {
204 kunmap(page: rq->out[no]);
205 strm->buf.out = NULL;
206 }
207
208 if (++no >= nrpages_out || !outlen) {
209 erofs_err(rq->sb, "decompressed buf out of bound");
210 err = -EFSCORRUPTED;
211 break;
212 }
213 strm->buf.out_pos = 0;
214 strm->buf.out_size = min_t(u32, outlen,
215 PAGE_SIZE - pageofs);
216 outlen -= strm->buf.out_size;
217 if (!rq->out[no] && rq->fillgaps) { /* deduped */
218 rq->out[no] = erofs_allocpage(pagepool: pgpl, gfp: rq->gfp);
219 if (!rq->out[no]) {
220 err = -ENOMEM;
221 break;
222 }
223 set_page_private(page: rq->out[no],
224 Z_EROFS_SHORTLIVED_PAGE);
225 }
226 if (rq->out[no])
227 strm->buf.out = kmap(page: rq->out[no]) + pageofs;
228 pageofs = 0;
229 } else if (strm->buf.in_pos == strm->buf.in_size) {
230 kunmap(page: rq->in[ni]);
231
232 if (++ni >= nrpages_in || !inlen) {
233 erofs_err(rq->sb, "compressed buf out of bound");
234 err = -EFSCORRUPTED;
235 break;
236 }
237 strm->buf.in_pos = 0;
238 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
239 inlen -= strm->buf.in_size;
240 kin = kmap(page: rq->in[ni]);
241 strm->buf.in = kin;
242 bounced = false;
243 }
244
245 /*
246 * Handle overlapping: Use bounced buffer if the compressed
247 * data is under processing; Otherwise, Use short-lived pages
248 * from the on-stack pagepool where pages share with the same
249 * request.
250 */
251 if (!bounced && rq->out[no] == rq->in[ni]) {
252 memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
253 strm->buf.in = strm->bounce;
254 bounced = true;
255 }
256 for (j = ni + 1; j < nrpages_in; ++j) {
257 struct page *tmppage;
258
259 if (rq->out[no] != rq->in[j])
260 continue;
261 tmppage = erofs_allocpage(pagepool: pgpl, gfp: rq->gfp);
262 if (!tmppage) {
263 err = -ENOMEM;
264 goto failed;
265 }
266 set_page_private(page: tmppage, Z_EROFS_SHORTLIVED_PAGE);
267 copy_highpage(to: tmppage, from: rq->in[j]);
268 rq->in[j] = tmppage;
269 }
270 xz_err = xz_dec_microlzma_run(s: strm->state, b: &strm->buf);
271 DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
272 DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
273
274 if (xz_err != XZ_OK) {
275 if (xz_err == XZ_STREAM_END && !outlen)
276 break;
277 erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
278 xz_err, rq->inputsize, rq->outputsize);
279 err = -EFSCORRUPTED;
280 break;
281 }
282 }
283failed:
284 if (no < nrpages_out && strm->buf.out)
285 kunmap(page: rq->out[no]);
286 if (ni < nrpages_in)
287 kunmap(page: rq->in[ni]);
288 /* 4. push back LZMA stream context to the global list */
289 spin_lock(lock: &z_erofs_lzma_lock);
290 strm->next = z_erofs_lzma_head;
291 z_erofs_lzma_head = strm;
292 spin_unlock(lock: &z_erofs_lzma_lock);
293 wake_up(&z_erofs_lzma_wq);
294 return err;
295}
296

source code of linux/fs/erofs/decompressor_lzma.c