1 // SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
2 /*
3 * Copyright (C), 2008-2020, OPPO Mobile Comm Corp., Ltd.
4 * Created by Huang Jianan <[email protected]>
5 */
6 #include <stdlib.h>
7
8 #include "erofs/decompress.h"
9 #include "erofs/err.h"
10 #include "erofs/print.h"
11
z_erofs_fixup_insize(const u8 * padbuf,unsigned int padbufsize)12 static unsigned int z_erofs_fixup_insize(const u8 *padbuf, unsigned int padbufsize)
13 {
14 unsigned int inputmargin;
15
16 for (inputmargin = 0; inputmargin < padbufsize &&
17 !padbuf[inputmargin]; ++inputmargin);
18 return inputmargin;
19 }
20
21 #ifdef HAVE_LIBZSTD
22 #include <zstd.h>
23 #include <zstd_errors.h>
24
25 /* also a very preliminary userspace version */
z_erofs_decompress_zstd(struct z_erofs_decompress_req * rq)26 static int z_erofs_decompress_zstd(struct z_erofs_decompress_req *rq)
27 {
28 int ret = 0;
29 char *dest = rq->out;
30 char *src = rq->in;
31 char *buff = NULL;
32 unsigned int inputmargin = 0;
33 unsigned long long total;
34
35 inputmargin = z_erofs_fixup_insize((u8 *)src, rq->inputsize);
36 if (inputmargin >= rq->inputsize)
37 return -EFSCORRUPTED;
38
39 #ifdef HAVE_ZSTD_GETFRAMECONTENTSIZE
40 total = ZSTD_getFrameContentSize(src + inputmargin,
41 rq->inputsize - inputmargin);
42 if (total == ZSTD_CONTENTSIZE_UNKNOWN ||
43 total == ZSTD_CONTENTSIZE_ERROR)
44 return -EFSCORRUPTED;
45 #else
46 total = ZSTD_getDecompressedSize(src + inputmargin,
47 rq->inputsize - inputmargin);
48 #endif
49 if (rq->decodedskip || total != rq->decodedlength) {
50 buff = malloc(total);
51 if (!buff)
52 return -ENOMEM;
53 dest = buff;
54 }
55
56 ret = ZSTD_decompress(dest, total,
57 src + inputmargin, rq->inputsize - inputmargin);
58 if (ZSTD_isError(ret)) {
59 erofs_err("ZSTD decompress failed %d: %s", ZSTD_getErrorCode(ret),
60 ZSTD_getErrorName(ret));
61 ret = -EIO;
62 goto out;
63 }
64
65 if (ret != (int)total) {
66 erofs_err("ZSTD decompress length mismatch %d, expected %d",
67 ret, total);
68 goto out;
69 }
70 if (rq->decodedskip || total != rq->decodedlength)
71 memcpy(rq->out, dest + rq->decodedskip,
72 rq->decodedlength - rq->decodedskip);
73 out:
74 if (buff)
75 free(buff);
76 return ret;
77 }
78 #endif
79
80 #ifdef HAVE_QPL
81 #include <qpl/qpl.h>
82
83 struct z_erofs_qpl_job {
84 struct z_erofs_qpl_job *next;
85 u8 job[];
86 };
87 static struct z_erofs_qpl_job *z_erofs_qpl_jobs;
88 static unsigned int z_erofs_qpl_reclaim_quot;
89 #ifdef HAVE_PTHREAD_H
90 static pthread_mutex_t z_erofs_qpl_mutex;
91 #endif
92
z_erofs_load_deflate_config(struct erofs_sb_info * sbi,struct erofs_super_block * dsb,void * data,int size)93 int z_erofs_load_deflate_config(struct erofs_sb_info *sbi,
94 struct erofs_super_block *dsb, void *data, int size)
95 {
96 struct z_erofs_deflate_cfgs *dfl = data;
97 static erofs_atomic_bool_t inited;
98
99 if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) {
100 erofs_err("invalid deflate cfgs, size=%u", size);
101 return -EINVAL;
102 }
103
104 /*
105 * In Intel QPL, decompression is supported for DEFLATE streams where
106 * the size of the history buffer is no more than 4 KiB, otherwise
107 * QPL_STS_BAD_DIST_ERR code is returned.
108 */
109 sbi->useqpl = (dfl->windowbits <= 12);
110 if (sbi->useqpl) {
111 if (!erofs_atomic_test_and_set(&inited))
112 z_erofs_qpl_reclaim_quot = erofs_get_available_processors();
113 erofs_info("Intel QPL will be used for DEFLATE decompression");
114 }
115 return 0;
116 }
117
z_erofs_qpl_get_job(void)118 static qpl_job *z_erofs_qpl_get_job(void)
119 {
120 qpl_path_t execution_path = qpl_path_auto;
121 struct z_erofs_qpl_job *job;
122 int32_t jobsize = 0;
123 qpl_status status;
124
125 #ifdef HAVE_PTHREAD_H
126 pthread_mutex_lock(&z_erofs_qpl_mutex);
127 #endif
128 job = z_erofs_qpl_jobs;
129 if (job)
130 z_erofs_qpl_jobs = job->next;
131 #ifdef HAVE_PTHREAD_H
132 pthread_mutex_unlock(&z_erofs_qpl_mutex);
133 #endif
134
135 if (!job) {
136 status = qpl_get_job_size(execution_path, &jobsize);
137 if (status != QPL_STS_OK) {
138 erofs_err("failed to get job size: %d", status);
139 return ERR_PTR(-EOPNOTSUPP);
140 }
141
142 job = malloc(jobsize + sizeof(struct z_erofs_qpl_job));
143 if (!job)
144 return ERR_PTR(-ENOMEM);
145
146 status = qpl_init_job(execution_path, (qpl_job *)job->job);
147 if (status != QPL_STS_OK) {
148 erofs_err("failed to initialize job: %d", status);
149 return ERR_PTR(-EOPNOTSUPP);
150 }
151 erofs_atomic_dec_return(&z_erofs_qpl_reclaim_quot);
152 }
153 return (qpl_job *)job->job;
154 }
155
z_erofs_qpl_put_job(qpl_job * qjob)156 static bool z_erofs_qpl_put_job(qpl_job *qjob)
157 {
158 struct z_erofs_qpl_job *job =
159 container_of((void *)qjob, struct z_erofs_qpl_job, job);
160
161 if (erofs_atomic_inc_return(&z_erofs_qpl_reclaim_quot) <= 0) {
162 qpl_status status = qpl_fini_job(qjob);
163
164 free(job);
165 if (status != QPL_STS_OK)
166 erofs_err("failed to finalize job: %d", status);
167 return status == QPL_STS_OK;
168 }
169 #ifdef HAVE_PTHREAD_H
170 pthread_mutex_lock(&z_erofs_qpl_mutex);
171 #endif
172 job->next = z_erofs_qpl_jobs;
173 z_erofs_qpl_jobs = job;
174 #ifdef HAVE_PTHREAD_H
175 pthread_mutex_unlock(&z_erofs_qpl_mutex);
176 #endif
177 return true;
178 }
179
z_erofs_decompress_qpl(struct z_erofs_decompress_req * rq)180 static int z_erofs_decompress_qpl(struct z_erofs_decompress_req *rq)
181 {
182 u8 *dest = (u8 *)rq->out;
183 u8 *src = (u8 *)rq->in;
184 u8 *buff = NULL;
185 unsigned int inputmargin;
186 qpl_status status;
187 qpl_job *job;
188 int ret;
189
190 job = z_erofs_qpl_get_job();
191 if (IS_ERR(job))
192 return PTR_ERR(job);
193
194 inputmargin = z_erofs_fixup_insize(src, rq->inputsize);
195 if (inputmargin >= rq->inputsize)
196 return -EFSCORRUPTED;
197
198 if (rq->decodedskip) {
199 buff = malloc(rq->decodedlength);
200 if (!buff)
201 return -ENOMEM;
202 dest = buff;
203 }
204
205 job->op = qpl_op_decompress;
206 job->next_in_ptr = src + inputmargin;
207 job->next_out_ptr = dest;
208 job->available_in = rq->inputsize - inputmargin;
209 job->available_out = rq->decodedlength;
210 job->flags = QPL_FLAG_FIRST | QPL_FLAG_LAST;
211 status = qpl_execute_job(job);
212 if (status != QPL_STS_OK) {
213 erofs_err("failed to decompress: %d", status);
214 ret = -EIO;
215 goto out_inflate_end;
216 }
217
218 if (rq->decodedskip)
219 memcpy(rq->out, dest + rq->decodedskip,
220 rq->decodedlength - rq->decodedskip);
221 ret = 0;
222 out_inflate_end:
223 if (!z_erofs_qpl_put_job(job))
224 ret = -EFAULT;
225 if (buff)
226 free(buff);
227 return ret;
228 }
229 #else
z_erofs_load_deflate_config(struct erofs_sb_info * sbi,struct erofs_super_block * dsb,void * data,int size)230 int z_erofs_load_deflate_config(struct erofs_sb_info *sbi,
231 struct erofs_super_block *dsb, void *data, int size)
232 {
233 return 0;
234 }
235 #endif
236
237 #ifdef HAVE_LIBDEFLATE
238 /* if libdeflate is available, use libdeflate instead. */
239 #include <libdeflate.h>
240
z_erofs_decompress_deflate(struct z_erofs_decompress_req * rq)241 static int z_erofs_decompress_deflate(struct z_erofs_decompress_req *rq)
242 {
243 u8 *dest = (u8 *)rq->out;
244 u8 *src = (u8 *)rq->in;
245 u8 *buff = NULL;
246 size_t actual_out;
247 unsigned int inputmargin;
248 struct libdeflate_decompressor *inf;
249 enum libdeflate_result ret;
250 unsigned int decodedcapacity;
251
252 inputmargin = z_erofs_fixup_insize(src, rq->inputsize);
253 if (inputmargin >= rq->inputsize)
254 return -EFSCORRUPTED;
255
256 decodedcapacity = rq->decodedlength << (4 * rq->partial_decoding);
257 if (rq->decodedskip || rq->partial_decoding) {
258 buff = malloc(decodedcapacity);
259 if (!buff)
260 return -ENOMEM;
261 dest = buff;
262 }
263
264 inf = libdeflate_alloc_decompressor();
265 if (!inf) {
266 ret = -ENOMEM;
267 goto out_free_mem;
268 }
269
270 if (rq->partial_decoding) {
271 while (1) {
272 ret = libdeflate_deflate_decompress(inf, src + inputmargin,
273 rq->inputsize - inputmargin, dest,
274 decodedcapacity, &actual_out);
275 if (ret == LIBDEFLATE_SUCCESS)
276 break;
277 if (ret != LIBDEFLATE_INSUFFICIENT_SPACE) {
278 ret = -EIO;
279 goto out_inflate_end;
280 }
281 decodedcapacity = decodedcapacity << 1;
282 dest = realloc(buff, decodedcapacity);
283 if (!dest) {
284 ret = -ENOMEM;
285 goto out_inflate_end;
286 }
287 buff = dest;
288 }
289
290 if (actual_out < rq->decodedlength) {
291 ret = -EIO;
292 goto out_inflate_end;
293 }
294 } else {
295 ret = libdeflate_deflate_decompress(inf, src + inputmargin,
296 rq->inputsize - inputmargin, dest,
297 rq->decodedlength, NULL);
298 if (ret != LIBDEFLATE_SUCCESS) {
299 ret = -EIO;
300 goto out_inflate_end;
301 }
302 }
303
304 if (rq->decodedskip || rq->partial_decoding)
305 memcpy(rq->out, dest + rq->decodedskip,
306 rq->decodedlength - rq->decodedskip);
307
308 out_inflate_end:
309 libdeflate_free_decompressor(inf);
310 out_free_mem:
311 if (buff)
312 free(buff);
313 return ret;
314 }
315 #elif defined(HAVE_ZLIB)
316 #include <zlib.h>
317
318 /* report a zlib or i/o error */
zerr(int ret)319 static int zerr(int ret)
320 {
321 switch (ret) {
322 case Z_STREAM_ERROR:
323 return -EINVAL;
324 case Z_DATA_ERROR:
325 return -EIO;
326 case Z_MEM_ERROR:
327 return -ENOMEM;
328 case Z_ERRNO:
329 case Z_VERSION_ERROR:
330 default:
331 return -EFAULT;
332 }
333 }
334
z_erofs_decompress_deflate(struct z_erofs_decompress_req * rq)335 static int z_erofs_decompress_deflate(struct z_erofs_decompress_req *rq)
336 {
337 u8 *dest = (u8 *)rq->out;
338 u8 *src = (u8 *)rq->in;
339 u8 *buff = NULL;
340 unsigned int inputmargin;
341 z_stream strm;
342 int ret;
343
344 inputmargin = z_erofs_fixup_insize(src, rq->inputsize);
345 if (inputmargin >= rq->inputsize)
346 return -EFSCORRUPTED;
347
348 if (rq->decodedskip) {
349 buff = malloc(rq->decodedlength);
350 if (!buff)
351 return -ENOMEM;
352 dest = buff;
353 }
354
355 /* allocate inflate state */
356 strm.zalloc = Z_NULL;
357 strm.zfree = Z_NULL;
358 strm.opaque = Z_NULL;
359 strm.avail_in = 0;
360 strm.next_in = Z_NULL;
361 ret = inflateInit2(&strm, -15);
362 if (ret != Z_OK) {
363 free(buff);
364 return zerr(ret);
365 }
366
367 strm.next_in = src + inputmargin;
368 strm.avail_in = rq->inputsize - inputmargin;
369 strm.next_out = dest;
370 strm.avail_out = rq->decodedlength;
371
372 ret = inflate(&strm, rq->partial_decoding ? Z_SYNC_FLUSH : Z_FINISH);
373 if (ret != Z_STREAM_END || strm.total_out != rq->decodedlength) {
374 if (ret != Z_OK || !rq->partial_decoding) {
375 ret = zerr(ret);
376 goto out_inflate_end;
377 }
378 }
379
380 if (rq->decodedskip)
381 memcpy(rq->out, dest + rq->decodedskip,
382 rq->decodedlength - rq->decodedskip);
383
384 out_inflate_end:
385 inflateEnd(&strm);
386 if (buff)
387 free(buff);
388 return ret;
389 }
390 #endif
391
392 #ifdef HAVE_LIBLZMA
393 #include <lzma.h>
394
z_erofs_decompress_lzma(struct z_erofs_decompress_req * rq)395 static int z_erofs_decompress_lzma(struct z_erofs_decompress_req *rq)
396 {
397 int ret = 0;
398 u8 *dest = (u8 *)rq->out;
399 u8 *src = (u8 *)rq->in;
400 u8 *buff = NULL;
401 unsigned int inputmargin;
402 lzma_stream strm;
403 lzma_ret ret2;
404
405 inputmargin = z_erofs_fixup_insize(src, rq->inputsize);
406 if (inputmargin >= rq->inputsize)
407 return -EFSCORRUPTED;
408
409 if (rq->decodedskip) {
410 buff = malloc(rq->decodedlength);
411 if (!buff)
412 return -ENOMEM;
413 dest = buff;
414 }
415
416 strm = (lzma_stream)LZMA_STREAM_INIT;
417 strm.next_in = src + inputmargin;
418 strm.avail_in = rq->inputsize - inputmargin;
419 strm.next_out = dest;
420 strm.avail_out = rq->decodedlength;
421
422 ret2 = lzma_microlzma_decoder(&strm, strm.avail_in, rq->decodedlength,
423 !rq->partial_decoding,
424 Z_EROFS_LZMA_MAX_DICT_SIZE);
425 if (ret2 != LZMA_OK) {
426 erofs_err("fail to initialize lzma decoder %u", ret2 | 0U);
427 ret = -EFAULT;
428 goto out;
429 }
430
431 ret2 = lzma_code(&strm, LZMA_FINISH);
432 if (ret2 != LZMA_STREAM_END) {
433 ret = -EFSCORRUPTED;
434 goto out_lzma_end;
435 }
436
437 if (rq->decodedskip)
438 memcpy(rq->out, dest + rq->decodedskip,
439 rq->decodedlength - rq->decodedskip);
440
441 out_lzma_end:
442 lzma_end(&strm);
443 out:
444 if (buff)
445 free(buff);
446 return ret;
447 }
448 #endif
449
450 #ifdef LZ4_ENABLED
451 #include <lz4.h>
452
z_erofs_decompress_lz4(struct z_erofs_decompress_req * rq)453 static int z_erofs_decompress_lz4(struct z_erofs_decompress_req *rq)
454 {
455 int ret = 0;
456 char *dest = rq->out;
457 char *src = rq->in;
458 char *buff = NULL;
459 bool support_0padding = false;
460 unsigned int inputmargin = 0;
461 struct erofs_sb_info *sbi = rq->sbi;
462
463 if (erofs_sb_has_lz4_0padding(sbi)) {
464 support_0padding = true;
465
466 inputmargin = z_erofs_fixup_insize((u8 *)src, rq->inputsize);
467 if (inputmargin >= rq->inputsize)
468 return -EFSCORRUPTED;
469 }
470
471 if (rq->decodedskip) {
472 buff = malloc(rq->decodedlength);
473 if (!buff)
474 return -ENOMEM;
475 dest = buff;
476 }
477
478 if (rq->partial_decoding || !support_0padding)
479 ret = LZ4_decompress_safe_partial(src + inputmargin, dest,
480 rq->inputsize - inputmargin,
481 rq->decodedlength, rq->decodedlength);
482 else
483 ret = LZ4_decompress_safe(src + inputmargin, dest,
484 rq->inputsize - inputmargin,
485 rq->decodedlength);
486
487 if (ret != (int)rq->decodedlength) {
488 erofs_err("failed to %s decompress %d in[%u, %u] out[%u]",
489 rq->partial_decoding ? "partial" : "full",
490 ret, rq->inputsize, inputmargin, rq->decodedlength);
491 ret = -EIO;
492 goto out;
493 }
494
495 if (rq->decodedskip)
496 memcpy(rq->out, dest + rq->decodedskip,
497 rq->decodedlength - rq->decodedskip);
498
499 out:
500 if (buff)
501 free(buff);
502
503 return ret;
504 }
505 #endif
506
z_erofs_decompress(struct z_erofs_decompress_req * rq)507 int z_erofs_decompress(struct z_erofs_decompress_req *rq)
508 {
509 struct erofs_sb_info *sbi = rq->sbi;
510
511 if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) {
512 unsigned int count, rightpart, skip;
513
514 /* XXX: should support inputsize >= erofs_blksiz(sbi) later */
515 if (rq->inputsize > erofs_blksiz(sbi))
516 return -EFSCORRUPTED;
517
518 if (rq->decodedlength > erofs_blksiz(sbi))
519 return -EFSCORRUPTED;
520
521 if (rq->decodedlength < rq->decodedskip)
522 return -EFSCORRUPTED;
523
524 count = rq->decodedlength - rq->decodedskip;
525 skip = erofs_blkoff(sbi, rq->interlaced_offset + rq->decodedskip);
526 rightpart = min(erofs_blksiz(sbi) - skip, count);
527 memcpy(rq->out, rq->in + skip, rightpart);
528 memcpy(rq->out + rightpart, rq->in, count - rightpart);
529 return 0;
530 } else if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) {
531 if (rq->decodedlength > rq->inputsize)
532 return -EFSCORRUPTED;
533
534 DBG_BUGON(rq->decodedlength < rq->decodedskip);
535 memcpy(rq->out, rq->in + rq->decodedskip,
536 rq->decodedlength - rq->decodedskip);
537 return 0;
538 }
539
540 #ifdef LZ4_ENABLED
541 if (rq->alg == Z_EROFS_COMPRESSION_LZ4)
542 return z_erofs_decompress_lz4(rq);
543 #endif
544 #ifdef HAVE_LIBLZMA
545 if (rq->alg == Z_EROFS_COMPRESSION_LZMA)
546 return z_erofs_decompress_lzma(rq);
547 #endif
548 #ifdef HAVE_QPL
549 if (rq->alg == Z_EROFS_COMPRESSION_DEFLATE && rq->sbi->useqpl)
550 if (!z_erofs_decompress_qpl(rq))
551 return 0;
552 #endif
553 #if defined(HAVE_ZLIB) || defined(HAVE_LIBDEFLATE)
554 if (rq->alg == Z_EROFS_COMPRESSION_DEFLATE)
555 return z_erofs_decompress_deflate(rq);
556 #endif
557 #ifdef HAVE_LIBZSTD
558 if (rq->alg == Z_EROFS_COMPRESSION_ZSTD)
559 return z_erofs_decompress_zstd(rq);
560 #endif
561 return -EOPNOTSUPP;
562 }
563
z_erofs_load_lz4_config(struct erofs_sb_info * sbi,struct erofs_super_block * dsb,void * data,int size)564 static int z_erofs_load_lz4_config(struct erofs_sb_info *sbi,
565 struct erofs_super_block *dsb, void *data, int size)
566 {
567 struct z_erofs_lz4_cfgs *lz4 = data;
568 u16 distance;
569
570 if (lz4) {
571 if (size < sizeof(struct z_erofs_lz4_cfgs)) {
572 erofs_err("invalid lz4 cfgs, size=%u", size);
573 return -EINVAL;
574 }
575 distance = le16_to_cpu(lz4->max_distance);
576
577 sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks);
578 if (!sbi->lz4.max_pclusterblks)
579 sbi->lz4.max_pclusterblks = 1; /* reserved case */
580 } else {
581 distance = le16_to_cpu(dsb->u1.lz4_max_distance);
582 sbi->lz4.max_pclusterblks = 1;
583 }
584 sbi->lz4.max_distance = distance;
585 return 0;
586 }
587
z_erofs_parse_cfgs(struct erofs_sb_info * sbi,struct erofs_super_block * dsb)588 int z_erofs_parse_cfgs(struct erofs_sb_info *sbi, struct erofs_super_block *dsb)
589 {
590 unsigned int algs, alg;
591 erofs_off_t offset;
592 int size, ret = 0;
593
594 if (!erofs_sb_has_compr_cfgs(sbi)) {
595 sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4;
596 return z_erofs_load_lz4_config(sbi, dsb, NULL, 0);
597 }
598
599 sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs);
600 if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) {
601 erofs_err("unidentified algorithms %x, please upgrade erofs-utils",
602 sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS);
603 return -EOPNOTSUPP;
604 }
605
606 offset = EROFS_SUPER_OFFSET + sbi->sb_size;
607 alg = 0;
608 for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) {
609 void *data;
610
611 if (!(algs & 1))
612 continue;
613
614 data = erofs_read_metadata(sbi, 0, &offset, &size);
615 if (IS_ERR(data)) {
616 ret = PTR_ERR(data);
617 break;
618 }
619
620 ret = 0;
621 if (alg == Z_EROFS_COMPRESSION_LZ4)
622 ret = z_erofs_load_lz4_config(sbi, dsb, data, size);
623 else if (alg == Z_EROFS_COMPRESSION_DEFLATE)
624 ret = z_erofs_load_deflate_config(sbi, dsb, data, size);
625 free(data);
626 if (ret)
627 break;
628 }
629 return ret;
630 }
631