1 // SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
2 /*
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * http://www.huawei.com/
5 * Created by Miao Xie <[email protected]>
6 * with heavy changes by Gao Xiang <[email protected]>
7 */
8 #ifndef _LARGEFILE64_SOURCE
9 #define _LARGEFILE64_SOURCE
10 #endif
11 #include <string.h>
12 #include <stdlib.h>
13 #include <unistd.h>
14 #include "erofs/print.h"
15 #include "erofs/cache.h"
16 #include "erofs/compress.h"
17 #include "erofs/dedupe.h"
18 #include "compressor.h"
19 #include "erofs/block_list.h"
20 #include "erofs/compress_hints.h"
21 #include "erofs/fragments.h"
22 #ifdef EROFS_MT_ENABLED
23 #include "erofs/workqueue.h"
24 #endif
25
26 /* compressing configuration specified by users */
27 struct erofs_compress_cfg {
28 struct erofs_compress handle;
29 unsigned int algorithmtype;
30 bool enable;
31 } erofs_ccfg[EROFS_MAX_COMPR_CFGS];
32
33 struct z_erofs_extent_item {
34 struct list_head list;
35 struct z_erofs_inmem_extent e;
36 };
37
38 struct z_erofs_compress_ictx { /* inode context */
39 struct erofs_inode *inode;
40 struct erofs_compress_cfg *ccfg;
41 int fd;
42 u64 fpos;
43
44 u32 tof_chksum;
45 bool fix_dedupedfrag;
46 bool fragemitted;
47
48 /* fields for write indexes */
49 u8 *metacur;
50 struct list_head extents;
51 u16 clusterofs;
52
53 int seg_num;
54
55 #if EROFS_MT_ENABLED
56 pthread_mutex_t mutex;
57 pthread_cond_t cond;
58 int nfini;
59
60 struct erofs_compress_work *mtworks;
61 #endif
62 };
63
64 struct z_erofs_compress_sctx { /* segment context */
65 struct z_erofs_compress_ictx *ictx;
66
67 u8 *queue;
68 struct list_head extents;
69 struct z_erofs_extent_item *pivot;
70
71 struct erofs_compress *chandle;
72 char *destbuf;
73
74 erofs_off_t remaining;
75 unsigned int head, tail;
76
77 unsigned int pclustersize;
78 erofs_blk_t blkaddr; /* pointing to the next blkaddr */
79 u16 clusterofs;
80
81 int seg_idx;
82
83 void *membuf;
84 erofs_off_t memoff;
85 };
86
87 #ifdef EROFS_MT_ENABLED
88 struct erofs_compress_wq_tls {
89 u8 *queue;
90 char *destbuf;
91 struct erofs_compress_cfg *ccfg;
92 };
93
94 struct erofs_compress_work {
95 /* Note: struct erofs_work must be the first member */
96 struct erofs_work work;
97 struct z_erofs_compress_sctx ctx;
98 struct erofs_compress_work *next;
99
100 unsigned int alg_id;
101 char *alg_name;
102 unsigned int comp_level;
103 unsigned int dict_size;
104
105 int errcode;
106 };
107
108 static struct {
109 struct erofs_workqueue wq;
110 struct erofs_compress_work *idle;
111 pthread_mutex_t mutex;
112 } z_erofs_mt_ctrl;
113 #endif
114
115 static bool z_erofs_mt_enabled;
116
117 #define Z_EROFS_LEGACY_MAP_HEADER_SIZE Z_EROFS_FULL_INDEX_ALIGN(0)
118
z_erofs_write_indexes_final(struct z_erofs_compress_ictx * ctx)119 static void z_erofs_write_indexes_final(struct z_erofs_compress_ictx *ctx)
120 {
121 const unsigned int type = Z_EROFS_LCLUSTER_TYPE_PLAIN;
122 struct z_erofs_lcluster_index di;
123
124 if (!ctx->clusterofs)
125 return;
126
127 di.di_clusterofs = cpu_to_le16(ctx->clusterofs);
128 di.di_u.blkaddr = 0;
129 di.di_advise = cpu_to_le16(type << Z_EROFS_LI_LCLUSTER_TYPE_BIT);
130
131 memcpy(ctx->metacur, &di, sizeof(di));
132 ctx->metacur += sizeof(di);
133 }
134
z_erofs_write_extent(struct z_erofs_compress_ictx * ctx,struct z_erofs_inmem_extent * e)135 static void z_erofs_write_extent(struct z_erofs_compress_ictx *ctx,
136 struct z_erofs_inmem_extent *e)
137 {
138 struct erofs_inode *inode = ctx->inode;
139 struct erofs_sb_info *sbi = inode->sbi;
140 unsigned int clusterofs = ctx->clusterofs;
141 unsigned int count = e->length;
142 unsigned int d0 = 0, d1 = (clusterofs + count) / erofs_blksiz(sbi);
143 struct z_erofs_lcluster_index di;
144 unsigned int type, advise;
145
146 DBG_BUGON(!count);
147 di.di_clusterofs = cpu_to_le16(ctx->clusterofs);
148
149 /* whether the tail-end (un)compressed block or not */
150 if (!d1) {
151 /*
152 * A lcluster cannot have three parts with the middle one which
153 * is well-compressed for !ztailpacking cases.
154 */
155 DBG_BUGON(!e->raw && !cfg.c_ztailpacking && !cfg.c_fragments);
156 DBG_BUGON(e->partial);
157 type = e->raw ? Z_EROFS_LCLUSTER_TYPE_PLAIN :
158 Z_EROFS_LCLUSTER_TYPE_HEAD1;
159 advise = type << Z_EROFS_LI_LCLUSTER_TYPE_BIT;
160 di.di_advise = cpu_to_le16(advise);
161
162 if (inode->datalayout == EROFS_INODE_COMPRESSED_FULL &&
163 !e->compressedblks)
164 di.di_u.blkaddr = cpu_to_le32(inode->fragmentoff >> 32);
165 else
166 di.di_u.blkaddr = cpu_to_le32(e->blkaddr);
167 memcpy(ctx->metacur, &di, sizeof(di));
168 ctx->metacur += sizeof(di);
169
170 /* don't add the final index if the tail-end block exists */
171 ctx->clusterofs = 0;
172 return;
173 }
174
175 do {
176 advise = 0;
177 /* XXX: big pcluster feature should be per-inode */
178 if (d0 == 1 && erofs_sb_has_big_pcluster(sbi)) {
179 type = Z_EROFS_LCLUSTER_TYPE_NONHEAD;
180 di.di_u.delta[0] = cpu_to_le16(e->compressedblks |
181 Z_EROFS_LI_D0_CBLKCNT);
182 di.di_u.delta[1] = cpu_to_le16(d1);
183 } else if (d0) {
184 type = Z_EROFS_LCLUSTER_TYPE_NONHEAD;
185
186 /*
187 * If the |Z_EROFS_VLE_DI_D0_CBLKCNT| bit is set, parser
188 * will interpret |delta[0]| as size of pcluster, rather
189 * than distance to last head cluster. Normally this
190 * isn't a problem, because uncompressed extent size are
191 * below Z_EROFS_VLE_DI_D0_CBLKCNT * BLOCK_SIZE = 8MB.
192 * But with large pcluster it's possible to go over this
193 * number, resulting in corrupted compressed indices.
194 * To solve this, we replace d0 with
195 * Z_EROFS_VLE_DI_D0_CBLKCNT-1.
196 */
197 if (d0 >= Z_EROFS_LI_D0_CBLKCNT)
198 di.di_u.delta[0] = cpu_to_le16(
199 Z_EROFS_LI_D0_CBLKCNT - 1);
200 else
201 di.di_u.delta[0] = cpu_to_le16(d0);
202 di.di_u.delta[1] = cpu_to_le16(d1);
203 } else {
204 type = e->raw ? Z_EROFS_LCLUSTER_TYPE_PLAIN :
205 Z_EROFS_LCLUSTER_TYPE_HEAD1;
206
207 if (inode->datalayout == EROFS_INODE_COMPRESSED_FULL &&
208 !e->compressedblks)
209 di.di_u.blkaddr = cpu_to_le32(inode->fragmentoff >> 32);
210 else
211 di.di_u.blkaddr = cpu_to_le32(e->blkaddr);
212
213 if (e->partial) {
214 DBG_BUGON(e->raw);
215 advise |= Z_EROFS_LI_PARTIAL_REF;
216 }
217 }
218 advise |= type << Z_EROFS_LI_LCLUSTER_TYPE_BIT;
219 di.di_advise = cpu_to_le16(advise);
220
221 memcpy(ctx->metacur, &di, sizeof(di));
222 ctx->metacur += sizeof(di);
223
224 count -= erofs_blksiz(sbi) - clusterofs;
225 clusterofs = 0;
226
227 ++d0;
228 --d1;
229 } while (clusterofs + count >= erofs_blksiz(sbi));
230
231 ctx->clusterofs = clusterofs + count;
232 }
233
z_erofs_write_indexes(struct z_erofs_compress_ictx * ctx)234 static void z_erofs_write_indexes(struct z_erofs_compress_ictx *ctx)
235 {
236 struct z_erofs_extent_item *ei, *n;
237
238 ctx->clusterofs = 0;
239 list_for_each_entry_safe(ei, n, &ctx->extents, list) {
240 z_erofs_write_extent(ctx, &ei->e);
241
242 list_del(&ei->list);
243 free(ei);
244 }
245 z_erofs_write_indexes_final(ctx);
246 }
247
z_erofs_need_refill(struct z_erofs_compress_sctx * ctx)248 static bool z_erofs_need_refill(struct z_erofs_compress_sctx *ctx)
249 {
250 const bool final = !ctx->remaining;
251 unsigned int qh_aligned, qh_after;
252 struct erofs_inode *inode = ctx->ictx->inode;
253
254 if (final || ctx->head < EROFS_CONFIG_COMPR_MAX_SZ)
255 return false;
256
257 qh_aligned = round_down(ctx->head, erofs_blksiz(inode->sbi));
258 qh_after = ctx->head - qh_aligned;
259 memmove(ctx->queue, ctx->queue + qh_aligned, ctx->tail - qh_aligned);
260 ctx->tail -= qh_aligned;
261 ctx->head = qh_after;
262 return true;
263 }
264
265 static struct z_erofs_extent_item dummy_pivot = {
266 .e.length = 0
267 };
268
z_erofs_commit_extent(struct z_erofs_compress_sctx * ctx,struct z_erofs_extent_item * ei)269 static void z_erofs_commit_extent(struct z_erofs_compress_sctx *ctx,
270 struct z_erofs_extent_item *ei)
271 {
272 if (ei == &dummy_pivot)
273 return;
274
275 list_add_tail(&ei->list, &ctx->extents);
276 ctx->clusterofs = (ctx->clusterofs + ei->e.length) &
277 (erofs_blksiz(ctx->ictx->inode->sbi) - 1);
278 }
279
z_erofs_compress_dedupe(struct z_erofs_compress_sctx * ctx,unsigned int * len)280 static int z_erofs_compress_dedupe(struct z_erofs_compress_sctx *ctx,
281 unsigned int *len)
282 {
283 struct erofs_inode *inode = ctx->ictx->inode;
284 const unsigned int lclustermask = (1 << inode->z_logical_clusterbits) - 1;
285 struct erofs_sb_info *sbi = inode->sbi;
286 struct z_erofs_extent_item *ei = ctx->pivot;
287
288 if (!ei)
289 return 0;
290
291 /*
292 * No need dedupe for packed inode since it is composed of
293 * fragments which have already been deduplicated.
294 */
295 if (erofs_is_packed_inode(inode))
296 goto out;
297
298 do {
299 struct z_erofs_dedupe_ctx dctx = {
300 .start = ctx->queue + ctx->head - ({ int rc;
301 if (ei->e.length <= erofs_blksiz(sbi))
302 rc = 0;
303 else if (ei->e.length - erofs_blksiz(sbi) >= ctx->head)
304 rc = ctx->head;
305 else
306 rc = ei->e.length - erofs_blksiz(sbi);
307 rc; }),
308 .end = ctx->queue + ctx->head + *len,
309 .cur = ctx->queue + ctx->head,
310 };
311 int delta;
312
313 if (z_erofs_dedupe_match(&dctx))
314 break;
315
316 DBG_BUGON(dctx.e.inlined);
317 delta = ctx->queue + ctx->head - dctx.cur;
318 /*
319 * For big pcluster dedupe, leave two indices at least to store
320 * CBLKCNT as the first step. Even laterly, an one-block
321 * decompresssion could be done as another try in practice.
322 */
323 if (dctx.e.compressedblks > 1 &&
324 ((ctx->clusterofs + ei->e.length - delta) & lclustermask) +
325 dctx.e.length < 2 * (lclustermask + 1))
326 break;
327
328 ctx->pivot = malloc(sizeof(struct z_erofs_extent_item));
329 if (!ctx->pivot) {
330 z_erofs_commit_extent(ctx, ei);
331 return -ENOMEM;
332 }
333
334 if (delta) {
335 DBG_BUGON(delta < 0);
336 DBG_BUGON(!ei->e.length);
337
338 /*
339 * For big pcluster dedupe, if we decide to shorten the
340 * previous big pcluster, make sure that the previous
341 * CBLKCNT is still kept.
342 */
343 if (ei->e.compressedblks > 1 &&
344 (ctx->clusterofs & lclustermask) + ei->e.length
345 - delta < 2 * (lclustermask + 1))
346 break;
347 ei->e.partial = true;
348 ei->e.length -= delta;
349 }
350
351 /* fall back to noncompact indexes for deduplication */
352 inode->z_advise &= ~Z_EROFS_ADVISE_COMPACTED_2B;
353 inode->datalayout = EROFS_INODE_COMPRESSED_FULL;
354 erofs_sb_set_dedupe(sbi);
355
356 sbi->saved_by_deduplication +=
357 dctx.e.compressedblks * erofs_blksiz(sbi);
358 erofs_dbg("Dedupe %u %scompressed data (delta %d) to %u of %u blocks",
359 dctx.e.length, dctx.e.raw ? "un" : "",
360 delta, dctx.e.blkaddr, dctx.e.compressedblks);
361
362 z_erofs_commit_extent(ctx, ei);
363 ei = ctx->pivot;
364 init_list_head(&ei->list);
365 ei->e = dctx.e;
366
367 ctx->head += dctx.e.length - delta;
368 DBG_BUGON(*len < dctx.e.length - delta);
369 *len -= dctx.e.length - delta;
370
371 if (z_erofs_need_refill(ctx))
372 return 1;
373 } while (*len);
374 out:
375 z_erofs_commit_extent(ctx, ei);
376 ctx->pivot = NULL;
377 return 0;
378 }
379
write_uncompressed_extent(struct z_erofs_compress_sctx * ctx,unsigned int len,char * dst)380 static int write_uncompressed_extent(struct z_erofs_compress_sctx *ctx,
381 unsigned int len, char *dst)
382 {
383 struct erofs_inode *inode = ctx->ictx->inode;
384 struct erofs_sb_info *sbi = inode->sbi;
385 unsigned int count = min(erofs_blksiz(sbi), len);
386 unsigned int interlaced_offset, rightpart;
387 int ret;
388
389 /* write interlaced uncompressed data if needed */
390 if (inode->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER)
391 interlaced_offset = ctx->clusterofs;
392 else
393 interlaced_offset = 0;
394 rightpart = min(erofs_blksiz(sbi) - interlaced_offset, count);
395
396 memset(dst, 0, erofs_blksiz(sbi));
397
398 memcpy(dst + interlaced_offset, ctx->queue + ctx->head, rightpart);
399 memcpy(dst, ctx->queue + ctx->head + rightpart, count - rightpart);
400
401 if (ctx->membuf) {
402 erofs_dbg("Writing %u uncompressed data of %s", count,
403 inode->i_srcpath);
404 memcpy(ctx->membuf + ctx->memoff, dst, erofs_blksiz(sbi));
405 ctx->memoff += erofs_blksiz(sbi);
406 } else {
407 erofs_dbg("Writing %u uncompressed data to block %u", count,
408 ctx->blkaddr);
409 ret = erofs_blk_write(sbi, dst, ctx->blkaddr, 1);
410 if (ret)
411 return ret;
412 }
413 return count;
414 }
415
z_erofs_get_max_pclustersize(struct erofs_inode * inode)416 static unsigned int z_erofs_get_max_pclustersize(struct erofs_inode *inode)
417 {
418 if (erofs_is_packed_inode(inode)) {
419 return cfg.c_mkfs_pclustersize_packed;
420 #ifndef NDEBUG
421 } else if (cfg.c_random_pclusterblks) {
422 unsigned int pclusterblks =
423 cfg.c_mkfs_pclustersize_max >> inode->sbi->blkszbits;
424
425 return (1 + rand() % pclusterblks) << inode->sbi->blkszbits;
426 #endif
427 } else if (cfg.c_compress_hints_file) {
428 z_erofs_apply_compress_hints(inode);
429 DBG_BUGON(!inode->z_physical_clusterblks);
430 return inode->z_physical_clusterblks << inode->sbi->blkszbits;
431 }
432 return cfg.c_mkfs_pclustersize_def;
433 }
434
z_erofs_fill_inline_data(struct erofs_inode * inode,void * data,unsigned int len,bool raw)435 static int z_erofs_fill_inline_data(struct erofs_inode *inode, void *data,
436 unsigned int len, bool raw)
437 {
438 inode->z_advise |= Z_EROFS_ADVISE_INLINE_PCLUSTER;
439 inode->idata_size = len;
440 inode->compressed_idata = !raw;
441
442 inode->idata = malloc(inode->idata_size);
443 if (!inode->idata)
444 return -ENOMEM;
445 erofs_dbg("Recording %u %scompressed inline data",
446 inode->idata_size, raw ? "un" : "");
447 memcpy(inode->idata, data, inode->idata_size);
448 return len;
449 }
450
tryrecompress_trailing(struct z_erofs_compress_sctx * ctx,struct erofs_compress * ec,void * in,unsigned int * insize,void * out,unsigned int * compressedsize)451 static void tryrecompress_trailing(struct z_erofs_compress_sctx *ctx,
452 struct erofs_compress *ec,
453 void *in, unsigned int *insize,
454 void *out, unsigned int *compressedsize)
455 {
456 struct erofs_sb_info *sbi = ctx->ictx->inode->sbi;
457 char tmp[Z_EROFS_PCLUSTER_MAX_SIZE];
458 unsigned int count;
459 int ret = *compressedsize;
460
461 /* no need to recompress */
462 if (!(ret & (erofs_blksiz(sbi) - 1)))
463 return;
464
465 count = *insize;
466 ret = erofs_compress_destsize(ec, in, &count, (void *)tmp,
467 rounddown(ret, erofs_blksiz(sbi)));
468 if (ret <= 0 || ret + (*insize - count) >=
469 roundup(*compressedsize, erofs_blksiz(sbi)))
470 return;
471
472 /* replace the original compressed data if any gain */
473 memcpy(out, tmp, ret);
474 *insize = count;
475 *compressedsize = ret;
476 }
477
z_erofs_fixup_deduped_fragment(struct z_erofs_compress_sctx * ctx,unsigned int len)478 static bool z_erofs_fixup_deduped_fragment(struct z_erofs_compress_sctx *ctx,
479 unsigned int len)
480 {
481 struct z_erofs_compress_ictx *ictx = ctx->ictx;
482 struct erofs_inode *inode = ictx->inode;
483 struct erofs_sb_info *sbi = inode->sbi;
484 const unsigned int newsize = ctx->remaining + len;
485
486 DBG_BUGON(!inode->fragment_size);
487
488 /* try to fix again if it gets larger (should be rare) */
489 if (inode->fragment_size < newsize) {
490 ctx->pclustersize = min_t(erofs_off_t,
491 z_erofs_get_max_pclustersize(inode),
492 roundup(newsize - inode->fragment_size,
493 erofs_blksiz(sbi)));
494 return false;
495 }
496
497 inode->fragmentoff += inode->fragment_size - newsize;
498 inode->fragment_size = newsize;
499
500 erofs_dbg("Reducing fragment size to %u at %llu",
501 inode->fragment_size, inode->fragmentoff | 0ULL);
502
503 /* it's the end */
504 DBG_BUGON(ctx->tail - ctx->head + ctx->remaining != newsize);
505 ctx->head = ctx->tail;
506 ctx->remaining = 0;
507 return true;
508 }
509
__z_erofs_compress_one(struct z_erofs_compress_sctx * ctx,struct z_erofs_inmem_extent * e)510 static int __z_erofs_compress_one(struct z_erofs_compress_sctx *ctx,
511 struct z_erofs_inmem_extent *e)
512 {
513 static char g_dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_MAX_BLOCK_SIZE];
514 char *dstbuf = ctx->destbuf ?: g_dstbuf;
515 struct z_erofs_compress_ictx *ictx = ctx->ictx;
516 struct erofs_inode *inode = ictx->inode;
517 struct erofs_sb_info *sbi = inode->sbi;
518 unsigned int blksz = erofs_blksiz(sbi);
519 char *const dst = dstbuf + blksz;
520 struct erofs_compress *const h = ctx->chandle;
521 unsigned int len = ctx->tail - ctx->head;
522 bool is_packed_inode = erofs_is_packed_inode(inode);
523 bool tsg = (ctx->seg_idx + 1 >= ictx->seg_num), final = !ctx->remaining;
524 bool may_packing = (cfg.c_fragments && tsg && final &&
525 !is_packed_inode && !z_erofs_mt_enabled);
526 bool may_inline = (cfg.c_ztailpacking && tsg && final && !may_packing);
527 unsigned int compressedsize;
528 int ret;
529
530 *e = (struct z_erofs_inmem_extent){};
531 if (len <= ctx->pclustersize) {
532 if (!final || !len)
533 return 1;
534 if (inode->fragment_size && !ictx->fix_dedupedfrag) {
535 ctx->pclustersize = roundup(len, blksz);
536 goto fix_dedupedfrag;
537 }
538 if (may_packing) {
539 e->length = len;
540 goto frag_packing;
541 }
542 if (!may_inline && len <= blksz)
543 goto nocompression;
544 }
545
546 e->length = min(len, cfg.c_max_decompressed_extent_bytes);
547 ret = erofs_compress_destsize(h, ctx->queue + ctx->head,
548 &e->length, dst, ctx->pclustersize);
549 if (ret <= 0) {
550 erofs_err("failed to compress %s: %s", inode->i_srcpath,
551 erofs_strerror(ret));
552 return ret;
553 }
554
555 compressedsize = ret;
556 /* even compressed size is smaller, there is no real gain */
557 if (!(may_inline && e->length == len && ret < blksz))
558 ret = roundup(ret, blksz);
559
560 /* check if there is enough gain to keep the compressed data */
561 if (ret * h->compress_threshold / 100 >= e->length) {
562 if (may_inline && len < blksz) {
563 ret = z_erofs_fill_inline_data(inode,
564 ctx->queue + ctx->head, len, true);
565 if (ret < 0)
566 return ret;
567 e->inlined = true;
568 } else {
569 may_inline = false;
570 may_packing = false;
571 nocompression:
572 /* TODO: reset clusterofs to 0 if permitted */
573 ret = write_uncompressed_extent(ctx, len, dst);
574 if (ret < 0)
575 return ret;
576 }
577 e->length = ret;
578
579 /*
580 * XXX: For now, we have to leave `ctx->compressedblk = 1'
581 * since there is no way to generate compressed indexes after
582 * the time that ztailpacking is decided.
583 */
584 e->compressedblks = 1;
585 e->raw = true;
586 } else if (may_packing && len == e->length &&
587 compressedsize < ctx->pclustersize &&
588 (!inode->fragment_size || ictx->fix_dedupedfrag)) {
589 frag_packing:
590 ret = z_erofs_pack_fragments(inode, ctx->queue + ctx->head,
591 len, ictx->tof_chksum);
592 if (ret < 0)
593 return ret;
594 e->compressedblks = 0; /* indicate a fragment */
595 e->raw = false;
596 ictx->fragemitted = true;
597 /* tailpcluster should be less than 1 block */
598 } else if (may_inline && len == e->length && compressedsize < blksz) {
599 if (ctx->clusterofs + len <= blksz) {
600 inode->eof_tailraw = malloc(len);
601 if (!inode->eof_tailraw)
602 return -ENOMEM;
603
604 memcpy(inode->eof_tailraw, ctx->queue + ctx->head, len);
605 inode->eof_tailrawsize = len;
606 }
607
608 ret = z_erofs_fill_inline_data(inode, dst,
609 compressedsize, false);
610 if (ret < 0)
611 return ret;
612 e->inlined = true;
613 e->compressedblks = 1;
614 e->raw = false;
615 } else {
616 unsigned int tailused, padding;
617
618 /*
619 * If there's space left for the last round when deduping
620 * fragments, try to read the fragment and recompress a little
621 * more to check whether it can be filled up. Fix the fragment
622 * if succeeds. Otherwise, just drop it and go on packing.
623 */
624 if (may_packing && len == e->length &&
625 (compressedsize & (blksz - 1)) &&
626 ctx->tail < Z_EROFS_COMPR_QUEUE_SZ) {
627 ctx->pclustersize = roundup(compressedsize, blksz);
628 goto fix_dedupedfrag;
629 }
630
631 if (may_inline && len == e->length)
632 tryrecompress_trailing(ctx, h, ctx->queue + ctx->head,
633 &e->length, dst, &compressedsize);
634
635 e->compressedblks = BLK_ROUND_UP(sbi, compressedsize);
636 DBG_BUGON(e->compressedblks * blksz >= e->length);
637
638 padding = 0;
639 tailused = compressedsize & (blksz - 1);
640 if (tailused)
641 padding = blksz - tailused;
642
643 /* zero out garbage trailing data for non-0padding */
644 if (!erofs_sb_has_lz4_0padding(sbi)) {
645 memset(dst + compressedsize, 0, padding);
646 padding = 0;
647 }
648
649 /* write compressed data */
650 if (ctx->membuf) {
651 erofs_dbg("Writing %u compressed data of %u blocks of %s",
652 e->length, e->compressedblks, inode->i_srcpath);
653
654 memcpy(ctx->membuf + ctx->memoff, dst - padding,
655 e->compressedblks * blksz);
656 ctx->memoff += e->compressedblks * blksz;
657 } else {
658 erofs_dbg("Writing %u compressed data to %u of %u blocks",
659 e->length, ctx->blkaddr, e->compressedblks);
660
661 ret = erofs_blk_write(sbi, dst - padding, ctx->blkaddr,
662 e->compressedblks);
663 if (ret)
664 return ret;
665 }
666 e->raw = false;
667 may_inline = false;
668 may_packing = false;
669 }
670 e->partial = false;
671 e->blkaddr = ctx->blkaddr;
672 if (ctx->blkaddr != EROFS_NULL_ADDR)
673 ctx->blkaddr += e->compressedblks;
674 if (!may_inline && !may_packing && !is_packed_inode)
675 (void)z_erofs_dedupe_insert(e, ctx->queue + ctx->head);
676 ctx->head += e->length;
677 return 0;
678
679 fix_dedupedfrag:
680 DBG_BUGON(!inode->fragment_size);
681 ctx->remaining += inode->fragment_size;
682 ictx->fix_dedupedfrag = true;
683 return 1;
684 }
685
z_erofs_compress_one(struct z_erofs_compress_sctx * ctx)686 static int z_erofs_compress_one(struct z_erofs_compress_sctx *ctx)
687 {
688 struct z_erofs_compress_ictx *ictx = ctx->ictx;
689 unsigned int len = ctx->tail - ctx->head;
690 struct z_erofs_extent_item *ei;
691
692 while (len) {
693 int ret = z_erofs_compress_dedupe(ctx, &len);
694
695 if (ret > 0)
696 break;
697 else if (ret < 0)
698 return ret;
699
700 DBG_BUGON(ctx->pivot);
701 ei = malloc(sizeof(*ei));
702 if (!ei)
703 return -ENOMEM;
704
705 init_list_head(&ei->list);
706 ret = __z_erofs_compress_one(ctx, &ei->e);
707 if (ret) {
708 free(ei);
709 if (ret > 0)
710 break; /* need more data */
711 return ret;
712 }
713
714 len -= ei->e.length;
715 ctx->pivot = ei;
716 if (ictx->fix_dedupedfrag && !ictx->fragemitted &&
717 z_erofs_fixup_deduped_fragment(ctx, len))
718 break;
719
720 if (z_erofs_need_refill(ctx))
721 break;
722 }
723 return 0;
724 }
725
726 struct z_erofs_compressindex_vec {
727 union {
728 erofs_blk_t blkaddr;
729 u16 delta[2];
730 } u;
731 u16 clusterofs;
732 u8 clustertype;
733 };
734
parse_legacy_indexes(struct z_erofs_compressindex_vec * cv,unsigned int nr,void * metacur)735 static void *parse_legacy_indexes(struct z_erofs_compressindex_vec *cv,
736 unsigned int nr, void *metacur)
737 {
738 struct z_erofs_lcluster_index *const db = metacur;
739 unsigned int i;
740
741 for (i = 0; i < nr; ++i, ++cv) {
742 struct z_erofs_lcluster_index *const di = db + i;
743 const unsigned int advise = le16_to_cpu(di->di_advise);
744
745 cv->clustertype = (advise >> Z_EROFS_LI_LCLUSTER_TYPE_BIT) &
746 ((1 << Z_EROFS_LI_LCLUSTER_TYPE_BITS) - 1);
747 cv->clusterofs = le16_to_cpu(di->di_clusterofs);
748
749 if (cv->clustertype == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
750 cv->u.delta[0] = le16_to_cpu(di->di_u.delta[0]);
751 cv->u.delta[1] = le16_to_cpu(di->di_u.delta[1]);
752 } else {
753 cv->u.blkaddr = le32_to_cpu(di->di_u.blkaddr);
754 }
755 }
756 return db + nr;
757 }
758
write_compacted_indexes(u8 * out,struct z_erofs_compressindex_vec * cv,erofs_blk_t * blkaddr_ret,unsigned int destsize,unsigned int lclusterbits,bool final,bool * dummy_head,bool update_blkaddr)759 static void *write_compacted_indexes(u8 *out,
760 struct z_erofs_compressindex_vec *cv,
761 erofs_blk_t *blkaddr_ret,
762 unsigned int destsize,
763 unsigned int lclusterbits,
764 bool final, bool *dummy_head,
765 bool update_blkaddr)
766 {
767 unsigned int vcnt, lobits, encodebits, pos, i, cblks;
768 erofs_blk_t blkaddr;
769
770 if (destsize == 4)
771 vcnt = 2;
772 else if (destsize == 2 && lclusterbits <= 12)
773 vcnt = 16;
774 else
775 return ERR_PTR(-EINVAL);
776 lobits = max(lclusterbits, ilog2(Z_EROFS_LI_D0_CBLKCNT) + 1U);
777 encodebits = (vcnt * destsize * 8 - 32) / vcnt;
778 blkaddr = *blkaddr_ret;
779
780 pos = 0;
781 for (i = 0; i < vcnt; ++i) {
782 unsigned int offset, v;
783 u8 ch, rem;
784
785 if (cv[i].clustertype == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
786 if (cv[i].u.delta[0] & Z_EROFS_LI_D0_CBLKCNT) {
787 cblks = cv[i].u.delta[0] & ~Z_EROFS_LI_D0_CBLKCNT;
788 offset = cv[i].u.delta[0];
789 blkaddr += cblks;
790 *dummy_head = false;
791 } else if (i + 1 == vcnt) {
792 offset = min_t(u16, cv[i].u.delta[1],
793 (1 << lobits) - 1);
794 } else {
795 offset = cv[i].u.delta[0];
796 }
797 } else {
798 offset = cv[i].clusterofs;
799 if (*dummy_head) {
800 ++blkaddr;
801 if (update_blkaddr)
802 *blkaddr_ret = blkaddr;
803 }
804 *dummy_head = true;
805 update_blkaddr = false;
806
807 if (cv[i].u.blkaddr != blkaddr) {
808 if (i + 1 != vcnt)
809 DBG_BUGON(!final);
810 DBG_BUGON(cv[i].u.blkaddr);
811 }
812 }
813 v = (cv[i].clustertype << lobits) | offset;
814 rem = pos & 7;
815 ch = out[pos / 8] & ((1 << rem) - 1);
816 out[pos / 8] = (v << rem) | ch;
817 out[pos / 8 + 1] = v >> (8 - rem);
818 out[pos / 8 + 2] = v >> (16 - rem);
819 pos += encodebits;
820 }
821 DBG_BUGON(destsize * vcnt * 8 != pos + 32);
822 *(__le32 *)(out + destsize * vcnt - 4) = cpu_to_le32(*blkaddr_ret);
823 *blkaddr_ret = blkaddr;
824 return out + destsize * vcnt;
825 }
826
z_erofs_convert_to_compacted_format(struct erofs_inode * inode,erofs_blk_t blkaddr,unsigned int legacymetasize,void * compressmeta)827 int z_erofs_convert_to_compacted_format(struct erofs_inode *inode,
828 erofs_blk_t blkaddr,
829 unsigned int legacymetasize,
830 void *compressmeta)
831 {
832 const unsigned int mpos = roundup(inode->inode_isize +
833 inode->xattr_isize, 8) +
834 sizeof(struct z_erofs_map_header);
835 const unsigned int totalidx = (legacymetasize -
836 Z_EROFS_LEGACY_MAP_HEADER_SIZE) /
837 sizeof(struct z_erofs_lcluster_index);
838 const unsigned int logical_clusterbits = inode->z_logical_clusterbits;
839 u8 *out, *in;
840 struct z_erofs_compressindex_vec cv[16];
841 struct erofs_sb_info *sbi = inode->sbi;
842 /* # of 8-byte units so that it can be aligned with 32 bytes */
843 unsigned int compacted_4b_initial, compacted_4b_end;
844 unsigned int compacted_2b;
845 bool dummy_head;
846 bool big_pcluster = erofs_sb_has_big_pcluster(sbi);
847
848 if (logical_clusterbits < sbi->blkszbits)
849 return -EINVAL;
850 if (logical_clusterbits > 14) {
851 erofs_err("compact format is unsupported for lcluster size %u",
852 1 << logical_clusterbits);
853 return -EOPNOTSUPP;
854 }
855
856 if (inode->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) {
857 if (logical_clusterbits > 12) {
858 erofs_err("compact 2B is unsupported for lcluster size %u",
859 1 << logical_clusterbits);
860 return -EINVAL;
861 }
862
863 compacted_4b_initial = (32 - mpos % 32) / 4;
864 if (compacted_4b_initial == 32 / 4)
865 compacted_4b_initial = 0;
866
867 if (compacted_4b_initial > totalidx) {
868 compacted_4b_initial = compacted_2b = 0;
869 compacted_4b_end = totalidx;
870 } else {
871 compacted_2b = rounddown(totalidx -
872 compacted_4b_initial, 16);
873 compacted_4b_end = totalidx - compacted_4b_initial -
874 compacted_2b;
875 }
876 } else {
877 compacted_2b = compacted_4b_initial = 0;
878 compacted_4b_end = totalidx;
879 }
880
881 out = in = compressmeta;
882
883 out += sizeof(struct z_erofs_map_header);
884 in += Z_EROFS_LEGACY_MAP_HEADER_SIZE;
885
886 dummy_head = false;
887 /* prior to bigpcluster, blkaddr was bumped up once coming into HEAD */
888 if (!big_pcluster) {
889 --blkaddr;
890 dummy_head = true;
891 }
892
893 /* generate compacted_4b_initial */
894 while (compacted_4b_initial) {
895 in = parse_legacy_indexes(cv, 2, in);
896 out = write_compacted_indexes(out, cv, &blkaddr,
897 4, logical_clusterbits, false,
898 &dummy_head, big_pcluster);
899 compacted_4b_initial -= 2;
900 }
901 DBG_BUGON(compacted_4b_initial);
902
903 /* generate compacted_2b */
904 while (compacted_2b) {
905 in = parse_legacy_indexes(cv, 16, in);
906 out = write_compacted_indexes(out, cv, &blkaddr,
907 2, logical_clusterbits, false,
908 &dummy_head, big_pcluster);
909 compacted_2b -= 16;
910 }
911 DBG_BUGON(compacted_2b);
912
913 /* generate compacted_4b_end */
914 while (compacted_4b_end > 1) {
915 in = parse_legacy_indexes(cv, 2, in);
916 out = write_compacted_indexes(out, cv, &blkaddr,
917 4, logical_clusterbits, false,
918 &dummy_head, big_pcluster);
919 compacted_4b_end -= 2;
920 }
921
922 /* generate final compacted_4b_end if needed */
923 if (compacted_4b_end) {
924 memset(cv, 0, sizeof(cv));
925 in = parse_legacy_indexes(cv, 1, in);
926 out = write_compacted_indexes(out, cv, &blkaddr,
927 4, logical_clusterbits, true,
928 &dummy_head, big_pcluster);
929 }
930 inode->extent_isize = out - (u8 *)compressmeta;
931 return 0;
932 }
933
z_erofs_write_mapheader(struct erofs_inode * inode,void * compressmeta)934 static void z_erofs_write_mapheader(struct erofs_inode *inode,
935 void *compressmeta)
936 {
937 struct erofs_sb_info *sbi = inode->sbi;
938 struct z_erofs_map_header h = {
939 .h_advise = cpu_to_le16(inode->z_advise),
940 .h_algorithmtype = inode->z_algorithmtype[1] << 4 |
941 inode->z_algorithmtype[0],
942 /* lclustersize */
943 .h_clusterbits = inode->z_logical_clusterbits - sbi->blkszbits,
944 };
945
946 if (inode->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER)
947 h.h_fragmentoff = cpu_to_le32(inode->fragmentoff);
948 else
949 h.h_idata_size = cpu_to_le16(inode->idata_size);
950
951 memset(compressmeta, 0, Z_EROFS_LEGACY_MAP_HEADER_SIZE);
952 /* write out map header */
953 memcpy(compressmeta, &h, sizeof(struct z_erofs_map_header));
954 }
955
z_erofs_drop_inline_pcluster(struct erofs_inode * inode)956 void z_erofs_drop_inline_pcluster(struct erofs_inode *inode)
957 {
958 struct erofs_sb_info *sbi = inode->sbi;
959 const unsigned int type = Z_EROFS_LCLUSTER_TYPE_PLAIN;
960 struct z_erofs_map_header *h = inode->compressmeta;
961
962 h->h_advise = cpu_to_le16(le16_to_cpu(h->h_advise) &
963 ~Z_EROFS_ADVISE_INLINE_PCLUSTER);
964 h->h_idata_size = 0;
965 if (!inode->eof_tailraw)
966 return;
967 DBG_BUGON(inode->compressed_idata != true);
968
969 /* patch the EOF lcluster to uncompressed type first */
970 if (inode->datalayout == EROFS_INODE_COMPRESSED_FULL) {
971 struct z_erofs_lcluster_index *di =
972 (inode->compressmeta + inode->extent_isize) -
973 sizeof(struct z_erofs_lcluster_index);
974 __le16 advise =
975 cpu_to_le16(type << Z_EROFS_LI_LCLUSTER_TYPE_BIT);
976
977 di->di_advise = advise;
978 } else if (inode->datalayout == EROFS_INODE_COMPRESSED_COMPACT) {
979 /* handle the last compacted 4B pack */
980 unsigned int eofs, base, pos, v, lo;
981 u8 *out;
982
983 eofs = inode->extent_isize -
984 (4 << (BLK_ROUND_UP(sbi, inode->i_size) & 1));
985 base = round_down(eofs, 8);
986 pos = 16 /* encodebits */ * ((eofs - base) / 4);
987 out = inode->compressmeta + base;
988 lo = erofs_blkoff(sbi, get_unaligned_le32(out + pos / 8));
989 v = (type << sbi->blkszbits) | lo;
990 out[pos / 8] = v & 0xff;
991 out[pos / 8 + 1] = v >> 8;
992 } else {
993 DBG_BUGON(1);
994 return;
995 }
996 free(inode->idata);
997 /* replace idata with prepared uncompressed data */
998 inode->idata = inode->eof_tailraw;
999 inode->idata_size = inode->eof_tailrawsize;
1000 inode->compressed_idata = false;
1001 inode->eof_tailraw = NULL;
1002 }
1003
z_erofs_compress_segment(struct z_erofs_compress_sctx * ctx,u64 offset,erofs_blk_t blkaddr)1004 int z_erofs_compress_segment(struct z_erofs_compress_sctx *ctx,
1005 u64 offset, erofs_blk_t blkaddr)
1006 {
1007 struct z_erofs_compress_ictx *ictx = ctx->ictx;
1008 int fd = ictx->fd;
1009
1010 ctx->blkaddr = blkaddr;
1011 while (ctx->remaining) {
1012 const u64 rx = min_t(u64, ctx->remaining,
1013 Z_EROFS_COMPR_QUEUE_SZ - ctx->tail);
1014 int ret;
1015
1016 ret = (offset == -1 ?
1017 read(fd, ctx->queue + ctx->tail, rx) :
1018 pread(fd, ctx->queue + ctx->tail, rx,
1019 ictx->fpos + offset));
1020 if (ret != rx)
1021 return -errno;
1022
1023 ctx->remaining -= rx;
1024 ctx->tail += rx;
1025 if (offset != -1)
1026 offset += rx;
1027
1028 ret = z_erofs_compress_one(ctx);
1029 if (ret)
1030 return ret;
1031 }
1032 DBG_BUGON(ctx->head != ctx->tail);
1033
1034 if (ctx->pivot) {
1035 z_erofs_commit_extent(ctx, ctx->pivot);
1036 ctx->pivot = NULL;
1037 }
1038
1039 /* generate an extra extent for the deduplicated fragment */
1040 if (ctx->seg_idx >= ictx->seg_num - 1 &&
1041 ictx->inode->fragment_size && !ictx->fragemitted) {
1042 struct z_erofs_extent_item *ei;
1043
1044 ei = malloc(sizeof(*ei));
1045 if (!ei)
1046 return -ENOMEM;
1047
1048 ei->e = (struct z_erofs_inmem_extent) {
1049 .length = ictx->inode->fragment_size,
1050 .compressedblks = 0,
1051 .raw = false,
1052 .partial = false,
1053 .blkaddr = ctx->blkaddr,
1054 };
1055 init_list_head(&ei->list);
1056 z_erofs_commit_extent(ctx, ei);
1057 }
1058 return 0;
1059 }
1060
erofs_commit_compressed_file(struct z_erofs_compress_ictx * ictx,struct erofs_buffer_head * bh,erofs_blk_t blkaddr,erofs_blk_t compressed_blocks)1061 int erofs_commit_compressed_file(struct z_erofs_compress_ictx *ictx,
1062 struct erofs_buffer_head *bh,
1063 erofs_blk_t blkaddr,
1064 erofs_blk_t compressed_blocks)
1065 {
1066 struct erofs_inode *inode = ictx->inode;
1067 struct erofs_sb_info *sbi = inode->sbi;
1068 unsigned int legacymetasize;
1069 u8 *compressmeta;
1070 int ret;
1071
1072 z_erofs_fragments_commit(inode);
1073
1074 /* fall back to no compression mode */
1075 DBG_BUGON(compressed_blocks < !!inode->idata_size);
1076 compressed_blocks -= !!inode->idata_size;
1077
1078 compressmeta = malloc(BLK_ROUND_UP(sbi, inode->i_size) *
1079 sizeof(struct z_erofs_lcluster_index) +
1080 Z_EROFS_LEGACY_MAP_HEADER_SIZE);
1081 if (!compressmeta) {
1082 ret = -ENOMEM;
1083 goto err_free_idata;
1084 }
1085 ictx->metacur = compressmeta + Z_EROFS_LEGACY_MAP_HEADER_SIZE;
1086 z_erofs_write_indexes(ictx);
1087
1088 legacymetasize = ictx->metacur - compressmeta;
1089 /* estimate if data compression saves space or not */
1090 if (!inode->fragment_size &&
1091 compressed_blocks * erofs_blksiz(sbi) + inode->idata_size +
1092 legacymetasize >= inode->i_size) {
1093 z_erofs_dedupe_commit(true);
1094 ret = -ENOSPC;
1095 goto err_free_meta;
1096 }
1097 z_erofs_dedupe_commit(false);
1098 z_erofs_write_mapheader(inode, compressmeta);
1099
1100 if (!ictx->fragemitted)
1101 sbi->saved_by_deduplication += inode->fragment_size;
1102
1103 /* if the entire file is a fragment, a simplified form is used. */
1104 if (inode->i_size <= inode->fragment_size) {
1105 DBG_BUGON(inode->i_size < inode->fragment_size);
1106 DBG_BUGON(inode->fragmentoff >> 63);
1107 *(__le64 *)compressmeta =
1108 cpu_to_le64(inode->fragmentoff | 1ULL << 63);
1109 inode->datalayout = EROFS_INODE_COMPRESSED_FULL;
1110 legacymetasize = Z_EROFS_LEGACY_MAP_HEADER_SIZE;
1111 }
1112
1113 if (compressed_blocks) {
1114 ret = erofs_bh_balloon(bh, erofs_pos(sbi, compressed_blocks));
1115 DBG_BUGON(ret != erofs_blksiz(sbi));
1116 } else {
1117 if (!cfg.c_fragments && !cfg.c_dedupe)
1118 DBG_BUGON(!inode->idata_size);
1119 }
1120
1121 erofs_info("compressed %s (%llu bytes) into %u blocks",
1122 inode->i_srcpath, (unsigned long long)inode->i_size,
1123 compressed_blocks);
1124
1125 if (inode->idata_size) {
1126 bh->op = &erofs_skip_write_bhops;
1127 inode->bh_data = bh;
1128 } else {
1129 erofs_bdrop(bh, false);
1130 }
1131
1132 inode->u.i_blocks = compressed_blocks;
1133
1134 if (inode->datalayout == EROFS_INODE_COMPRESSED_FULL) {
1135 inode->extent_isize = legacymetasize;
1136 } else {
1137 ret = z_erofs_convert_to_compacted_format(inode, blkaddr,
1138 legacymetasize,
1139 compressmeta);
1140 DBG_BUGON(ret);
1141 }
1142 inode->compressmeta = compressmeta;
1143 if (!erofs_is_packed_inode(inode))
1144 erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks);
1145 return 0;
1146
1147 err_free_meta:
1148 free(compressmeta);
1149 inode->compressmeta = NULL;
1150 err_free_idata:
1151 erofs_bdrop(bh, true); /* revoke buffer */
1152 if (inode->idata) {
1153 free(inode->idata);
1154 inode->idata = NULL;
1155 }
1156 return ret;
1157 }
1158
1159 #ifdef EROFS_MT_ENABLED
z_erofs_mt_wq_tls_alloc(struct erofs_workqueue * wq,void * ptr)1160 void *z_erofs_mt_wq_tls_alloc(struct erofs_workqueue *wq, void *ptr)
1161 {
1162 struct erofs_compress_wq_tls *tls;
1163
1164 tls = calloc(1, sizeof(*tls));
1165 if (!tls)
1166 return NULL;
1167
1168 tls->queue = malloc(Z_EROFS_COMPR_QUEUE_SZ);
1169 if (!tls->queue)
1170 goto err_free_priv;
1171
1172 tls->destbuf = calloc(1, EROFS_CONFIG_COMPR_MAX_SZ +
1173 EROFS_MAX_BLOCK_SIZE);
1174 if (!tls->destbuf)
1175 goto err_free_queue;
1176
1177 tls->ccfg = calloc(EROFS_MAX_COMPR_CFGS, sizeof(*tls->ccfg));
1178 if (!tls->ccfg)
1179 goto err_free_destbuf;
1180 return tls;
1181
1182 err_free_destbuf:
1183 free(tls->destbuf);
1184 err_free_queue:
1185 free(tls->queue);
1186 err_free_priv:
1187 free(tls);
1188 return NULL;
1189 }
1190
z_erofs_mt_wq_tls_init_compr(struct erofs_sb_info * sbi,struct erofs_compress_wq_tls * tls,unsigned int alg_id,char * alg_name,unsigned int comp_level,unsigned int dict_size)1191 int z_erofs_mt_wq_tls_init_compr(struct erofs_sb_info *sbi,
1192 struct erofs_compress_wq_tls *tls,
1193 unsigned int alg_id, char *alg_name,
1194 unsigned int comp_level,
1195 unsigned int dict_size)
1196 {
1197 struct erofs_compress_cfg *lc = &tls->ccfg[alg_id];
1198 int ret;
1199
1200 if (__erofs_likely(lc->enable))
1201 return 0;
1202
1203 ret = erofs_compressor_init(sbi, &lc->handle, alg_name,
1204 comp_level, dict_size);
1205 if (ret)
1206 return ret;
1207 lc->algorithmtype = alg_id;
1208 lc->enable = true;
1209 return 0;
1210 }
1211
z_erofs_mt_wq_tls_free(struct erofs_workqueue * wq,void * priv)1212 void *z_erofs_mt_wq_tls_free(struct erofs_workqueue *wq, void *priv)
1213 {
1214 struct erofs_compress_wq_tls *tls = priv;
1215 int i;
1216
1217 for (i = 0; i < EROFS_MAX_COMPR_CFGS; i++)
1218 if (tls->ccfg[i].enable)
1219 erofs_compressor_exit(&tls->ccfg[i].handle);
1220
1221 free(tls->ccfg);
1222 free(tls->destbuf);
1223 free(tls->queue);
1224 free(tls);
1225 return NULL;
1226 }
1227
z_erofs_mt_workfn(struct erofs_work * work,void * tlsp)1228 void z_erofs_mt_workfn(struct erofs_work *work, void *tlsp)
1229 {
1230 struct erofs_compress_work *cwork = (struct erofs_compress_work *)work;
1231 struct erofs_compress_wq_tls *tls = tlsp;
1232 struct z_erofs_compress_sctx *sctx = &cwork->ctx;
1233 struct z_erofs_compress_ictx *ictx = sctx->ictx;
1234 struct erofs_inode *inode = ictx->inode;
1235 struct erofs_sb_info *sbi = inode->sbi;
1236 int ret = 0;
1237
1238 ret = z_erofs_mt_wq_tls_init_compr(sbi, tls, cwork->alg_id,
1239 cwork->alg_name, cwork->comp_level,
1240 cwork->dict_size);
1241 if (ret)
1242 goto out;
1243
1244 sctx->pclustersize = z_erofs_get_max_pclustersize(inode);
1245 sctx->queue = tls->queue;
1246 sctx->destbuf = tls->destbuf;
1247 sctx->chandle = &tls->ccfg[cwork->alg_id].handle;
1248 erofs_compressor_reset(sctx->chandle);
1249 sctx->membuf = malloc(round_up(sctx->remaining, erofs_blksiz(sbi)));
1250 if (!sctx->membuf) {
1251 ret = -ENOMEM;
1252 goto out;
1253 }
1254 sctx->memoff = 0;
1255
1256 ret = z_erofs_compress_segment(sctx, sctx->seg_idx * cfg.c_mkfs_segment_size,
1257 EROFS_NULL_ADDR);
1258
1259 out:
1260 cwork->errcode = ret;
1261 pthread_mutex_lock(&ictx->mutex);
1262 if (++ictx->nfini >= ictx->seg_num) {
1263 DBG_BUGON(ictx->nfini > ictx->seg_num);
1264 pthread_cond_signal(&ictx->cond);
1265 }
1266 pthread_mutex_unlock(&ictx->mutex);
1267 }
1268
z_erofs_merge_segment(struct z_erofs_compress_ictx * ictx,struct z_erofs_compress_sctx * sctx)1269 int z_erofs_merge_segment(struct z_erofs_compress_ictx *ictx,
1270 struct z_erofs_compress_sctx *sctx)
1271 {
1272 struct z_erofs_extent_item *ei, *n;
1273 struct erofs_sb_info *sbi = ictx->inode->sbi;
1274 erofs_blk_t blkoff = 0;
1275 int ret = 0, ret2;
1276
1277 list_for_each_entry_safe(ei, n, &sctx->extents, list) {
1278 list_del(&ei->list);
1279 list_add_tail(&ei->list, &ictx->extents);
1280
1281 if (ei->e.blkaddr != EROFS_NULL_ADDR) /* deduped extents */
1282 continue;
1283
1284 ei->e.blkaddr = sctx->blkaddr;
1285 sctx->blkaddr += ei->e.compressedblks;
1286
1287 /* skip write data but leave blkaddr for inline fallback */
1288 if (ei->e.inlined || !ei->e.compressedblks)
1289 continue;
1290 ret2 = erofs_blk_write(sbi, sctx->membuf + blkoff * erofs_blksiz(sbi),
1291 ei->e.blkaddr, ei->e.compressedblks);
1292 blkoff += ei->e.compressedblks;
1293 if (ret2) {
1294 ret = ret2;
1295 continue;
1296 }
1297 }
1298 free(sctx->membuf);
1299 return ret;
1300 }
1301
z_erofs_mt_compress(struct z_erofs_compress_ictx * ictx)1302 int z_erofs_mt_compress(struct z_erofs_compress_ictx *ictx)
1303 {
1304 struct erofs_compress_work *cur, *head = NULL, **last = &head;
1305 struct erofs_compress_cfg *ccfg = ictx->ccfg;
1306 struct erofs_inode *inode = ictx->inode;
1307 int nsegs = DIV_ROUND_UP(inode->i_size, cfg.c_mkfs_segment_size);
1308 int i;
1309
1310 ictx->seg_num = nsegs;
1311 ictx->nfini = 0;
1312 pthread_mutex_init(&ictx->mutex, NULL);
1313 pthread_cond_init(&ictx->cond, NULL);
1314
1315 for (i = 0; i < nsegs; i++) {
1316 pthread_mutex_lock(&z_erofs_mt_ctrl.mutex);
1317 cur = z_erofs_mt_ctrl.idle;
1318 if (cur) {
1319 z_erofs_mt_ctrl.idle = cur->next;
1320 cur->next = NULL;
1321 }
1322 pthread_mutex_unlock(&z_erofs_mt_ctrl.mutex);
1323 if (!cur) {
1324 cur = calloc(1, sizeof(*cur));
1325 if (!cur)
1326 return -ENOMEM;
1327 }
1328 *last = cur;
1329 last = &cur->next;
1330
1331 cur->ctx = (struct z_erofs_compress_sctx) {
1332 .ictx = ictx,
1333 .seg_idx = i,
1334 .pivot = &dummy_pivot,
1335 };
1336 init_list_head(&cur->ctx.extents);
1337
1338 if (i == nsegs - 1)
1339 cur->ctx.remaining = inode->i_size -
1340 inode->fragment_size -
1341 i * cfg.c_mkfs_segment_size;
1342 else
1343 cur->ctx.remaining = cfg.c_mkfs_segment_size;
1344
1345 cur->alg_id = ccfg->handle.alg->id;
1346 cur->alg_name = ccfg->handle.alg->name;
1347 cur->comp_level = ccfg->handle.compression_level;
1348 cur->dict_size = ccfg->handle.dict_size;
1349
1350 cur->work.fn = z_erofs_mt_workfn;
1351 erofs_queue_work(&z_erofs_mt_ctrl.wq, &cur->work);
1352 }
1353 ictx->mtworks = head;
1354 return 0;
1355 }
1356
erofs_mt_write_compressed_file(struct z_erofs_compress_ictx * ictx)1357 int erofs_mt_write_compressed_file(struct z_erofs_compress_ictx *ictx)
1358 {
1359 struct erofs_sb_info *sbi = ictx->inode->sbi;
1360 struct erofs_buffer_head *bh = NULL;
1361 struct erofs_compress_work *head = ictx->mtworks, *cur;
1362 erofs_blk_t blkaddr, compressed_blocks = 0;
1363 int ret;
1364
1365 pthread_mutex_lock(&ictx->mutex);
1366 while (ictx->nfini < ictx->seg_num)
1367 pthread_cond_wait(&ictx->cond, &ictx->mutex);
1368 pthread_mutex_unlock(&ictx->mutex);
1369
1370 bh = erofs_balloc(sbi->bmgr, DATA, 0, 0, 0);
1371 if (IS_ERR(bh)) {
1372 ret = PTR_ERR(bh);
1373 goto out;
1374 }
1375
1376 DBG_BUGON(!head);
1377 blkaddr = erofs_mapbh(NULL, bh->block);
1378
1379 ret = 0;
1380 do {
1381 cur = head;
1382 head = cur->next;
1383
1384 if (cur->errcode) {
1385 ret = cur->errcode;
1386 } else {
1387 int ret2;
1388
1389 cur->ctx.blkaddr = blkaddr;
1390 ret2 = z_erofs_merge_segment(ictx, &cur->ctx);
1391 if (ret2)
1392 ret = ret2;
1393
1394 compressed_blocks += cur->ctx.blkaddr - blkaddr;
1395 blkaddr = cur->ctx.blkaddr;
1396 }
1397
1398 pthread_mutex_lock(&z_erofs_mt_ctrl.mutex);
1399 cur->next = z_erofs_mt_ctrl.idle;
1400 z_erofs_mt_ctrl.idle = cur;
1401 pthread_mutex_unlock(&z_erofs_mt_ctrl.mutex);
1402 } while (head);
1403
1404 if (ret)
1405 goto out;
1406 ret = erofs_commit_compressed_file(ictx, bh,
1407 blkaddr - compressed_blocks, compressed_blocks);
1408
1409 out:
1410 close(ictx->fd);
1411 free(ictx);
1412 return ret;
1413 }
1414 #endif
1415
1416 static struct z_erofs_compress_ictx g_ictx;
1417
erofs_begin_compressed_file(struct erofs_inode * inode,int fd,u64 fpos)1418 void *erofs_begin_compressed_file(struct erofs_inode *inode, int fd, u64 fpos)
1419 {
1420 struct erofs_sb_info *sbi = inode->sbi;
1421 struct z_erofs_compress_ictx *ictx;
1422 int ret;
1423
1424 /* initialize per-file compression setting */
1425 inode->z_advise = 0;
1426 inode->z_logical_clusterbits = sbi->blkszbits;
1427 if (!cfg.c_legacy_compress && inode->z_logical_clusterbits <= 14) {
1428 if (inode->z_logical_clusterbits <= 12)
1429 inode->z_advise |= Z_EROFS_ADVISE_COMPACTED_2B;
1430 inode->datalayout = EROFS_INODE_COMPRESSED_COMPACT;
1431 } else {
1432 inode->datalayout = EROFS_INODE_COMPRESSED_FULL;
1433 }
1434
1435 if (erofs_sb_has_big_pcluster(sbi)) {
1436 inode->z_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_1;
1437 if (inode->datalayout == EROFS_INODE_COMPRESSED_COMPACT)
1438 inode->z_advise |= Z_EROFS_ADVISE_BIG_PCLUSTER_2;
1439 }
1440 if (cfg.c_fragments && !cfg.c_dedupe)
1441 inode->z_advise |= Z_EROFS_ADVISE_INTERLACED_PCLUSTER;
1442
1443 #ifndef NDEBUG
1444 if (cfg.c_random_algorithms) {
1445 while (1) {
1446 inode->z_algorithmtype[0] =
1447 rand() % EROFS_MAX_COMPR_CFGS;
1448 if (erofs_ccfg[inode->z_algorithmtype[0]].enable)
1449 break;
1450 }
1451 }
1452 #endif
1453 inode->idata_size = 0;
1454 inode->fragment_size = 0;
1455
1456 if (z_erofs_mt_enabled) {
1457 ictx = malloc(sizeof(*ictx));
1458 if (!ictx)
1459 return ERR_PTR(-ENOMEM);
1460 ictx->fd = dup(fd);
1461 } else {
1462 #ifdef EROFS_MT_ENABLED
1463 pthread_mutex_lock(&g_ictx.mutex);
1464 if (g_ictx.seg_num)
1465 pthread_cond_wait(&g_ictx.cond, &g_ictx.mutex);
1466 g_ictx.seg_num = 1;
1467 pthread_mutex_unlock(&g_ictx.mutex);
1468 #endif
1469 ictx = &g_ictx;
1470 ictx->fd = fd;
1471 }
1472
1473 ictx->ccfg = &erofs_ccfg[inode->z_algorithmtype[0]];
1474 inode->z_algorithmtype[0] = ictx->ccfg->algorithmtype;
1475 inode->z_algorithmtype[1] = 0;
1476
1477 /*
1478 * Handle tails in advance to avoid writing duplicated
1479 * parts into the packed inode.
1480 */
1481 if (cfg.c_fragments && !erofs_is_packed_inode(inode)) {
1482 ret = z_erofs_fragments_dedupe(inode, fd, &ictx->tof_chksum);
1483 if (ret < 0)
1484 goto err_free_ictx;
1485 }
1486
1487 ictx->inode = inode;
1488 ictx->fpos = fpos;
1489 init_list_head(&ictx->extents);
1490 ictx->fix_dedupedfrag = false;
1491 ictx->fragemitted = false;
1492
1493 if (cfg.c_all_fragments && !erofs_is_packed_inode(inode) &&
1494 !inode->fragment_size) {
1495 ret = z_erofs_pack_file_from_fd(inode, fd, ictx->tof_chksum);
1496 if (ret)
1497 goto err_free_idata;
1498 }
1499 #ifdef EROFS_MT_ENABLED
1500 if (ictx != &g_ictx) {
1501 ret = z_erofs_mt_compress(ictx);
1502 if (ret)
1503 goto err_free_idata;
1504 }
1505 #endif
1506 return ictx;
1507
1508 err_free_idata:
1509 if (inode->idata) {
1510 free(inode->idata);
1511 inode->idata = NULL;
1512 }
1513 err_free_ictx:
1514 if (ictx != &g_ictx)
1515 free(ictx);
1516 return ERR_PTR(ret);
1517 }
1518
erofs_write_compressed_file(struct z_erofs_compress_ictx * ictx)1519 int erofs_write_compressed_file(struct z_erofs_compress_ictx *ictx)
1520 {
1521 static u8 g_queue[Z_EROFS_COMPR_QUEUE_SZ];
1522 struct erofs_buffer_head *bh;
1523 static struct z_erofs_compress_sctx sctx;
1524 struct erofs_compress_cfg *ccfg = ictx->ccfg;
1525 struct erofs_inode *inode = ictx->inode;
1526 erofs_blk_t blkaddr;
1527 int ret;
1528
1529 #ifdef EROFS_MT_ENABLED
1530 if (ictx != &g_ictx)
1531 return erofs_mt_write_compressed_file(ictx);
1532 #endif
1533
1534 /* allocate main data buffer */
1535 bh = erofs_balloc(inode->sbi->bmgr, DATA, 0, 0, 0);
1536 if (IS_ERR(bh)) {
1537 ret = PTR_ERR(bh);
1538 goto err_free_idata;
1539 }
1540 blkaddr = erofs_mapbh(NULL, bh->block); /* start_blkaddr */
1541
1542 ictx->seg_num = 1;
1543 sctx = (struct z_erofs_compress_sctx) {
1544 .ictx = ictx,
1545 .queue = g_queue,
1546 .chandle = &ccfg->handle,
1547 .remaining = inode->i_size - inode->fragment_size,
1548 .seg_idx = 0,
1549 .pivot = &dummy_pivot,
1550 .pclustersize = z_erofs_get_max_pclustersize(inode),
1551 };
1552 init_list_head(&sctx.extents);
1553
1554 ret = z_erofs_compress_segment(&sctx, -1, blkaddr);
1555 if (ret)
1556 goto err_free_idata;
1557
1558 list_splice_tail(&sctx.extents, &ictx->extents);
1559 ret = erofs_commit_compressed_file(ictx, bh, blkaddr,
1560 sctx.blkaddr - blkaddr);
1561 goto out;
1562
1563 err_free_idata:
1564 erofs_bdrop(bh, true); /* revoke buffer */
1565 if (inode->idata) {
1566 free(inode->idata);
1567 inode->idata = NULL;
1568 }
1569 out:
1570 #ifdef EROFS_MT_ENABLED
1571 pthread_mutex_lock(&ictx->mutex);
1572 ictx->seg_num = 0;
1573 pthread_cond_signal(&ictx->cond);
1574 pthread_mutex_unlock(&ictx->mutex);
1575 #endif
1576 return ret;
1577 }
1578
z_erofs_build_compr_cfgs(struct erofs_sb_info * sbi,struct erofs_buffer_head * sb_bh,u32 * max_dict_size)1579 static int z_erofs_build_compr_cfgs(struct erofs_sb_info *sbi,
1580 struct erofs_buffer_head *sb_bh,
1581 u32 *max_dict_size)
1582 {
1583 struct erofs_buffer_head *bh = sb_bh;
1584 int ret = 0;
1585
1586 if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZ4)) {
1587 struct {
1588 __le16 size;
1589 struct z_erofs_lz4_cfgs lz4;
1590 } __packed lz4alg = {
1591 .size = cpu_to_le16(sizeof(struct z_erofs_lz4_cfgs)),
1592 .lz4 = {
1593 .max_distance =
1594 cpu_to_le16(sbi->lz4.max_distance),
1595 .max_pclusterblks =
1596 cfg.c_mkfs_pclustersize_max >> sbi->blkszbits,
1597 }
1598 };
1599
1600 bh = erofs_battach(bh, META, sizeof(lz4alg));
1601 if (IS_ERR(bh)) {
1602 DBG_BUGON(1);
1603 return PTR_ERR(bh);
1604 }
1605 erofs_mapbh(NULL, bh->block);
1606 ret = erofs_dev_write(sbi, &lz4alg, erofs_btell(bh, false),
1607 sizeof(lz4alg));
1608 bh->op = &erofs_drop_directly_bhops;
1609 }
1610 #ifdef HAVE_LIBLZMA
1611 if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZMA)) {
1612 struct {
1613 __le16 size;
1614 struct z_erofs_lzma_cfgs lzma;
1615 } __packed lzmaalg = {
1616 .size = cpu_to_le16(sizeof(struct z_erofs_lzma_cfgs)),
1617 .lzma = {
1618 .dict_size = cpu_to_le32(
1619 max_dict_size
1620 [Z_EROFS_COMPRESSION_LZMA]),
1621 }
1622 };
1623
1624 bh = erofs_battach(bh, META, sizeof(lzmaalg));
1625 if (IS_ERR(bh)) {
1626 DBG_BUGON(1);
1627 return PTR_ERR(bh);
1628 }
1629 erofs_mapbh(NULL, bh->block);
1630 ret = erofs_dev_write(sbi, &lzmaalg, erofs_btell(bh, false),
1631 sizeof(lzmaalg));
1632 bh->op = &erofs_drop_directly_bhops;
1633 }
1634 #endif
1635 if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_DEFLATE)) {
1636 struct {
1637 __le16 size;
1638 struct z_erofs_deflate_cfgs z;
1639 } __packed zalg = {
1640 .size = cpu_to_le16(sizeof(struct z_erofs_deflate_cfgs)),
1641 .z = {
1642 .windowbits = cpu_to_le32(ilog2(
1643 max_dict_size
1644 [Z_EROFS_COMPRESSION_DEFLATE])),
1645 }
1646 };
1647
1648 bh = erofs_battach(bh, META, sizeof(zalg));
1649 if (IS_ERR(bh)) {
1650 DBG_BUGON(1);
1651 return PTR_ERR(bh);
1652 }
1653 erofs_mapbh(NULL, bh->block);
1654 ret = erofs_dev_write(sbi, &zalg, erofs_btell(bh, false),
1655 sizeof(zalg));
1656 bh->op = &erofs_drop_directly_bhops;
1657 }
1658 #ifdef HAVE_LIBZSTD
1659 if (sbi->available_compr_algs & (1 << Z_EROFS_COMPRESSION_ZSTD)) {
1660 struct {
1661 __le16 size;
1662 struct z_erofs_zstd_cfgs z;
1663 } __packed zalg = {
1664 .size = cpu_to_le16(sizeof(struct z_erofs_zstd_cfgs)),
1665 .z = {
1666 .windowlog =
1667 ilog2(max_dict_size[Z_EROFS_COMPRESSION_ZSTD]) - 10,
1668 }
1669 };
1670
1671 bh = erofs_battach(bh, META, sizeof(zalg));
1672 if (IS_ERR(bh)) {
1673 DBG_BUGON(1);
1674 return PTR_ERR(bh);
1675 }
1676 erofs_mapbh(NULL, bh->block);
1677 ret = erofs_dev_write(sbi, &zalg, erofs_btell(bh, false),
1678 sizeof(zalg));
1679 bh->op = &erofs_drop_directly_bhops;
1680 }
1681 #endif
1682 return ret;
1683 }
1684
z_erofs_compress_init(struct erofs_sb_info * sbi,struct erofs_buffer_head * sb_bh)1685 int z_erofs_compress_init(struct erofs_sb_info *sbi, struct erofs_buffer_head *sb_bh)
1686 {
1687 int i, ret, id;
1688 u32 max_dict_size[Z_EROFS_COMPRESSION_MAX] = {};
1689 u32 available_compr_algs = 0;
1690
1691 for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
1692 struct erofs_compress *c = &erofs_ccfg[i].handle;
1693
1694 ret = erofs_compressor_init(sbi, c, cfg.c_compr_opts[i].alg,
1695 cfg.c_compr_opts[i].level,
1696 cfg.c_compr_opts[i].dict_size);
1697 if (ret)
1698 return ret;
1699
1700 id = z_erofs_get_compress_algorithm_id(c);
1701 erofs_ccfg[i].algorithmtype = id;
1702 erofs_ccfg[i].enable = true;
1703 available_compr_algs |= 1 << erofs_ccfg[i].algorithmtype;
1704 if (erofs_ccfg[i].algorithmtype != Z_EROFS_COMPRESSION_LZ4)
1705 erofs_sb_set_compr_cfgs(sbi);
1706 if (c->dict_size > max_dict_size[id])
1707 max_dict_size[id] = c->dict_size;
1708 }
1709
1710 /*
1711 * if primary algorithm is empty (e.g. compression off),
1712 * clear 0PADDING feature for old kernel compatibility.
1713 */
1714 if (!available_compr_algs ||
1715 (cfg.c_legacy_compress && available_compr_algs == 1))
1716 erofs_sb_clear_lz4_0padding(sbi);
1717
1718 if (!available_compr_algs)
1719 return 0;
1720
1721 if (!sb_bh) {
1722 u32 dalg = available_compr_algs & (~sbi->available_compr_algs);
1723
1724 if (dalg) {
1725 erofs_err("unavailable algorithms 0x%x on incremental builds",
1726 dalg);
1727 return -EOPNOTSUPP;
1728 }
1729 if (available_compr_algs & (1 << Z_EROFS_COMPRESSION_LZ4) &&
1730 sbi->lz4.max_pclusterblks << sbi->blkszbits <
1731 cfg.c_mkfs_pclustersize_max) {
1732 erofs_err("pclustersize %u is too large on incremental builds",
1733 cfg.c_mkfs_pclustersize_max);
1734 return -EOPNOTSUPP;
1735 }
1736 } else {
1737 sbi->available_compr_algs = available_compr_algs;
1738 }
1739
1740 /*
1741 * if big pcluster is enabled, an extra CBLKCNT lcluster index needs
1742 * to be loaded in order to get those compressed block counts.
1743 */
1744 if (cfg.c_mkfs_pclustersize_max > erofs_blksiz(sbi)) {
1745 if (cfg.c_mkfs_pclustersize_max > Z_EROFS_PCLUSTER_MAX_SIZE) {
1746 erofs_err("unsupported pclustersize %u (too large)",
1747 cfg.c_mkfs_pclustersize_max);
1748 return -EINVAL;
1749 }
1750 erofs_sb_set_big_pcluster(sbi);
1751 }
1752 if (cfg.c_mkfs_pclustersize_packed > cfg.c_mkfs_pclustersize_max) {
1753 erofs_err("invalid pclustersize for the packed file %u",
1754 cfg.c_mkfs_pclustersize_packed);
1755 return -EINVAL;
1756 }
1757
1758 if (sb_bh && erofs_sb_has_compr_cfgs(sbi)) {
1759 ret = z_erofs_build_compr_cfgs(sbi, sb_bh, max_dict_size);
1760 if (ret)
1761 return ret;
1762 }
1763
1764 z_erofs_mt_enabled = false;
1765 #ifdef EROFS_MT_ENABLED
1766 if (cfg.c_mt_workers >= 1 && (cfg.c_dedupe ||
1767 (cfg.c_fragments && !cfg.c_all_fragments))) {
1768 if (cfg.c_dedupe)
1769 erofs_warn("multi-threaded dedupe is NOT implemented for now");
1770 if (cfg.c_fragments)
1771 erofs_warn("multi-threaded fragments is NOT implemented for now");
1772 cfg.c_mt_workers = 0;
1773 }
1774
1775 if (cfg.c_mt_workers >= 1) {
1776 ret = erofs_alloc_workqueue(&z_erofs_mt_ctrl.wq,
1777 cfg.c_mt_workers,
1778 cfg.c_mt_workers << 2,
1779 z_erofs_mt_wq_tls_alloc,
1780 z_erofs_mt_wq_tls_free);
1781 z_erofs_mt_enabled = !ret;
1782 }
1783 pthread_mutex_init(&g_ictx.mutex, NULL);
1784 pthread_cond_init(&g_ictx.cond, NULL);
1785 #endif
1786 return 0;
1787 }
1788
z_erofs_compress_exit(void)1789 int z_erofs_compress_exit(void)
1790 {
1791 int i, ret;
1792
1793 for (i = 0; cfg.c_compr_opts[i].alg; ++i) {
1794 ret = erofs_compressor_exit(&erofs_ccfg[i].handle);
1795 if (ret)
1796 return ret;
1797 }
1798
1799 if (z_erofs_mt_enabled) {
1800 #ifdef EROFS_MT_ENABLED
1801 ret = erofs_destroy_workqueue(&z_erofs_mt_ctrl.wq);
1802 if (ret)
1803 return ret;
1804 while (z_erofs_mt_ctrl.idle) {
1805 struct erofs_compress_work *tmp =
1806 z_erofs_mt_ctrl.idle->next;
1807 free(z_erofs_mt_ctrl.idle);
1808 z_erofs_mt_ctrl.idle = tmp;
1809 }
1810 #endif
1811 }
1812 return 0;
1813 }
1814