1 /*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11 #include "config.h"
12
13 /* Define a config for each fast level we want to test with. */
14 #define FAST_LEVEL(x) \
15 param_value_t const level_fast##x##_param_values[] = { \
16 {.param = ZSTD_c_compressionLevel, .value = -x}, \
17 }; \
18 config_t const level_fast##x = { \
19 .name = "level -" #x, \
20 .cli_args = "--fast=" #x, \
21 .param_values = PARAM_VALUES(level_fast##x##_param_values), \
22 }; \
23 config_t const level_fast##x##_dict = { \
24 .name = "level -" #x " with dict", \
25 .cli_args = "--fast=" #x, \
26 .param_values = PARAM_VALUES(level_fast##x##_param_values), \
27 .use_dictionary = 1, \
28 };
29
30 /* Define a config for each level we want to test with. */
31 #define LEVEL(x) \
32 param_value_t const level_##x##_param_values[] = { \
33 {.param = ZSTD_c_compressionLevel, .value = x}, \
34 }; \
35 param_value_t const level_##x##_param_values_dms[] = { \
36 {.param = ZSTD_c_compressionLevel, .value = x}, \
37 {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
38 {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
39 }; \
40 param_value_t const level_##x##_param_values_dds[] = { \
41 {.param = ZSTD_c_compressionLevel, .value = x}, \
42 {.param = ZSTD_c_enableDedicatedDictSearch, .value = 1}, \
43 {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
44 }; \
45 param_value_t const level_##x##_param_values_dictcopy[] = { \
46 {.param = ZSTD_c_compressionLevel, .value = x}, \
47 {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
48 {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceCopy}, \
49 }; \
50 param_value_t const level_##x##_param_values_dictload[] = { \
51 {.param = ZSTD_c_compressionLevel, .value = x}, \
52 {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
53 {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceLoad}, \
54 }; \
55 config_t const level_##x = { \
56 .name = "level " #x, \
57 .cli_args = "-" #x, \
58 .param_values = PARAM_VALUES(level_##x##_param_values), \
59 }; \
60 config_t const level_##x##_dict = { \
61 .name = "level " #x " with dict", \
62 .cli_args = "-" #x, \
63 .param_values = PARAM_VALUES(level_##x##_param_values), \
64 .use_dictionary = 1, \
65 }; \
66 config_t const level_##x##_dict_dms = { \
67 .name = "level " #x " with dict dms", \
68 .cli_args = "-" #x, \
69 .param_values = PARAM_VALUES(level_##x##_param_values_dms), \
70 .use_dictionary = 1, \
71 .advanced_api_only = 1, \
72 }; \
73 config_t const level_##x##_dict_dds = { \
74 .name = "level " #x " with dict dds", \
75 .cli_args = "-" #x, \
76 .param_values = PARAM_VALUES(level_##x##_param_values_dds), \
77 .use_dictionary = 1, \
78 .advanced_api_only = 1, \
79 }; \
80 config_t const level_##x##_dict_copy = { \
81 .name = "level " #x " with dict copy", \
82 .cli_args = "-" #x, \
83 .param_values = PARAM_VALUES(level_##x##_param_values_dictcopy), \
84 .use_dictionary = 1, \
85 .advanced_api_only = 1, \
86 }; \
87 config_t const level_##x##_dict_load = { \
88 .name = "level " #x " with dict load", \
89 .cli_args = "-" #x, \
90 .param_values = PARAM_VALUES(level_##x##_param_values_dictload), \
91 .use_dictionary = 1, \
92 .advanced_api_only = 1, \
93 };
94
95 /* Define a config specifically to test row hash based levels and settings.
96 */
97 #define ROW_LEVEL(x, y) \
98 param_value_t const row_##y##_level_##x##_param_values[] = { \
99 {.param = ZSTD_c_useRowMatchFinder, .value = y}, \
100 {.param = ZSTD_c_compressionLevel, .value = x}, \
101 }; \
102 param_value_t const row_##y##_level_##x##_param_values_dms[] = { \
103 {.param = ZSTD_c_useRowMatchFinder, .value = y}, \
104 {.param = ZSTD_c_compressionLevel, .value = x}, \
105 {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
106 {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
107 }; \
108 param_value_t const row_##y##_level_##x##_param_values_dds[] = { \
109 {.param = ZSTD_c_useRowMatchFinder, .value = y}, \
110 {.param = ZSTD_c_compressionLevel, .value = x}, \
111 {.param = ZSTD_c_enableDedicatedDictSearch, .value = 1}, \
112 {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
113 }; \
114 param_value_t const row_##y##_level_##x##_param_values_dictcopy[] = { \
115 {.param = ZSTD_c_useRowMatchFinder, .value = y}, \
116 {.param = ZSTD_c_compressionLevel, .value = x}, \
117 {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
118 {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceCopy}, \
119 }; \
120 param_value_t const row_##y##_level_##x##_param_values_dictload[] = { \
121 {.param = ZSTD_c_useRowMatchFinder, .value = y}, \
122 {.param = ZSTD_c_compressionLevel, .value = x}, \
123 {.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
124 {.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceLoad}, \
125 }; \
126 config_t const row_##y##_level_##x = { \
127 .name = "level " #x " row " #y, \
128 .cli_args = "-" #x, \
129 .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values), \
130 .advanced_api_only = 1, \
131 }; \
132 config_t const row_##y##_level_##x##_dict_dms = { \
133 .name = "level " #x " row " #y " with dict dms", \
134 .cli_args = "-" #x, \
135 .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dms), \
136 .use_dictionary = 1, \
137 .advanced_api_only = 1, \
138 }; \
139 config_t const row_##y##_level_##x##_dict_dds = { \
140 .name = "level " #x " row " #y " with dict dds", \
141 .cli_args = "-" #x, \
142 .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dds), \
143 .use_dictionary = 1, \
144 .advanced_api_only = 1, \
145 }; \
146 config_t const row_##y##_level_##x##_dict_copy = { \
147 .name = "level " #x " row " #y" with dict copy", \
148 .cli_args = "-" #x, \
149 .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dictcopy), \
150 .use_dictionary = 1, \
151 .advanced_api_only = 1, \
152 }; \
153 config_t const row_##y##_level_##x##_dict_load = { \
154 .name = "level " #x " row " #y " with dict load", \
155 .cli_args = "-" #x, \
156 .param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dictload), \
157 .use_dictionary = 1, \
158 .advanced_api_only = 1, \
159 };
160
161 #define PARAM_VALUES(pv) \
162 { .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) }
163
164 #include "levels.h"
165
166 #undef LEVEL
167 #undef FAST_LEVEL
168 #undef ROW_LEVEL
169
170 static config_t no_pledged_src_size = {
171 .name = "no source size",
172 .cli_args = "",
173 .param_values = PARAM_VALUES(level_0_param_values),
174 .no_pledged_src_size = 1,
175 };
176
177 static config_t no_pledged_src_size_with_dict = {
178 .name = "no source size with dict",
179 .cli_args = "",
180 .param_values = PARAM_VALUES(level_0_param_values),
181 .no_pledged_src_size = 1,
182 .use_dictionary = 1,
183 };
184
185 static param_value_t const ldm_param_values[] = {
186 {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
187 };
188
189 static config_t ldm = {
190 .name = "long distance mode",
191 .cli_args = "--long",
192 .param_values = PARAM_VALUES(ldm_param_values),
193 };
194
195 static param_value_t const mt_param_values[] = {
196 {.param = ZSTD_c_nbWorkers, .value = 2},
197 };
198
199 static config_t mt = {
200 .name = "multithreaded",
201 .cli_args = "-T2",
202 .param_values = PARAM_VALUES(mt_param_values),
203 };
204
205 static param_value_t const mt_ldm_param_values[] = {
206 {.param = ZSTD_c_nbWorkers, .value = 2},
207 {.param = ZSTD_c_enableLongDistanceMatching, .value = ZSTD_ps_enable},
208 };
209
210 static config_t mt_ldm = {
211 .name = "multithreaded long distance mode",
212 .cli_args = "-T2 --long",
213 .param_values = PARAM_VALUES(mt_ldm_param_values),
214 };
215
216 static param_value_t mt_advanced_param_values[] = {
217 {.param = ZSTD_c_nbWorkers, .value = 2},
218 {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable},
219 };
220
221 static config_t mt_advanced = {
222 .name = "multithreaded with advanced params",
223 .cli_args = "-T2 --no-compress-literals",
224 .param_values = PARAM_VALUES(mt_advanced_param_values),
225 };
226
227 static param_value_t const small_wlog_param_values[] = {
228 {.param = ZSTD_c_windowLog, .value = 10},
229 };
230
231 static config_t small_wlog = {
232 .name = "small window log",
233 .cli_args = "--zstd=wlog=10",
234 .param_values = PARAM_VALUES(small_wlog_param_values),
235 };
236
237 static param_value_t const small_hlog_param_values[] = {
238 {.param = ZSTD_c_hashLog, .value = 6},
239 {.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt},
240 };
241
242 static config_t small_hlog = {
243 .name = "small hash log",
244 .cli_args = "--zstd=hlog=6,strat=7",
245 .param_values = PARAM_VALUES(small_hlog_param_values),
246 };
247
248 static param_value_t const small_clog_param_values[] = {
249 {.param = ZSTD_c_chainLog, .value = 6},
250 {.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt},
251 };
252
253 static config_t small_clog = {
254 .name = "small chain log",
255 .cli_args = "--zstd=clog=6,strat=7",
256 .param_values = PARAM_VALUES(small_clog_param_values),
257 };
258
259 static param_value_t const uncompressed_literals_param_values[] = {
260 {.param = ZSTD_c_compressionLevel, .value = 3},
261 {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable},
262 };
263
264 static config_t uncompressed_literals = {
265 .name = "uncompressed literals",
266 .cli_args = "-3 --no-compress-literals",
267 .param_values = PARAM_VALUES(uncompressed_literals_param_values),
268 };
269
270 static param_value_t const uncompressed_literals_opt_param_values[] = {
271 {.param = ZSTD_c_compressionLevel, .value = 19},
272 {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_disable},
273 };
274
275 static config_t uncompressed_literals_opt = {
276 .name = "uncompressed literals optimal",
277 .cli_args = "-19 --no-compress-literals",
278 .param_values = PARAM_VALUES(uncompressed_literals_opt_param_values),
279 };
280
281 static param_value_t const huffman_literals_param_values[] = {
282 {.param = ZSTD_c_compressionLevel, .value = -1},
283 {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_ps_enable},
284 };
285
286 static config_t huffman_literals = {
287 .name = "huffman literals",
288 .cli_args = "--fast=1 --compress-literals",
289 .param_values = PARAM_VALUES(huffman_literals_param_values),
290 };
291
292 static param_value_t const explicit_params_param_values[] = {
293 {.param = ZSTD_c_checksumFlag, .value = 1},
294 {.param = ZSTD_c_contentSizeFlag, .value = 0},
295 {.param = ZSTD_c_dictIDFlag, .value = 0},
296 {.param = ZSTD_c_strategy, .value = (int)ZSTD_greedy},
297 {.param = ZSTD_c_windowLog, .value = 18},
298 {.param = ZSTD_c_hashLog, .value = 21},
299 {.param = ZSTD_c_chainLog, .value = 21},
300 {.param = ZSTD_c_targetLength, .value = 100},
301 };
302
303 static config_t explicit_params = {
304 .name = "explicit params",
305 .cli_args = "--no-check --no-dictID --zstd=strategy=3,wlog=18,hlog=21,clog=21,tlen=100",
306 .param_values = PARAM_VALUES(explicit_params_param_values),
307 };
308
309 static config_t const* g_configs[] = {
310
311 #define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict,
312 #define LEVEL(x) &level_##x, &level_##x##_dict, &level_##x##_dict_dms, &level_##x##_dict_dds, &level_##x##_dict_copy, &level_##x##_dict_load,
313 #define ROW_LEVEL(x, y) &row_##y##_level_##x, &row_##y##_level_##x##_dict_dms, &row_##y##_level_##x##_dict_dds, &row_##y##_level_##x##_dict_copy, &row_##y##_level_##x##_dict_load,
314 #include "levels.h"
315 #undef ROW_LEVEL
316 #undef LEVEL
317 #undef FAST_LEVEL
318
319 &no_pledged_src_size,
320 &no_pledged_src_size_with_dict,
321 &ldm,
322 &mt,
323 &mt_ldm,
324 &small_wlog,
325 &small_hlog,
326 &small_clog,
327 &explicit_params,
328 &uncompressed_literals,
329 &uncompressed_literals_opt,
330 &huffman_literals,
331 &mt_advanced,
332 NULL,
333 };
334
335 config_t const* const* configs = g_configs;
336
config_skip_data(config_t const * config,data_t const * data)337 int config_skip_data(config_t const* config, data_t const* data) {
338 return config->use_dictionary && !data_has_dict(data);
339 }
340
config_get_level(config_t const * config)341 int config_get_level(config_t const* config)
342 {
343 param_values_t const params = config->param_values;
344 size_t i;
345 for (i = 0; i < params.size; ++i) {
346 if (params.data[i].param == ZSTD_c_compressionLevel)
347 return (int)params.data[i].value;
348 }
349 return CONFIG_NO_LEVEL;
350 }
351
config_get_zstd_params(config_t const * config,uint64_t srcSize,size_t dictSize)352 ZSTD_parameters config_get_zstd_params(
353 config_t const* config,
354 uint64_t srcSize,
355 size_t dictSize)
356 {
357 ZSTD_parameters zparams = {};
358 param_values_t const params = config->param_values;
359 int level = config_get_level(config);
360 if (level == CONFIG_NO_LEVEL)
361 level = 3;
362 zparams = ZSTD_getParams(
363 level,
364 config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : srcSize,
365 dictSize);
366 for (size_t i = 0; i < params.size; ++i) {
367 unsigned const value = params.data[i].value;
368 switch (params.data[i].param) {
369 case ZSTD_c_contentSizeFlag:
370 zparams.fParams.contentSizeFlag = value;
371 break;
372 case ZSTD_c_checksumFlag:
373 zparams.fParams.checksumFlag = value;
374 break;
375 case ZSTD_c_dictIDFlag:
376 zparams.fParams.noDictIDFlag = !value;
377 break;
378 case ZSTD_c_windowLog:
379 zparams.cParams.windowLog = value;
380 break;
381 case ZSTD_c_chainLog:
382 zparams.cParams.chainLog = value;
383 break;
384 case ZSTD_c_hashLog:
385 zparams.cParams.hashLog = value;
386 break;
387 case ZSTD_c_searchLog:
388 zparams.cParams.searchLog = value;
389 break;
390 case ZSTD_c_minMatch:
391 zparams.cParams.minMatch = value;
392 break;
393 case ZSTD_c_targetLength:
394 zparams.cParams.targetLength = value;
395 break;
396 case ZSTD_c_strategy:
397 zparams.cParams.strategy = (ZSTD_strategy)value;
398 break;
399 default:
400 break;
401 }
402 }
403 return zparams;
404 }
405