1 /*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12 /*-************************************
13 * Compiler specific
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
19 #endif
20
21
22 /*-************************************
23 * Includes
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <time.h> /* time(), time_t */
29 #undef NDEBUG /* always enable assert() */
30 #include <assert.h>
31 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
32 #include "debug.h" /* DEBUG_STATIC_ASSERT */
33 #include "fse.h"
34 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
35 #include "zstd.h" /* ZSTD_VERSION_STRING */
36 #include "zstd_errors.h" /* ZSTD_getErrorCode */
37 #define ZDICT_STATIC_LINKING_ONLY
38 #include "zdict.h" /* ZDICT_trainFromBuffer */
39 #include "mem.h"
40 #include "datagen.h" /* RDG_genBuffer */
41 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
42 #include "xxhash.h" /* XXH64 */
43 #include "util.h"
44 #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
45 /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
46 #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
47 #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
48
49
50 /*-************************************
51 * Constants
52 **************************************/
53 #define GB *(1U<<30)
54
55 static const int FUZ_compressibility_default = 50;
56 static const int nbTestsDefault = 30000;
57
58
59 /*-************************************
60 * Display Macros
61 **************************************/
62 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
64 static U32 g_displayLevel = 2;
65
66 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
67 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
68
69 #define DISPLAYUPDATE(l, ...) \
70 if (g_displayLevel>=l) { \
71 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
72 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
73 if (g_displayLevel>=4) fflush(stderr); } \
74 }
75
76
77 /*-*******************************************************
78 * Compile time test
79 *********************************************************/
80 #undef MIN
81 #undef MAX
82 /* Declaring the function, to avoid -Wmissing-prototype */
83 void FUZ_bug976(void);
FUZ_bug976(void)84 void FUZ_bug976(void)
85 { /* these constants shall not depend on MIN() macro */
86 DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
87 DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
88 }
89
90
91 /*-*******************************************************
92 * Internal functions
93 *********************************************************/
94 #define MIN(a,b) ((a)<(b)?(a):(b))
95 #define MAX(a,b) ((a)>(b)?(a):(b))
96
97 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
FUZ_rand(U32 * src)98 static U32 FUZ_rand(U32* src)
99 {
100 static const U32 prime1 = 2654435761U;
101 static const U32 prime2 = 2246822519U;
102 U32 rand32 = *src;
103 rand32 *= prime1;
104 rand32 += prime2;
105 rand32 = FUZ_rotl32(rand32, 13);
106 *src = rand32;
107 return rand32 >> 5;
108 }
109
FUZ_highbit32(U32 v32)110 static U32 FUZ_highbit32(U32 v32)
111 {
112 unsigned nbBits = 0;
113 if (v32==0) return 0;
114 while (v32) v32 >>= 1, nbBits++;
115 return nbBits;
116 }
117
118
119 /*=============================================
120 * Test macros
121 =============================================*/
122 #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
123
124 #define CHECK_Z(f) { \
125 size_t const err = f; \
126 if (ZSTD_isError(err)) { \
127 DISPLAY("Error => %s : %s ", \
128 #f, ZSTD_getErrorName(err)); \
129 exit(1); \
130 } }
131
132 #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
133 #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
134 #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
135
136 #define CHECK_OP(op, lhs, rhs) { \
137 if (!((lhs) op (rhs))) { \
138 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
139 exit(1); \
140 } \
141 }
142 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
143 #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
144
145
146 /*=============================================
147 * Memory Tests
148 =============================================*/
149 #if defined(__APPLE__) && defined(__MACH__)
150
151 #include <malloc/malloc.h> /* malloc_size */
152
153 typedef struct {
154 unsigned long long totalMalloc;
155 size_t currentMalloc;
156 size_t peakMalloc;
157 unsigned nbMalloc;
158 unsigned nbFree;
159 } mallocCounter_t;
160
161 static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
162
FUZ_mallocDebug(void * counter,size_t size)163 static void* FUZ_mallocDebug(void* counter, size_t size)
164 {
165 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
166 void* const ptr = malloc(size);
167 if (ptr==NULL) return NULL;
168 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
169 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */
170 mcPtr->totalMalloc += size;
171 mcPtr->currentMalloc += size;
172 if (mcPtr->currentMalloc > mcPtr->peakMalloc)
173 mcPtr->peakMalloc = mcPtr->currentMalloc;
174 mcPtr->nbMalloc += 1;
175 return ptr;
176 }
177
FUZ_freeDebug(void * counter,void * address)178 static void FUZ_freeDebug(void* counter, void* address)
179 {
180 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
181 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
182 mcPtr->nbFree += 1;
183 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
184 free(address);
185 }
186
FUZ_displayMallocStats(mallocCounter_t count)187 static void FUZ_displayMallocStats(mallocCounter_t count)
188 {
189 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
190 (unsigned)(count.peakMalloc >> 10),
191 count.nbMalloc,
192 (unsigned)(count.totalMalloc >> 10));
193 }
194
FUZ_mallocTests_internal(unsigned seed,double compressibility,unsigned part,void * inBuffer,size_t inSize,void * outBuffer,size_t outSize)195 static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
196 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
197 {
198 /* test only played in verbose mode, as they are long */
199 if (g_displayLevel<3) return 0;
200
201 /* Create compressible noise */
202 if (!inBuffer || !outBuffer) {
203 DISPLAY("Not enough memory, aborting\n");
204 exit(1);
205 }
206 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
207
208 /* simple compression tests */
209 if (part <= 1)
210 { int compressionLevel;
211 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
212 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
213 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
214 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
215 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
216 ZSTD_freeCCtx(cctx);
217 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
218 FUZ_displayMallocStats(malcount);
219 } }
220
221 /* streaming compression tests */
222 if (part <= 2)
223 { int compressionLevel;
224 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
225 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
226 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
227 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
228 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
229 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
230 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
231 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
232 CHECK_Z( ZSTD_endStream(cstream, &out) );
233 ZSTD_freeCStream(cstream);
234 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
235 FUZ_displayMallocStats(malcount);
236 } }
237
238 /* advanced MT API test */
239 if (part <= 3)
240 { int nbThreads;
241 for (nbThreads=1; nbThreads<=4; nbThreads++) {
242 int compressionLevel;
243 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
244 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
245 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
246 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
247 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
248 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
249 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
250 ZSTD_freeCCtx(cctx);
251 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
252 nbThreads, compressionLevel);
253 FUZ_displayMallocStats(malcount);
254 } } }
255
256 /* advanced MT streaming API test */
257 if (part <= 4)
258 { int nbThreads;
259 for (nbThreads=1; nbThreads<=4; nbThreads++) {
260 int compressionLevel;
261 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
262 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
263 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
264 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
265 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
266 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
267 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
268 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
269 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
270 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
271 ZSTD_freeCCtx(cctx);
272 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
273 nbThreads, compressionLevel);
274 FUZ_displayMallocStats(malcount);
275 } } }
276
277 return 0;
278 }
279
FUZ_mallocTests(unsigned seed,double compressibility,unsigned part)280 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
281 {
282 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
283 size_t const outSize = ZSTD_compressBound(inSize);
284 void* const inBuffer = malloc(inSize);
285 void* const outBuffer = malloc(outSize);
286 int result;
287
288 /* Create compressible noise */
289 if (!inBuffer || !outBuffer) {
290 DISPLAY("Not enough memory, aborting \n");
291 exit(1);
292 }
293
294 result = FUZ_mallocTests_internal(seed, compressibility, part,
295 inBuffer, inSize, outBuffer, outSize);
296
297 free(inBuffer);
298 free(outBuffer);
299 return result;
300 }
301
302 #else
303
FUZ_mallocTests(unsigned seed,double compressibility,unsigned part)304 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
305 {
306 (void)seed; (void)compressibility; (void)part;
307 return 0;
308 }
309
310 #endif
311
FUZ_decodeSequences(BYTE * dst,ZSTD_Sequence * seqs,size_t seqsSize,BYTE * src,size_t size,ZSTD_sequenceFormat_e format)312 static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
313 BYTE* src, size_t size, ZSTD_sequenceFormat_e format)
314 {
315 size_t i;
316 size_t j;
317 for(i = 0; i < seqsSize; ++i) {
318 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
319 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
320 if (format == ZSTD_sf_noBlockDelimiters) {
321 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
322 }
323
324 memcpy(dst, src, seqs[i].litLength);
325 dst += seqs[i].litLength;
326 src += seqs[i].litLength;
327 size -= seqs[i].litLength;
328
329 if (seqs[i].offset != 0) {
330 for (j = 0; j < seqs[i].matchLength; ++j)
331 dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)];
332 dst += seqs[i].matchLength;
333 src += seqs[i].matchLength;
334 size -= seqs[i].matchLength;
335 }
336 }
337 if (format == ZSTD_sf_noBlockDelimiters) {
338 memcpy(dst, src, size);
339 }
340 }
341
342 #ifdef ZSTD_MULTITHREAD
343
344 typedef struct {
345 ZSTD_CCtx* cctx;
346 ZSTD_threadPool* pool;
347 void* CNBuffer;
348 size_t CNBuffSize;
349 void* compressedBuffer;
350 size_t compressedBufferSize;
351 void* decodedBuffer;
352 int err;
353 } threadPoolTests_compressionJob_payload;
354
threadPoolTests_compressionJob(void * payload)355 static void* threadPoolTests_compressionJob(void* payload) {
356 threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload;
357 size_t cSize;
358 if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1;
359 cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize);
360 if (ZSTD_isError(cSize)) args->err = 1;
361 if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1;
362 return payload;
363 }
364
threadPoolTests(void)365 static int threadPoolTests(void) {
366 int testResult = 0;
367 size_t err;
368
369 size_t const CNBuffSize = 5 MB;
370 void* const CNBuffer = malloc(CNBuffSize);
371 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
372 void* const compressedBuffer = malloc(compressedBufferSize);
373 void* const decodedBuffer = malloc(CNBuffSize);
374
375 size_t const kPoolNumThreads = 8;
376
377 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0);
378
379 DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: ");
380 {
381 ZSTD_CCtx* cctx = ZSTD_createCCtx();
382 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
383
384 size_t nbThreads = 1;
385 for (; nbThreads <= kPoolNumThreads; ++nbThreads) {
386 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
387 ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads);
388 err = ZSTD_CCtx_refThreadPool(cctx, pool);
389 if (ZSTD_isError(err)) {
390 DISPLAYLEVEL(3, "refThreadPool error!\n");
391 ZSTD_freeCCtx(cctx);
392 goto _output_error;
393 }
394 err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
395 if (ZSTD_isError(err)) {
396 DISPLAYLEVEL(3, "Compression error!\n");
397 ZSTD_freeCCtx(cctx);
398 goto _output_error;
399 }
400 err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err);
401 if (ZSTD_isError(err)) {
402 DISPLAYLEVEL(3, "Decompression error!\n");
403 ZSTD_freeCCtx(cctx);
404 goto _output_error;
405 }
406 }
407
408 ZSTD_freeCCtx(cctx);
409 ZSTD_freeThreadPool(pool);
410 }
411 DISPLAYLEVEL(3, "OK \n");
412
413 DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: ");
414 {
415 void* const decodedBuffer2 = malloc(CNBuffSize);
416 void* const compressedBuffer2 = malloc(compressedBufferSize);
417 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
418 ZSTD_CCtx* cctx1 = ZSTD_createCCtx();
419 ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
420
421 ZSTD_pthread_t t1;
422 ZSTD_pthread_t t2;
423 threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize,
424 compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */};
425 threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize,
426 compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */};
427
428 ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2);
429 ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2);
430 ZSTD_CCtx_refThreadPool(cctx1, pool);
431 ZSTD_CCtx_refThreadPool(cctx2, pool);
432
433 ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
434 ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
435 ZSTD_pthread_join(t1);
436 ZSTD_pthread_join(t2);
437
438 assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
439 free(decodedBuffer2);
440 free(compressedBuffer2);
441
442 ZSTD_freeThreadPool(pool);
443 ZSTD_freeCCtx(cctx1);
444 ZSTD_freeCCtx(cctx2);
445
446 if (p1.err || p2.err) goto _output_error;
447 }
448 DISPLAYLEVEL(3, "OK \n");
449
450 _end:
451 free(CNBuffer);
452 free(compressedBuffer);
453 free(decodedBuffer);
454 return testResult;
455
456 _output_error:
457 testResult = 1;
458 DISPLAY("Error detected in Unit tests ! \n");
459 goto _end;
460 }
461 #endif /* ZSTD_MULTITHREAD */
462
463 /*=============================================
464 * Unit tests
465 =============================================*/
466
test_compressBound(unsigned tnb)467 static void test_compressBound(unsigned tnb)
468 {
469 DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
470
471 /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
472 * for a large range of known valid values */
473 DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
474 { int s;
475 for (s=0; s<30; s++) {
476 size_t const w = (size_t)1 << s;
477 CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
478 } }
479
480 /* Ensure error if srcSize too big */
481 { size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
482 CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
483 CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
484 }
485
486 DISPLAYLEVEL(3, "OK \n");
487 }
488
test_decompressBound(unsigned tnb)489 static void test_decompressBound(unsigned tnb)
490 {
491 DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
492
493 /* Simple compression, with size : should provide size; */
494 { const char example[] = "abcd";
495 char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
496 size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
497 CHECK_Z(cSize);
498 CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
499 }
500
501 /* Simple small compression without size : should provide 1 block size */
502 { char cBuffer[ZSTD_COMPRESSBOUND(0)];
503 ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
504 ZSTD_inBuffer in = { NULL, 0, 0 };
505 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
506 assert(cctx);
507 CHECK_Z( ZSTD_initCStream(cctx, 0) );
508 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
509 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
510 CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
511 ZSTD_freeCCtx(cctx);
512 }
513
514 /* Attempt to overflow 32-bit intermediate multiplication result
515 * This requires dBound >= 4 GB, aka 2^32.
516 * This requires 2^32 / 2^17 = 2^15 blocks
517 * => create 2^15 blocks (can be empty, or just 1 byte). */
518 { const char input[] = "a";
519 size_t const nbBlocks = (1 << 15) + 1;
520 size_t blockNb;
521 size_t const outCapacity = 1 << 18; /* large margin */
522 char* const outBuffer = malloc (outCapacity);
523 ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
524 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
525 assert(cctx);
526 assert(outBuffer);
527 CHECK_Z( ZSTD_initCStream(cctx, 0) );
528 for (blockNb=0; blockNb<nbBlocks; blockNb++) {
529 ZSTD_inBuffer in = { input, sizeof(input), 0 };
530 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
531 CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
532 }
533 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
534 CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ );
535 ZSTD_freeCCtx(cctx);
536 free(outBuffer);
537 }
538
539 DISPLAYLEVEL(3, "OK \n");
540 }
541
test_setCParams(unsigned tnb)542 static void test_setCParams(unsigned tnb)
543 {
544 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
545 ZSTD_compressionParameters cparams;
546 assert(cctx);
547
548 DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
549
550 /* valid cparams */
551 cparams = ZSTD_getCParams(1, 0, 0);
552 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
553
554 /* invalid cparams (must fail) */
555 cparams.windowLog = 99;
556 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
557
558 free(cctx);
559 DISPLAYLEVEL(3, "OK \n");
560 }
561
basicUnitTests(U32 const seed,double compressibility)562 static int basicUnitTests(U32 const seed, double compressibility)
563 {
564 size_t const CNBuffSize = 5 MB;
565 void* const CNBuffer = malloc(CNBuffSize);
566 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
567 void* const compressedBuffer = malloc(compressedBufferSize);
568 void* const decodedBuffer = malloc(CNBuffSize);
569 int testResult = 0;
570 unsigned testNb=0;
571 size_t cSize;
572
573 /* Create compressible noise */
574 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
575 DISPLAY("Not enough memory, aborting\n");
576 testResult = 1;
577 goto _end;
578 }
579 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
580
581 /* Basic tests */
582 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
583 { const char* errorString = ZSTD_getErrorName(0);
584 DISPLAYLEVEL(3, "OK : %s \n", errorString);
585 }
586
587 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
588 { const char* errorString = ZSTD_getErrorName(499);
589 DISPLAYLEVEL(3, "OK : %s \n", errorString);
590 }
591
592 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
593 { int const mcl = ZSTD_minCLevel();
594 DISPLAYLEVEL(3, "%i (OK) \n", mcl);
595 }
596
597 DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
598 { int const defaultCLevel = ZSTD_defaultCLevel();
599 if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
600 DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
601 }
602
603 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
604 { unsigned const vn = ZSTD_versionNumber();
605 DISPLAYLEVEL(3, "%u (OK) \n", vn);
606 }
607
608 test_compressBound(testNb++);
609
610 test_decompressBound(testNb++);
611
612 test_setCParams(testNb++);
613
614 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
615 {
616 ZSTD_compressionParameters params;
617 memset(¶ms, 0, sizeof(params));
618 params.windowLog = 10;
619 params.hashLog = 19;
620 params.chainLog = 19;
621 params = ZSTD_adjustCParams(params, 1000, 100000);
622 if (params.hashLog != 18) goto _output_error;
623 if (params.chainLog != 17) goto _output_error;
624 }
625 DISPLAYLEVEL(3, "OK \n");
626
627 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
628 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
629 if (cctx==NULL) goto _output_error;
630 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
631 compressedBuffer, compressedBufferSize,
632 CNBuffer, CNBuffSize, 1) );
633 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
634
635 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
636 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
637 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
638 }
639 ZSTD_freeCCtx(cctx);
640 }
641
642 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
643 {
644 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
645 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
646 if (!ZSTD_isError(size)) goto _output_error;
647 }
648 DISPLAYLEVEL(3, "OK \n");
649
650 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
651 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
652 if (rSize != CNBuffSize) goto _output_error;
653 }
654 DISPLAYLEVEL(3, "OK \n");
655
656 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
657 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
658 if (rSize != CNBuffSize) goto _output_error;
659 }
660 DISPLAYLEVEL(3, "OK \n");
661
662 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
663 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
664 if (rSize != CNBuffSize) goto _output_error;
665 }
666 DISPLAYLEVEL(3, "OK \n");
667
668 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
669 {
670 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
671 if (bound != CNBuffSize) goto _output_error;
672 }
673 DISPLAYLEVEL(3, "OK \n");
674
675 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
676 {
677 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
678 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
679 }
680 DISPLAYLEVEL(3, "OK \n");
681
682 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
683 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
684 if (r != CNBuffSize) goto _output_error; }
685 DISPLAYLEVEL(3, "OK \n");
686
687 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
688 {
689 ZSTD_DCtx* dctx = ZSTD_createDCtx();
690 size_t r;
691 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
692 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
693 if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
694 ZSTD_freeDCtx(dctx);
695 }
696 DISPLAYLEVEL(3, "OK \n");
697
698 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
699 { size_t u;
700 for (u=0; u<CNBuffSize; u++) {
701 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
702 } }
703 DISPLAYLEVEL(3, "OK \n");
704
705 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
706 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
707 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
708 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
709 if (cctx==NULL) goto _output_error;
710 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */
711 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */
712 ZSTD_freeCCtx(cctx);
713 }
714 DISPLAYLEVEL(3, "OK \n");
715
716 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
717 {
718 ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
719 assert(!ZSTD_checkCParams(params.cParams));
720 }
721 DISPLAYLEVEL(3, "OK \n");
722
723 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
724 {
725 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
726 assert(dctx != NULL);
727 assert(ZSTD_sizeof_DCtx(dctx) != 0);
728 ZSTD_freeDCtx(dctx);
729 }
730 DISPLAYLEVEL(3, "OK \n");
731
732 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
733 {
734 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
735 * to a data pointer. (Although in POSIX you're required to be allowed
736 * to do it...) So we have to fall back to our trusty friend memcpy. */
737 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
738 ZSTD_getDictID_fromDDict;
739 ZSTD_DStream* (* const funcptr_createDStream)(
740 ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
741 void (* const funcptr_copyDCtx)(
742 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
743 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
744 ZSTD_nextInputType;
745 const void *voidptr_getDictID;
746 const void *voidptr_createDStream;
747 const void *voidptr_copyDCtx;
748 const void *voidptr_nextInputType;
749 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
750 memcpy(
751 (void*)&voidptr_getDictID,
752 (const void*)&funcptr_getDictID,
753 sizeof(void*));
754 memcpy(
755 (void*)&voidptr_createDStream,
756 (const void*)&funcptr_createDStream,
757 sizeof(void*));
758 memcpy(
759 (void*)&voidptr_copyDCtx,
760 (const void*)&funcptr_copyDCtx,
761 sizeof(void*));
762 memcpy(
763 (void*)&voidptr_nextInputType,
764 (const void*)&funcptr_nextInputType,
765 sizeof(void*));
766 DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
767 DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
768 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
769 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
770 }
771 DISPLAYLEVEL(3, ": OK \n");
772
773 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
774 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
775 { size_t const r = ZSTD_decompress_usingDict(dctx,
776 decodedBuffer, CNBuffSize,
777 compressedBuffer, cSize,
778 NULL, 0);
779 if (r != CNBuffSize) goto _output_error;
780 }
781 ZSTD_freeDCtx(dctx);
782 }
783 DISPLAYLEVEL(3, "OK \n");
784
785 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
786 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
787 { size_t const r = ZSTD_decompress_usingDDict(dctx,
788 decodedBuffer, CNBuffSize,
789 compressedBuffer, cSize,
790 NULL);
791 if (r != CNBuffSize) goto _output_error;
792 }
793 ZSTD_freeDCtx(dctx);
794 }
795 DISPLAYLEVEL(3, "OK \n");
796
797 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
798 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
799 if (!ZSTD_isError(r)) goto _output_error;
800 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
801 DISPLAYLEVEL(3, "OK \n");
802
803 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
804 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
805 if (!ZSTD_isError(r)) goto _output_error;
806 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
807 DISPLAYLEVEL(3, "OK \n");
808
809 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
810 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
811 if (!ZSTD_isError(r)) goto _output_error;
812 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
813 DISPLAYLEVEL(3, "OK \n");
814
815 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
816 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
817 if (!ZSTD_isError(r)) goto _output_error;
818 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
819 DISPLAYLEVEL(3, "OK \n");
820
821 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
822 { /* create compressed buffer with checksumming enabled */
823 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
824 if (!cctx) {
825 DISPLAY("Not enough memory, aborting\n");
826 testResult = 1;
827 goto _end;
828 }
829 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
830 CHECK_VAR(cSize, ZSTD_compress2(cctx,
831 compressedBuffer, compressedBufferSize,
832 CNBuffer, CNBuffSize) );
833 ZSTD_freeCCtx(cctx);
834 }
835 { /* copy the compressed buffer and corrupt the checksum */
836 size_t r;
837 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
838 if (!dctx) {
839 DISPLAY("Not enough memory, aborting\n");
840 testResult = 1;
841 goto _end;
842 }
843
844 ((char*)compressedBuffer)[cSize-1] += 1;
845 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
846 if (!ZSTD_isError(r)) goto _output_error;
847 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;
848
849 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
850 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
851 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */
852 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
853 if (ZSTD_isError(r)) goto _output_error;
854
855 ZSTD_freeDCtx(dctx);
856 }
857 DISPLAYLEVEL(3, "OK \n");
858
859
860 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
861 { /* create compressed buffer with content size missing */
862 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
863 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
864 CHECK_VAR(cSize, ZSTD_compress2(cctx,
865 compressedBuffer, compressedBufferSize,
866 CNBuffer, CNBuffSize) );
867 ZSTD_freeCCtx(cctx);
868 }
869 { /* ensure frame content size is missing */
870 ZSTD_frameHeader zfh;
871 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
872 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
873 }
874 { /* ensure CNBuffSize <= decompressBound */
875 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
876 if (CNBuffSize > bound) goto _output_error;
877 }
878 DISPLAYLEVEL(3, "OK \n");
879
880 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
881 {
882 size_t const largeFrameSrcSize = 200;
883 size_t const smallFrameSrcSize = 10;
884 size_t const nbFrames = 256;
885
886 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
887 int sizeReduced = 0;
888
889 BYTE* const dst = (BYTE*)compressedBuffer;
890 ZSTD_DCtx* dctx = ZSTD_createDCtx();
891
892 /* create a large frame and then a bunch of small frames */
893 size_t srcSize = ZSTD_compress((void*)dst,
894 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
895 for (i = 0; i < nbFrames; i++)
896 srcSize += ZSTD_compress((void*)(dst + srcSize),
897 compressedBufferSize - srcSize, CNBuffer,
898 smallFrameSrcSize, 3);
899
900 /* decompressStream and make sure that dctx size was reduced at least once */
901 while (consumed < srcSize) {
902 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
903 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
904 ZSTD_decompressStream(dctx, &out, &in);
905 consumed += in.pos;
906 produced += out.pos;
907
908 /* success! size was reduced from the previous frame */
909 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
910 sizeReduced = 1;
911
912 prevDCtxSize = ZSTD_sizeof_DCtx(dctx);
913 }
914
915 assert(sizeReduced);
916
917 ZSTD_freeDCtx(dctx);
918 }
919 DISPLAYLEVEL(3, "OK \n");
920
921 {
922 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
923 ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1);
924 ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
925 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
926
927 DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++);
928 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1));
929 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
930 DISPLAYLEVEL(3, "OK \n");
931
932 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++);
933 CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1));
934 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
935 DISPLAYLEVEL(3, "OK \n");
936
937 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++);
938 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict));
939 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
940 DISPLAYLEVEL(3, "OK \n");
941
942 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++);
943 CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params));
944 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
945 DISPLAYLEVEL(3, "OK \n");
946
947 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++);
948 CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams));
949 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
950 DISPLAYLEVEL(3, "OK \n");
951
952 ZSTD_freeCDict(cdict);
953 ZSTD_freeCCtx(cctx);
954 }
955
956 DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2K", testNb++);
957 {
958 ZSTD_CCtx* cctx = ZSTD_createCCtx();
959 ZSTD_DCtx* dctx = ZSTD_createDCtx();
960 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
961 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048));
962 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048));
963
964 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
965 CHECK_Z(cSize);
966 CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize));
967
968 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024));
969 CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize)));
970
971 ZSTD_freeDCtx(dctx);
972 ZSTD_freeCCtx(cctx);
973 }
974
975 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
976 {
977 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
978
979 size_t const size = (1U << 10);
980 size_t const dstCapacity = ZSTD_compressBound(size);
981 void* dict = (void*)malloc(size);
982 void* src = (void*)malloc(size);
983 void* dst = (void*)malloc(dstCapacity);
984
985 RDG_genBuffer(dict, size, 0.5, 0.5, seed);
986 RDG_genBuffer(src, size, 0.5, 0.5, seed);
987
988 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
989 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
990
991 ZSTD_freeCCtx(cctx);
992 free(dict);
993 free(src);
994 free(dst);
995 }
996 DISPLAYLEVEL(3, "OK \n");
997
998 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
999 {
1000 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1001 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1002 void* dict = (void*)malloc(CNBuffSize);
1003 int nbWorkers;
1004
1005 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) {
1006 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
1007 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
1008
1009 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
1010 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1011 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
1012 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1013 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
1014 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1015 CHECK_Z(cSize);
1016 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize));
1017 }
1018
1019 ZSTD_freeCCtx(cctx);
1020 ZSTD_freeDCtx(dctx);
1021 free(dict);
1022 }
1023 DISPLAYLEVEL(3, "OK \n");
1024
1025 DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
1026 {
1027 size_t const testSize = 1024;
1028 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1029 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1030 char* dict = (char*)malloc(2 * testSize);
1031 int ldmEnabled, level;
1032
1033 RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
1034 RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
1035 memcpy(dict + testSize, CNBuffer, testSize);
1036 for (level = 1; level <= 5; ++level) {
1037 for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
1038 size_t cSize0;
1039 XXH64_hash_t compressedChecksum0;
1040
1041 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1042 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
1043 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
1044 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
1045
1046 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1047 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
1048 CHECK_Z(cSize);
1049 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
1050
1051 cSize0 = cSize;
1052 compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
1053
1054 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1055 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
1056 CHECK_Z(cSize);
1057
1058 if (cSize != cSize0) goto _output_error;
1059 if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
1060 }
1061 }
1062
1063 ZSTD_freeCCtx(cctx);
1064 ZSTD_freeDCtx(dctx);
1065 free(dict);
1066 }
1067 DISPLAYLEVEL(3, "OK \n");
1068
1069 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
1070 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1071 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1072 size_t const srcSize = 300 KB;
1073 size_t const flushSize = 128 KB + 5;
1074 size_t const dstSize = ZSTD_compressBound(srcSize);
1075 char* src = (char*)CNBuffer;
1076 char* dst = (char*)compressedBuffer;
1077
1078 ZSTD_outBuffer out = { dst, dstSize, 0 };
1079 ZSTD_inBuffer in = { src, flushSize, 0 };
1080
1081 if (!cctx || !dctx) {
1082 DISPLAY("Not enough memory, aborting\n");
1083 testResult = 1;
1084 goto _end;
1085 }
1086
1087 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed);
1088 /* Force an LDM to exist that crosses block boundary into uncompressible block */
1089 memcpy(src + 125 KB, src, 3 KB + 5);
1090
1091 /* Enable MT, LDM, and opt parser */
1092 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
1093 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1094 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1095 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1096
1097 /* Flushes a block of 128 KB and block of 5 bytes */
1098 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1099
1100 /* Compress the rest */
1101 in.size = 300 KB;
1102 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1103
1104 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos));
1105
1106 ZSTD_freeCCtx(cctx);
1107 ZSTD_freeDCtx(dctx);
1108 }
1109 DISPLAYLEVEL(3, "OK \n");
1110
1111 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++);
1112 {
1113 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1114 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1115 void* dict = (void*)malloc(CNBuffSize);
1116 size_t const kWindowLog = 10;
1117 size_t const kWindowSize = (size_t)1 << kWindowLog;
1118 size_t const dictSize = kWindowSize * 10;
1119 size_t const srcSize1 = kWindowSize / 2;
1120 size_t const srcSize2 = kWindowSize * 10;
1121
1122 CHECK(cctx!=NULL);
1123 CHECK(dctx!=NULL);
1124 CHECK(dict!=NULL);
1125 if (CNBuffSize < dictSize) goto _output_error;
1126
1127 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
1128 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed);
1129
1130 /* Enable checksum to verify round trip. */
1131 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1132 /* Disable content size to skip single-pass decompression. */
1133 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
1134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
1135 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1136 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
1137 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
1138 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
1139 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
1140
1141 /* Round trip once with a dictionary. */
1142 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1143 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
1144 CHECK_Z(cSize);
1145 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1146
1147 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
1148 /* Streaming decompression to catch out of bounds offsets. */
1149 {
1150 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1151 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1152 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1153 CHECK_Z(dSize);
1154 if (dSize != 0) goto _output_error;
1155 }
1156
1157 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
1158 /* Round trip once with a dictionary. */
1159 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1160 { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
1161 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1162 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1163 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1164 cSize = out.pos;
1165 }
1166 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1167
1168 { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
1169 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1170 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1171 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1172 cSize = out.pos;
1173 }
1174 /* Streaming decompression to catch out of bounds offsets. */
1175 { ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1176 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1177 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1178 CHECK_Z(dSize);
1179 if (dSize != 0) goto _output_error;
1180 }
1181
1182 ZSTD_freeCCtx(cctx);
1183 ZSTD_freeDCtx(dctx);
1184 free(dict);
1185 }
1186 DISPLAYLEVEL(3, "OK \n");
1187
1188 /* Note: this test takes 0.5 seconds to run */
1189 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
1190 {
1191 /* test a big buffer so that ldm can take effect */
1192 size_t const size = 100 MB;
1193 int const windowLog = 27;
1194 size_t const dstSize = ZSTD_compressBound(size);
1195
1196 void* dict = (void*)malloc(size);
1197 void* src = (void*)malloc(size);
1198 void* dst = (void*)malloc(dstSize);
1199 void* recon = (void*)malloc(size);
1200
1201 size_t refPrefixCompressedSize = 0;
1202 size_t refPrefixLdmCompressedSize = 0;
1203 size_t reconSize = 0;
1204
1205 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1206 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1207
1208 /* make dict and src the same uncompressible data */
1209 RDG_genBuffer(src, size, 0, 0, seed);
1210 memcpy(dict, src, size);
1211 assert(!memcmp(dict, src, size));
1212
1213 /* set level 1 and windowLog to cover src */
1214 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
1215 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
1216
1217 /* compress on level 1 using just refPrefix and no ldm */
1218 ZSTD_CCtx_refPrefix(cctx, dict, size);
1219 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1220 assert(!ZSTD_isError(refPrefixCompressedSize));
1221
1222 /* test round trip just refPrefix */
1223 ZSTD_DCtx_refPrefix(dctx, dict, size);
1224 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
1225 assert(!ZSTD_isError(reconSize));
1226 assert(reconSize == size);
1227 assert(!memcmp(recon, src, size));
1228
1229 /* compress on level 1 using refPrefix and ldm */
1230 ZSTD_CCtx_refPrefix(cctx, dict, size);;
1231 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
1232 refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1233 assert(!ZSTD_isError(refPrefixLdmCompressedSize));
1234
1235 /* test round trip refPrefix + ldm*/
1236 ZSTD_DCtx_refPrefix(dctx, dict, size);
1237 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize);
1238 assert(!ZSTD_isError(reconSize));
1239 assert(reconSize == size);
1240 assert(!memcmp(recon, src, size));
1241
1242 /* make sure that refPrefixCompressedSize is significantly greater */
1243 assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize);
1244 /* make sure the ldm compressed size is less than 1% of original */
1245 assert((double)refPrefixLdmCompressedSize / (double)size < 0.01);
1246
1247 ZSTD_freeDCtx(dctx);
1248 ZSTD_freeCCtx(cctx);
1249 free(recon);
1250 free(dict);
1251 free(src);
1252 free(dst);
1253 }
1254 DISPLAYLEVEL(3, "OK \n");
1255
1256 DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
1257 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
1258 CHECK_Z(cSize);
1259 CHECK_LT(CNBuffSize, cSize);
1260 {
1261 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1262 size_t const outputSize = (CNBuffSize + margin);
1263 char* output = malloc(outputSize);
1264 char* input = output + outputSize - cSize;
1265 CHECK_LT(cSize, CNBuffSize + margin);
1266 CHECK(output != NULL);
1267 CHECK_Z(margin);
1268 CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
1269 memcpy(input, compressedBuffer, cSize);
1270
1271 {
1272 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1273 CHECK_Z(dSize);
1274 CHECK_EQ(dSize, CNBuffSize);
1275 }
1276 CHECK(!memcmp(output, CNBuffer, CNBuffSize));
1277 free(output);
1278 }
1279 DISPLAYLEVEL(3, "OK \n");
1280
1281 DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
1282 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1283 CHECK_Z(cSize);
1284 {
1285 size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1286 CHECK_Z(cSize2);
1287 cSize += cSize2;
1288 }
1289 {
1290 size_t const srcSize = (CNBuffSize / 3) * 2;
1291 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1292 size_t const outputSize = (CNBuffSize + margin);
1293 char* output = malloc(outputSize);
1294 char* input = output + outputSize - cSize;
1295 CHECK_LT(cSize, CNBuffSize + margin);
1296 CHECK(output != NULL);
1297 CHECK_Z(margin);
1298 memcpy(input, compressedBuffer, cSize);
1299
1300 {
1301 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1302 CHECK_Z(dSize);
1303 CHECK_EQ(dSize, srcSize);
1304 }
1305 CHECK(!memcmp(output, CNBuffer, srcSize));
1306 free(output);
1307 }
1308 DISPLAYLEVEL(3, "OK \n");
1309
1310 DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++);
1311 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1312 size_t const srcSize = 256 * 1024;
1313 U32 const compressibleLenU32 = 32 * 1024 / 4;
1314 U32 const blockSizeU32 = 128 * 1024 / 4;
1315 U32 const litLenU32 = 64 * 1024 / 4;
1316 U32* data = (U32*)malloc(srcSize);
1317 size_t dSize;
1318
1319 if (data == NULL || cctx == NULL) goto _output_error;
1320
1321 /* Generate data without any matches */
1322 RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U);
1323 /* Generate 32K of compressible data */
1324 RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe);
1325
1326 /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */
1327 data[compressibleLenU32 + 0] = 0xFFFFFFFF;
1328 data[compressibleLenU32 + 1] = 0xEEEEEEEE;
1329 data[compressibleLenU32 + 4] = 0xFFFFFFFF;
1330 data[compressibleLenU32 + 5] = 0xEEEEEEEE;
1331
1332 /* Add a match of offset=16, length=8 at idx=64K + 64.
1333 * This generates a sequence with llen=64K, and repeat code 1.
1334 * The block splitter thought this was ll0, and corrupted the
1335 * repeat offset history.
1336 */
1337 data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD;
1338 data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC;
1339 data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD;
1340 data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC;
1341
1342 /* Add a match of offset=16, length=8 at idx=128K + 16.
1343 * This should generate a sequence with repeat code = 1.
1344 * But the block splitters mistake caused zstd to generate
1345 * repeat code = 2, corrupting the data.
1346 */
1347 data[blockSizeU32] = 0xBBBBBBBB;
1348 data[blockSizeU32 + 1] = 0xAAAAAAAA;
1349 data[blockSizeU32 + 4] = 0xBBBBBBBB;
1350 data[blockSizeU32 + 5] = 0xAAAAAAAA;
1351
1352 /* Generate a golden file from this data in case datagen changes and
1353 * doesn't generate the exact same data. We will also test this golden file.
1354 */
1355 if (0) {
1356 FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb");
1357 fwrite(data, 1, srcSize, f);
1358 fclose(f);
1359 }
1360
1361 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1362 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7));
1363 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable));
1364
1365 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize);
1366 CHECK_Z(cSize);
1367 dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1368 CHECK_Z(dSize);
1369 CHECK_EQ(dSize, srcSize);
1370 CHECK(!memcmp(decodedBuffer, data, srcSize));
1371
1372 free(data);
1373 ZSTD_freeCCtx(cctx);
1374 }
1375 DISPLAYLEVEL(3, "OK \n");
1376
1377 DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++);
1378 {
1379 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1380 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
1381 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
1382
1383 /* This is the number of bytes to stream before ending. This value
1384 * was obtained by trial and error :/. */
1385
1386 const size_t streamCompressThreshold = 161792;
1387 const size_t streamCompressDelta = 1024;
1388
1389 /* The first 1/5 of the buffer is compressible and the last 4/5 is
1390 * uncompressible. This is an approximation of the type of data
1391 * the fuzzer generated to catch this bug. Streams like this were making
1392 * zstd generate noCompress superblocks (which are larger than the src
1393 * they come from). Do this enough times, and we'll run out of room
1394 * and throw a dstSize_tooSmall error. */
1395
1396 const size_t compressiblePartSize = srcSize/5;
1397 const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
1398 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
1399 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
1400
1401 /* Setting target block size so that superblock is used */
1402
1403 assert(cctx != NULL);
1404 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
1405
1406 { size_t read;
1407 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
1408 ZSTD_inBuffer in = {src, streamCompressDelta, 0};
1409 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1410 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1411 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1412 src += streamCompressDelta; srcSize -= streamCompressDelta;
1413 dst += out.pos; dstCapacity -= out.pos;
1414 } }
1415
1416 /* This is trying to catch a dstSize_tooSmall error */
1417
1418 { ZSTD_inBuffer in = {src, srcSize, 0};
1419 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1420 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1421 }
1422 ZSTD_freeCCtx(cctx);
1423 }
1424 DISPLAYLEVEL(3, "OK \n");
1425
1426 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++);
1427 /* Generate the same data 20 times over */
1428 { size_t const avgChunkSize = CNBuffSize / 20;
1429 size_t b;
1430 for (b = 0; b < CNBuffSize; b += avgChunkSize) {
1431 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize);
1432 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed);
1433 } }
1434 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1435 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1436 size_t const allowedExpansion = (CNBuffSize * 3 / 1000);
1437 size_t superCSize;
1438 CHECK_Z(normalCSize);
1439 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
1440 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000);
1441 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1442 CHECK_Z(superCSize);
1443 if (superCSize > normalCSize + allowedExpansion) {
1444 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion);
1445 goto _output_error;
1446 }
1447 ZSTD_freeCCtx(cctx);
1448 }
1449 DISPLAYLEVEL(3, "OK \n");
1450
1451 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
1452 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
1453 /* This tests whether or not we leave enough room for the checksum at the end
1454 * of the dst buffer. The bug that motivated this test was found by the
1455 * stream_round_trip fuzzer but this crashes for the same reason and is
1456 * far more compact than re-creating the stream_round_trip fuzzer's code path */
1457 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1458 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64);
1459 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278)));
1460 ZSTD_freeCCtx(cctx);
1461 }
1462 DISPLAYLEVEL(3, "OK \n");
1463
1464 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
1465 { int level = -1;
1466 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1467 if (!cctx) goto _output_error;
1468 for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
1469 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
1470 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
1471 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
1472 }
1473 ZSTD_freeCCtx(cctx);
1474 }
1475 DISPLAYLEVEL(3, "OK \n");
1476
1477 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
1478 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1479 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
1480 if (ZSTD_isError(r)) goto _output_error;
1481 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
1482 ZSTD_freeCCtx(cctx);
1483 cSize = r;
1484 }
1485 DISPLAYLEVEL(3, "OK \n");
1486
1487 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++);
1488 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize);
1489 if (ZSTD_isError(r)) goto _output_error;
1490 if (r != 0) goto _output_error;
1491 }
1492 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1493 ZSTD_outBuffer output;
1494 if (cctx==NULL) goto _output_error;
1495 output.dst = compressedBuffer;
1496 output.size = compressedBufferSize;
1497 output.pos = 0;
1498 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */
1499 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */
1500 CHECK_Z( ZSTD_endStream(cctx, &output) );
1501 ZSTD_freeCCtx(cctx);
1502 /* single scan decompression */
1503 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos);
1504 if (ZSTD_isError(r)) goto _output_error;
1505 if (r != 0) goto _output_error;
1506 }
1507 /* streaming decompression */
1508 { ZSTD_DCtx* const dstream = ZSTD_createDStream();
1509 ZSTD_inBuffer dinput;
1510 ZSTD_outBuffer doutput;
1511 size_t ipos;
1512 if (dstream==NULL) goto _output_error;
1513 dinput.src = compressedBuffer;
1514 dinput.size = 0;
1515 dinput.pos = 0;
1516 doutput.dst = NULL;
1517 doutput.size = 0;
1518 doutput.pos = 0;
1519 CHECK_Z ( ZSTD_initDStream(dstream) );
1520 for (ipos=1; ipos<=output.pos; ipos++) {
1521 dinput.size = ipos;
1522 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) );
1523 }
1524 if (doutput.pos != 0) goto _output_error;
1525 ZSTD_freeDStream(dstream);
1526 }
1527 }
1528 DISPLAYLEVEL(3, "OK \n");
1529
1530 DISPLAYLEVEL(3, "test%3d : reuse CCtx with expanding block size : ", testNb++);
1531 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1532 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1533 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
1534 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
1535 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
1536
1537 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */
1538 { size_t const inSize = 2* 128 KB;
1539 size_t const outSize = ZSTD_compressBound(inSize);
1540 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
1541 /* will fail if blockSize is not resized */
1542 }
1543 ZSTD_freeCCtx(cctx);
1544 }
1545 DISPLAYLEVEL(3, "OK \n");
1546
1547 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
1548 { size_t const sampleSize = 30;
1549 int i;
1550 for (i=0; i<20; i++)
1551 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
1552 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
1553 for (i=1; i<=19; i++) {
1554 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1555 size_t size1, size2;
1556 DISPLAYLEVEL(5, "l%i ", i);
1557 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1558 CHECK_Z(size1);
1559
1560 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1561 CHECK_Z(size2);
1562 CHECK_EQ(size1, size2);
1563
1564 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) );
1565 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize);
1566 CHECK_Z(size2);
1567 CHECK_EQ(size1, size2);
1568
1569 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */
1570 CHECK_Z(size2);
1571 CHECK_EQ(size1, size2);
1572
1573 { ZSTD_inBuffer inb;
1574 ZSTD_outBuffer outb;
1575 inb.src = CNBuffer;
1576 inb.pos = 0;
1577 inb.size = sampleSize;
1578 outb.dst = compressedBuffer;
1579 outb.pos = 0;
1580 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
1581 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );
1582 assert(inb.pos == inb.size);
1583 CHECK_EQ(size1, outb.pos);
1584 }
1585
1586 ZSTD_freeCCtx(cctx);
1587 }
1588 }
1589 DISPLAYLEVEL(3, "OK \n");
1590
1591 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
1592 { size_t const sampleSize = 1024;
1593 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1594 ZSTD_inBuffer inb;
1595 ZSTD_outBuffer outb;
1596 inb.src = CNBuffer;
1597 inb.pos = 0;
1598 inb.size = 0;
1599 outb.dst = compressedBuffer;
1600 outb.pos = 0;
1601 outb.size = compressedBufferSize;
1602 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
1603
1604 inb.size = sampleSize; /* start with something, so that context is already used */
1605 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1606 assert(inb.pos == inb.size);
1607 outb.pos = 0; /* cancel output */
1608
1609 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
1610 inb.size = 4; /* too small size : compression will be skipped */
1611 inb.pos = 0;
1612 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1613 assert(inb.pos == inb.size);
1614
1615 inb.size += 5; /* too small size : compression will be skipped */
1616 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1617 assert(inb.pos == inb.size);
1618
1619 inb.size += 11; /* small enough to attempt compression */
1620 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1621 assert(inb.pos == inb.size);
1622
1623 assert(inb.pos < sampleSize);
1624 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */
1625 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1626 assert(inb.pos == inb.size);
1627 ZSTD_freeCCtx(cctx);
1628 }
1629 DISPLAYLEVEL(3, "OK \n");
1630
1631 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
1632 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1633 ZSTD_outBuffer out = {NULL, 0, 0};
1634 ZSTD_inBuffer in = {NULL, 0, 0};
1635 int value;
1636
1637 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1638 CHECK_EQ(value, 3);
1639 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1640 CHECK_EQ(value, 0);
1641 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
1642 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1643 CHECK_EQ(value, 3);
1644 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1645 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1646 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
1647 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1648 CHECK_EQ(value, 7);
1649 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1650 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1651 /* Start a compression job */
1652 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1653 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1654 CHECK_EQ(value, 7);
1655 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1656 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1657 /* Reset the CCtx */
1658 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
1659 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1660 CHECK_EQ(value, 7);
1661 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1662 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1663 /* Reset the parameters */
1664 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
1665 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1666 CHECK_EQ(value, 3);
1667 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1668 CHECK_EQ(value, 0);
1669
1670 ZSTD_freeCCtx(cctx);
1671 }
1672 DISPLAYLEVEL(3, "OK \n");
1673
1674 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++);
1675 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1676 int value;
1677 ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0);
1678 cparams.strategy = -1;
1679 /* Set invalid cParams == no change. */
1680 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
1681
1682 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1683 CHECK_EQ(value, 0);
1684 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1685 CHECK_EQ(value, 0);
1686 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1687 CHECK_EQ(value, 0);
1688 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1689 CHECK_EQ(value, 0);
1690 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1691 CHECK_EQ(value, 0);
1692 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1693 CHECK_EQ(value, 0);
1694 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1695 CHECK_EQ(value, 0);
1696
1697 cparams = ZSTD_getCParams(12, 0, 0);
1698 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
1699
1700 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1701 CHECK_EQ(value, (int)cparams.windowLog);
1702 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1703 CHECK_EQ(value, (int)cparams.chainLog);
1704 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1705 CHECK_EQ(value, (int)cparams.hashLog);
1706 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1707 CHECK_EQ(value, (int)cparams.searchLog);
1708 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1709 CHECK_EQ(value, (int)cparams.minMatch);
1710 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1711 CHECK_EQ(value, (int)cparams.targetLength);
1712 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1713 CHECK_EQ(value, (int)cparams.strategy);
1714
1715 ZSTD_freeCCtx(cctx);
1716 }
1717
1718 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++);
1719 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1720 int value;
1721 ZSTD_frameParameters fparams = {0, 1, 1};
1722
1723 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1724 CHECK_EQ(value, 1);
1725 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1726 CHECK_EQ(value, 0);
1727 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1728 CHECK_EQ(value, 1);
1729
1730 CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams));
1731
1732 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1733 CHECK_EQ(value, fparams.contentSizeFlag);
1734 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1735 CHECK_EQ(value, fparams.checksumFlag);
1736 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1737 CHECK_EQ(value, !fparams.noDictIDFlag);
1738
1739 ZSTD_freeCCtx(cctx);
1740 }
1741
1742 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++);
1743 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1744 int value;
1745 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
1746 params.cParams.strategy = -1;
1747 /* Set invalid params == no change. */
1748 CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params)));
1749
1750 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1751 CHECK_EQ(value, 0);
1752 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1753 CHECK_EQ(value, 0);
1754 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1755 CHECK_EQ(value, 0);
1756 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1757 CHECK_EQ(value, 0);
1758 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1759 CHECK_EQ(value, 0);
1760 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1761 CHECK_EQ(value, 0);
1762 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1763 CHECK_EQ(value, 0);
1764 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1765 CHECK_EQ(value, 1);
1766 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1767 CHECK_EQ(value, 0);
1768 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1769 CHECK_EQ(value, 1);
1770
1771 params = ZSTD_getParams(12, 0, 0);
1772 params.fParams.contentSizeFlag = 0;
1773 params.fParams.checksumFlag = 1;
1774 params.fParams.noDictIDFlag = 1;
1775 CHECK_Z(ZSTD_CCtx_setParams(cctx, params));
1776
1777 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1778 CHECK_EQ(value, (int)params.cParams.windowLog);
1779 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1780 CHECK_EQ(value, (int)params.cParams.chainLog);
1781 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1782 CHECK_EQ(value, (int)params.cParams.hashLog);
1783 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1784 CHECK_EQ(value, (int)params.cParams.searchLog);
1785 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1786 CHECK_EQ(value, (int)params.cParams.minMatch);
1787 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1788 CHECK_EQ(value, (int)params.cParams.targetLength);
1789 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1790 CHECK_EQ(value, (int)params.cParams.strategy);
1791 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1792 CHECK_EQ(value, params.fParams.contentSizeFlag);
1793 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1794 CHECK_EQ(value, params.fParams.checksumFlag);
1795 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1796 CHECK_EQ(value, !params.fParams.noDictIDFlag);
1797
1798 ZSTD_freeCCtx(cctx);
1799 }
1800
1801 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
1802 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1803 ZSTD_outBuffer out = {NULL, 0, 0};
1804 ZSTD_inBuffer in = {NULL, 0, 0};
1805 int value;
1806
1807 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
1808 * we should not modify the actual parameter specified by the user within the CCtx
1809 */
1810 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
1811 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
1812
1813 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1814 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
1815 CHECK_EQ(value, 0);
1816
1817 ZSTD_freeCCtx(cctx);
1818 }
1819 DISPLAYLEVEL(3, "OK \n");
1820
1821 /* this test is really too long, and should be made faster */
1822 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
1823 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1824 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1825 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
1826 size_t cnb;
1827 assert(cctx != NULL);
1828 params.fParams.contentSizeFlag = 0;
1829 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
1830 for (cnb = 0; cnb < nbCompressions; ++cnb) {
1831 DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
1832 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */
1833 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
1834 }
1835 ZSTD_freeCCtx(cctx);
1836 }
1837 DISPLAYLEVEL(3, "OK \n");
1838
1839 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
1840 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
1841 assert(largeCCtx != NULL);
1842 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
1843 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
1844 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
1845 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
1846 assert(smallCCtx != NULL);
1847 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1848 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
1849 DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
1850 largeCCtxSize>>10, smallCCtxSize>>10);
1851 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
1852 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
1853 }
1854 ZSTD_freeCCtx(smallCCtx);
1855 }
1856 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
1857 * currently defined as 128x, but could be adjusted in the future.
1858 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
1859 unsigned u;
1860 for (u=0; u<maxNbAttempts; u++) {
1861 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1862 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
1863 }
1864 DISPLAYLEVEL(5, "size down after %u attempts : ", u);
1865 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
1866 }
1867 }
1868 ZSTD_freeCCtx(largeCCtx);
1869 }
1870 DISPLAYLEVEL(3, "OK \n");
1871
1872 /* Static CCtx tests */
1873 #define STATIC_CCTX_LEVEL 4
1874 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1875 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
1876 void* const staticCCtxBuffer = malloc(staticCStreamSize);
1877 size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
1878 void* const staticDCtxBuffer = malloc(staticDCtxSize);
1879 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
1880 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
1881 free(staticCCtxBuffer);
1882 free(staticDCtxBuffer);
1883 DISPLAY("Not enough memory, aborting\n");
1884 testResult = 1;
1885 goto _end;
1886 }
1887 { size_t const smallInSize = 32 KB;
1888 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
1889 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
1890 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
1891 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
1892 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
1893 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
1894 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
1895 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
1896 DISPLAYLEVEL(3, "OK \n");
1897
1898 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
1899 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1900 compressedBuffer, compressedBufferSize,
1901 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
1902 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1903 (unsigned)cSize, (double)cSize/smallInSize*100);
1904
1905 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
1906 { size_t const r = ZSTD_compressCCtx(staticCCtx,
1907 compressedBuffer, compressedBufferSize,
1908 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
1909 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
1910 }
1911 DISPLAYLEVEL(3, "OK \n");
1912
1913 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
1914 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
1915 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
1916 if (staticCCtx == NULL) goto _output_error;
1917 DISPLAYLEVEL(3, "OK \n");
1918
1919 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
1920 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1921 compressedBuffer, compressedBufferSize,
1922 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1923 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1924 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1925
1926 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
1927 { int nbc;
1928 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */
1929 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
1930 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
1931 CHECK_Z(ZSTD_compressCCtx(staticCCtx,
1932 compressedBuffer, compressedBufferSize,
1933 (char*)CNBuffer + nbc, smallInSize,
1934 STATIC_CCTX_LEVEL) );
1935 } }
1936 DISPLAYLEVEL(3, "OK \n")
1937
1938 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1939 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
1940 DISPLAYLEVEL(3, "OK \n");
1941
1942 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
1943 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1944 compressedBuffer, compressedBufferSize,
1945 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1946 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1947 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1948
1949 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
1950 { size_t const r = ZSTD_decompressDCtx(staticDCtx,
1951 decodedBuffer, CNBuffSize,
1952 compressedBuffer, cSize);
1953 if (r != CNBuffSize) goto _output_error; }
1954 DISPLAYLEVEL(3, "OK \n");
1955
1956 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1957 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
1958 DISPLAYLEVEL(3, "OK \n");
1959
1960 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
1961 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
1962 if (!ZSTD_isError(r)) goto _output_error; }
1963 DISPLAYLEVEL(3, "OK \n");
1964
1965 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
1966 CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
1967 DISPLAYLEVEL(3, "OK \n");
1968
1969 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
1970 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1971 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1972 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1973 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
1974 if (!ZSTD_isError(r)) goto _output_error;
1975 }
1976 DISPLAYLEVEL(3, "OK \n");
1977
1978 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
1979 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
1980 assert(staticCCtx != NULL);
1981 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1982 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1983 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1984 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1985 }
1986 DISPLAYLEVEL(3, "OK \n");
1987
1988 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
1989 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
1990 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1991 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1992 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1993 }
1994 DISPLAYLEVEL(3, "OK \n");
1995
1996 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
1997 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
1998 if (!ZSTD_isError(r)) goto _output_error; }
1999 DISPLAYLEVEL(3, "OK \n");
2000
2001 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++);
2002 CHECK_Z( ZSTD_initDStream(staticDCtx) );
2003 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
2004 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
2005 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
2006 if (!ZSTD_isError(r)) goto _output_error;
2007 }
2008 DISPLAYLEVEL(3, "OK \n");
2009
2010 DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
2011 {
2012 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2013 {
2014 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2015 size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2016 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2017 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2018 compressedBuffer, compressedBufferSize,
2019 CNBuffer, CNBuffSize, 3));
2020
2021 {
2022 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2023 decodedBuffer, CNBuffSize,
2024 compressedBuffer, cSize);
2025 if (r != CNBuffSize) goto _output_error;
2026 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2027 }
2028 ZSTD_freeCCtxParams(params);
2029 }
2030
2031 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2032 {
2033 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2034 size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2035 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2036 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2037 compressedBuffer, compressedBufferSize,
2038 CNBuffer, CNBuffSize, 3) );
2039
2040 {
2041 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2042 decodedBuffer, CNBuffSize,
2043 compressedBuffer, cSize);
2044 if (r != CNBuffSize) goto _output_error;
2045 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2046 }
2047 ZSTD_freeCCtxParams(params);
2048 }
2049 }
2050 DISPLAYLEVEL(3, "OK \n");
2051
2052 DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
2053 {
2054 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2055 {
2056 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2057 size_t cctxSizeDefault;
2058 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2059 cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2060 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2061 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2062 compressedBuffer, compressedBufferSize,
2063 CNBuffer, CNBuffSize, 3) );
2064
2065 {
2066 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2067 decodedBuffer, CNBuffSize,
2068 compressedBuffer, cSize);
2069 if (r != CNBuffSize) goto _output_error;
2070 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2071 }
2072 ZSTD_freeCCtxParams(params);
2073 }
2074
2075 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2076 {
2077 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2078 size_t cctxSizeDefault;
2079 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2080 cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2081 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2082 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2083 compressedBuffer, compressedBufferSize,
2084 CNBuffer, CNBuffSize, 3) );
2085
2086 {
2087 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2088 decodedBuffer, CNBuffSize,
2089 compressedBuffer, cSize);
2090 if (r != CNBuffSize) goto _output_error;
2091 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2092 }
2093 ZSTD_freeCCtxParams(params);
2094 }
2095 }
2096 DISPLAYLEVEL(3, "OK \n");
2097 }
2098 free(staticCCtxBuffer);
2099 free(staticDCtxBuffer);
2100 }
2101
2102 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++);
2103 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1);
2104 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1);
2105 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1);
2106 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1);
2107
2108 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error;
2109 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error;
2110 }
2111 DISPLAYLEVEL(3, "OK \n");
2112
2113
2114 /* ZSTDMT simple MT compression test */
2115 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
2116 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx();
2117 if (mtctx==NULL) {
2118 DISPLAY("mtctx : not enough memory, aborting \n");
2119 testResult = 1;
2120 goto _end;
2121 }
2122 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
2123 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
2124 DISPLAYLEVEL(3, "OK \n");
2125
2126 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
2127 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2128 compressedBuffer, compressedBufferSize,
2129 CNBuffer, CNBuffSize) );
2130 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2131
2132 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
2133 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2134 if (rSize != CNBuffSize) {
2135 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize);
2136 goto _output_error;
2137 } }
2138 DISPLAYLEVEL(3, "OK \n");
2139
2140 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2141 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2142 if (r != CNBuffSize) goto _output_error; }
2143 DISPLAYLEVEL(3, "OK \n");
2144
2145 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2146 { size_t u;
2147 for (u=0; u<CNBuffSize; u++) {
2148 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
2149 } }
2150 DISPLAYLEVEL(3, "OK \n");
2151
2152 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
2153 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
2154 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
2155 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
2156 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2157 compressedBuffer, compressedBufferSize,
2158 CNBuffer, CNBuffSize) );
2159 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2160
2161 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2162 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2163 if (r != CNBuffSize) goto _output_error; }
2164 DISPLAYLEVEL(3, "OK \n");
2165
2166 ZSTD_freeCCtx(mtctx);
2167 }
2168
2169 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
2170 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2171 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2172 char out[32];
2173 if (cctx == NULL || dctx == NULL) goto _output_error;
2174 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
2175 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
2176 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
2177
2178 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
2179 { char const* outPtr = out;
2180 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
2181 ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
2182 size_t dSize;
2183 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
2184 if (dSize != 0) goto _output_error;
2185 }
2186
2187 ZSTD_freeDCtx(dctx);
2188 ZSTD_freeCCtx(cctx);
2189 }
2190
2191 DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
2192 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2193 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
2194 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2195 CHECK_Z(cSize);
2196 ZSTD_freeCCtx(cctx);
2197 }
2198 DISPLAYLEVEL(3, "OK \n");
2199
2200 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
2201 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2202 size_t cSize1, cSize2;
2203 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2204 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
2205 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2206 CHECK_Z(cSize1);
2207 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
2208 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2209 CHECK_Z(cSize2);
2210 CHECK_LT(cSize1, cSize2);
2211 ZSTD_freeCCtx(cctx);
2212 }
2213 DISPLAYLEVEL(3, "OK \n");
2214
2215 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
2216 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2217 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
2218 * ZSTDMT is forced to not take the shortcut.
2219 */
2220 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2221 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
2222 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
2223 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
2224 ZSTD_freeCCtx(cctx);
2225 }
2226 DISPLAYLEVEL(3, "OK \n");
2227
2228 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
2229 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2230 int const jobSize = 512 KB;
2231 int value;
2232 /* Check that the overlap log and job size are unset. */
2233 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2234 CHECK_EQ(value, 0);
2235 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2236 CHECK_EQ(value, 0);
2237 /* Set and check the overlap log and job size. */
2238 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
2239 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
2240 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2241 CHECK_EQ(value, 5);
2242 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2243 CHECK_EQ(value, jobSize);
2244 /* Set the number of workers and check the overlap log and job size. */
2245 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
2246 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2247 CHECK_EQ(value, 5);
2248 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2249 CHECK_EQ(value, jobSize);
2250 ZSTD_freeCCtxParams(params);
2251 }
2252 DISPLAYLEVEL(3, "OK \n");
2253
2254 /* Simple API multiframe test */
2255 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
2256 { size_t off = 0;
2257 int i;
2258 int const segs = 4;
2259 /* only use the first half so we don't push against size limit of compressedBuffer */
2260 size_t const segSize = (CNBuffSize / 2) / segs;
2261
2262 const U32 skipLen = 129 KB;
2263 char* const skipBuff = (char*)malloc(skipLen);
2264 assert(skipBuff != NULL);
2265 memset(skipBuff, 0, skipLen);
2266 for (i = 0; i < segs; i++) {
2267 CHECK_NEWV(r, ZSTD_compress(
2268 (BYTE*)compressedBuffer + off, CNBuffSize - off,
2269 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
2270 5) );
2271 off += r;
2272 if (i == segs/2) {
2273 /* insert skippable frame */
2274 size_t const skippableSize =
2275 ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize,
2276 skipBuff, skipLen, seed % 15);
2277 CHECK_Z(skippableSize);
2278 off += skippableSize;
2279 }
2280 }
2281 cSize = off;
2282 free(skipBuff);
2283 }
2284 DISPLAYLEVEL(3, "OK \n");
2285
2286 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
2287 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
2288 if (r != CNBuffSize / 2) goto _output_error; }
2289 DISPLAYLEVEL(3, "OK \n");
2290
2291 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
2292 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
2293 if (bound != CNBuffSize / 2) goto _output_error; }
2294 DISPLAYLEVEL(3, "OK \n");
2295
2296 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
2297 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
2298 if (r != CNBuffSize / 2) goto _output_error; }
2299 DISPLAYLEVEL(3, "OK \n");
2300
2301 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2302 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
2303 DISPLAYLEVEL(3, "OK \n");
2304
2305 /* Simple API skippable frame test */
2306 DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
2307 { U32 i;
2308 unsigned readMagic;
2309 unsigned long long receivedSize;
2310 size_t skippableSize;
2311 const U32 skipLen = 129 KB;
2312 char* const skipBuff = (char*)malloc(skipLen);
2313 assert(skipBuff != NULL);
2314 for (i = 0; i < skipLen; i++)
2315 skipBuff[i] = (char) ((seed + i) % 256);
2316 skippableSize = ZSTD_writeSkippableFrame(
2317 compressedBuffer, compressedBufferSize,
2318 skipBuff, skipLen, seed % 15);
2319 CHECK_Z(skippableSize);
2320 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2321 receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
2322 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2323 CHECK_EQ(seed % 15, readMagic);
2324 if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error;
2325
2326 free(skipBuff);
2327 }
2328 DISPLAYLEVEL(3, "OK \n");
2329
2330 DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++);
2331 {
2332 unsigned readMagic;
2333 unsigned long long receivedSize;
2334 size_t skippableSize;
2335 skippableSize = ZSTD_writeSkippableFrame(
2336 compressedBuffer, compressedBufferSize,
2337 CNBuffer, 0, seed % 15);
2338 CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
2339 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2340 receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
2341 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2342 CHECK_EQ(seed % 15, readMagic);
2343 }
2344 DISPLAYLEVEL(3, "OK \n");
2345
2346 /* Dictionary and CCtx Duplication tests */
2347 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
2348 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
2349 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2350 static const size_t dictSize = 551;
2351 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
2352
2353 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
2354 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
2355 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
2356 DISPLAYLEVEL(3, "OK \n");
2357
2358 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
2359 CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
2360 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
2361 DISPLAYLEVEL(3, "OK \n");
2362
2363 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
2364 cSize = 0;
2365 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
2366 compressedBuffer, compressedBufferSize,
2367 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2368 cSize += r);
2369 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2370
2371 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
2372 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2373 decodedBuffer, CNBuffSize,
2374 compressedBuffer, cSize,
2375 CNBuffer, dictSize),
2376 if (r != CNBuffSize - dictSize) goto _output_error);
2377 DISPLAYLEVEL(3, "OK \n");
2378
2379 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
2380 { size_t const cSizeOrig = cSize;
2381 cSize = 0;
2382 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
2383 compressedBuffer, compressedBufferSize,
2384 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2385 cSize += r);
2386 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
2387 }
2388 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2389
2390 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
2391 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2392 decodedBuffer, CNBuffSize,
2393 compressedBuffer, cSize,
2394 CNBuffer, dictSize),
2395 if (r != CNBuffSize - dictSize) goto _output_error);
2396 DISPLAYLEVEL(3, "OK \n");
2397
2398 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
2399 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
2400 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2401 if (r != CNBuffSize - dictSize) goto _output_error;
2402 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict));
2403 ZSTD_freeDDict(ddict);
2404 }
2405
2406 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
2407 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
2408 void* const ddictBuffer = malloc(ddictBufferSize);
2409 if (ddictBuffer == NULL) goto _output_error;
2410 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
2411 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2412 if (r != CNBuffSize - dictSize) goto _output_error;
2413 }
2414 free(ddictBuffer);
2415 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize);
2416 }
2417
2418 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
2419 { size_t const testSize = CNBuffSize / 3;
2420 CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
2421 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
2422
2423 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
2424 (const char*)CNBuffer + dictSize, testSize) );
2425 { ZSTD_frameHeader zfh;
2426 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
2427 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
2428 } }
2429 DISPLAYLEVEL(3, "OK \n");
2430
2431 #if !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \
2432 && !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
2433 && !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \
2434 && !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
2435 && !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
2436 && !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
2437 && !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
2438 && !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
2439 /* Note : these tests should be replaced by proper regression tests,
2440 * but existing ones do not focus on small data + dictionary + all levels.
2441 */
2442 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
2443 size_t const flatdictSize = 22 KB;
2444 size_t const contentSize = 9 KB;
2445 const void* const dict = (const char*)CNBuffer;
2446 const void* const contentStart = (const char*)dict + flatdictSize;
2447 /* These upper bounds are generally within a few bytes of the compressed size */
2448 size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
2449 3770, 3770, 3770, 3750, 3750,
2450 3742, 3675, 3674, 3665, 3664,
2451 3663, 3662, 3661, 3660, 3660,
2452 3660, 3660, 3660 };
2453 size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940,
2454 2950, 2950, 2925, 2900, 2892,
2455 2910, 2910, 2910, 2780, 2775,
2456 2765, 2760, 2755, 2754, 2753,
2457 2753, 2753, 2753 };
2458 int l = 1;
2459 int const maxLevel = ZSTD_maxCLevel();
2460 /* clevels with strategies that support rowhash on small inputs */
2461 int rowLevel = 4;
2462 int const rowLevelEnd = 8;
2463
2464 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
2465 assert(maxLevel == 22);
2466 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
2467 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n",
2468 (unsigned long long)XXH64(contentStart, contentSize, 0),
2469 (unsigned long long)XXH64(dict, flatdictSize, 0));
2470
2471 for ( ; l <= maxLevel; l++) {
2472 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
2473 contentStart, contentSize, l);
2474 if (nodict_cSize > target_nodict_cSize[l]) {
2475 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
2476 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2477 goto _output_error;
2478 }
2479 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
2480 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2481 }
2482 for ( l=1 ; l <= maxLevel; l++) {
2483 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
2484 compressedBuffer, compressedBufferSize,
2485 contentStart, contentSize,
2486 dict, flatdictSize,
2487 l);
2488 if (wdict_cSize > target_wdict_cSize[l]) {
2489 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
2490 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2491 goto _output_error;
2492 }
2493 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
2494 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2495 }
2496 /* Compression with ZSTD_compress2 and row match finder force enabled.
2497 * Give some slack for force-enabled row matchfinder since we're on a small input (9KB)
2498 */
2499 for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5;
2500 for (l=1 ; l <= maxLevel; l++) {
2501 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2502 size_t nodict_cSize;
2503 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l);
2504 ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable);
2505 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize,
2506 contentStart, contentSize);
2507 if (nodict_cSize > target_nodict_cSize[l]) {
2508 DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n",
2509 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2510 ZSTD_freeCCtx(cctx);
2511 goto _output_error;
2512 }
2513 DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n",
2514 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2515 ZSTD_freeCCtx(cctx);
2516 }
2517 /* Dict compression with DMS */
2518 for ( l=1 ; l <= maxLevel; l++) {
2519 size_t wdict_cSize;
2520 CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) );
2521 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) );
2522 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) );
2523 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) );
2524 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) );
2525 wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize);
2526 if (wdict_cSize > target_wdict_cSize[l]) {
2527 DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n",
2528 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2529 goto _output_error;
2530 }
2531 DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n",
2532 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2533 }
2534
2535 DISPLAYLEVEL(4, "compression efficiency tests OK \n");
2536 }
2537 #endif
2538
2539 ZSTD_freeCCtx(ctxOrig);
2540 ZSTD_freeCCtx(ctxDuplicated);
2541 ZSTD_freeDCtx(dctx);
2542 }
2543
2544 /* Dictionary and dictBuilder tests */
2545 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2546 size_t const dictBufferCapacity = 16 KB;
2547 void* const dictBuffer = malloc(dictBufferCapacity);
2548 size_t const totalSampleSize = 1 MB;
2549 size_t const sampleUnitSize = 8 KB;
2550 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2551 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2552 size_t dictSize;
2553 U32 dictID;
2554 size_t dictHeaderSize;
2555 size_t dictBufferFixedSize = 144;
2556 unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
2557 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2558 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
2559 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
2560 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2561 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2562 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
2563 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
2564 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
2565 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
2566 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
2567 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69};
2568
2569 if (dictBuffer==NULL || samplesSizes==NULL) {
2570 free(dictBuffer);
2571 free(samplesSizes);
2572 goto _output_error;
2573 }
2574
2575 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
2576 assert(compressedBufferSize >= totalSampleSize);
2577 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }
2578 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2579 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2580 decodedBuffer, samplesSizes, nbSamples);
2581 if (ZDICT_isError(sDictSize)) goto _output_error;
2582 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize);
2583 }
2584
2585 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
2586 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2587 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2588 CNBuffer, samplesSizes, nbSamples);
2589 if (ZDICT_isError(dictSize)) goto _output_error;
2590 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2591
2592 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++);
2593 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2594 { ZDICT_cover_params_t coverParams;
2595 memset(&coverParams, 0, sizeof(coverParams));
2596 coverParams.steps = 8;
2597 coverParams.nbThreads = 4;
2598 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2599 dictBuffer, dictBufferCapacity,
2600 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2601 &coverParams);
2602 if (ZDICT_isError(dictSize)) goto _output_error;
2603 }
2604 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2605
2606 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++);
2607 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2608 { ZDICT_cover_params_t coverParams;
2609 memset(&coverParams, 0, sizeof(coverParams));
2610 coverParams.steps = 8;
2611 coverParams.nbThreads = 4;
2612 coverParams.shrinkDict = 1;
2613 coverParams.shrinkDictMaxRegression = 1;
2614 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2615 dictBuffer, dictBufferCapacity,
2616 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2617 &coverParams);
2618 if (ZDICT_isError(dictSize)) goto _output_error;
2619 }
2620 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2621
2622 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++);
2623 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2624 { ZDICT_fastCover_params_t fastCoverParams;
2625 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2626 fastCoverParams.steps = 8;
2627 fastCoverParams.nbThreads = 4;
2628 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2629 dictBuffer, dictBufferCapacity,
2630 CNBuffer, samplesSizes, nbSamples,
2631 &fastCoverParams);
2632 if (ZDICT_isError(dictSize)) goto _output_error;
2633 }
2634 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2635
2636 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++);
2637 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2638 { ZDICT_fastCover_params_t fastCoverParams;
2639 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2640 fastCoverParams.steps = 8;
2641 fastCoverParams.nbThreads = 4;
2642 fastCoverParams.shrinkDict = 1;
2643 fastCoverParams.shrinkDictMaxRegression = 1;
2644 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2645 dictBuffer, dictBufferCapacity,
2646 CNBuffer, samplesSizes, nbSamples,
2647 &fastCoverParams);
2648 if (ZDICT_isError(dictSize)) goto _output_error;
2649 }
2650 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2651
2652 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2653 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2654 if (dictID==0) goto _output_error;
2655 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2656
2657 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
2658 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
2659 if (dictHeaderSize==0) goto _output_error;
2660 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2661
2662 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
2663 { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize);
2664 if (dictHeaderSize != 115) goto _output_error;
2665 }
2666 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2667
2668 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
2669 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
2670 CNBuffer, CNBuffSize,
2671 dictBuffer, dictSize, 4);
2672 if (ZSTD_isError(cSize)) goto _output_error;
2673 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2674
2675 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
2676 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
2677 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2678 }
2679 DISPLAYLEVEL(3, "OK \n");
2680
2681 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2682 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2683 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2684 }
2685 DISPLAYLEVEL(3, "OK \n");
2686
2687 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2688 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2689 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2690 decodedBuffer, CNBuffSize,
2691 compressedBuffer, cSize,
2692 dictBuffer, dictSize),
2693 if (r != CNBuffSize) goto _output_error);
2694 ZSTD_freeDCtx(dctx);
2695 }
2696 DISPLAYLEVEL(3, "OK \n");
2697
2698 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
2699 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2700 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
2701 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize);
2702 }
2703
2704 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
2705 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2706 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
2707 ZSTD_dlm_byRef, ZSTD_dct_auto,
2708 cParams, ZSTD_defaultCMem);
2709 assert(cdict != NULL);
2710 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict));
2711 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict));
2712 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
2713 CNBuffer, CNBuffSize, cdict);
2714 ZSTD_freeCDict(cdict);
2715 if (ZSTD_isError(cSize)) goto _output_error;
2716 }
2717 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2718
2719 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2720 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2721 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2722 }
2723 DISPLAYLEVEL(3, "OK \n");
2724
2725 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2726 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2727 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2728 decodedBuffer, CNBuffSize,
2729 compressedBuffer, cSize,
2730 dictBuffer, dictSize),
2731 if (r != CNBuffSize) goto _output_error);
2732 ZSTD_freeDCtx(dctx);
2733 }
2734 DISPLAYLEVEL(3, "OK \n");
2735
2736 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
2737 { int const maxLevel = ZSTD_maxCLevel();
2738 int level;
2739 for (level = 1; level <= maxLevel; ++level) {
2740 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize);
2741 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
2742 void* const cdictBuffer = malloc(cdictSize);
2743 if (cdictBuffer==NULL) goto _output_error;
2744 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
2745 cdictBuffer, cdictSize,
2746 dictBuffer, dictSize,
2747 ZSTD_dlm_byCopy, ZSTD_dct_auto,
2748 cParams);
2749 if (cdict == NULL) {
2750 DISPLAY("ZSTD_initStaticCDict failed ");
2751 goto _output_error;
2752 }
2753 cSize = ZSTD_compress_usingCDict(cctx,
2754 compressedBuffer, compressedBufferSize,
2755 CNBuffer, MIN(10 KB, CNBuffSize), cdict);
2756 if (ZSTD_isError(cSize)) {
2757 DISPLAY("ZSTD_compress_usingCDict failed ");
2758 goto _output_error;
2759 } }
2760 free(cdictBuffer);
2761 } }
2762 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2763
2764 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
2765 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
2766 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2767 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
2768 assert(cdict != NULL);
2769 cSize = ZSTD_compress_usingCDict_advanced(cctx,
2770 compressedBuffer, compressedBufferSize,
2771 CNBuffer, CNBuffSize,
2772 cdict, fParams);
2773 ZSTD_freeCDict(cdict);
2774 if (ZSTD_isError(cSize)) goto _output_error;
2775 }
2776 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2777
2778 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
2779 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2780 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
2781 }
2782 DISPLAYLEVEL(3, "OK (unknown)\n");
2783
2784 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2785 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2786 assert(dctx != NULL);
2787 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2788 decodedBuffer, CNBuffSize,
2789 compressedBuffer, cSize,
2790 dictBuffer, dictSize),
2791 if (r != CNBuffSize) goto _output_error);
2792 ZSTD_freeDCtx(dctx);
2793 }
2794 DISPLAYLEVEL(3, "OK \n");
2795
2796 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
2797 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
2798 p.fParams.noDictIDFlag = 1;
2799 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
2800 CNBuffer, CNBuffSize,
2801 dictBuffer, dictSize, p);
2802 if (ZSTD_isError(cSize)) goto _output_error;
2803 }
2804 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2805
2806 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2807 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2808 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2809 decodedBuffer, CNBuffSize,
2810 compressedBuffer, cSize,
2811 dictBuffer, dictSize),
2812 if (r != CNBuffSize) goto _output_error);
2813 ZSTD_freeDCtx(dctx);
2814 }
2815 DISPLAYLEVEL(3, "OK \n");
2816
2817 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
2818 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2819 assert(dctx != NULL);
2820 { const size_t ret = ZSTD_decompress_usingDict(
2821 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
2822 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
2823 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
2824 goto _output_error;
2825 }
2826 ZSTD_freeDCtx(dctx);
2827 }
2828 DISPLAYLEVEL(3, "OK \n");
2829
2830 DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
2831 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2832 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2833 ZSTD_frameParameters const fParams = { 0, 1, 0 };
2834 size_t cBlockSize;
2835 cSize = 0;
2836 CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
2837 cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
2838 CHECK_Z(cBlockSize);
2839 cSize += cBlockSize;
2840 cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
2841 CHECK_Z(cBlockSize);
2842 cSize += cBlockSize;
2843
2844 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
2845
2846 ZSTD_freeCDict(cdict);
2847 ZSTD_freeDCtx(dctx);
2848 }
2849 DISPLAYLEVEL(3, "OK \n");
2850
2851 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
2852 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2853 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2854 if (cdict==NULL) goto _output_error;
2855 ZSTD_freeCDict(cdict);
2856 }
2857 DISPLAYLEVEL(3, "OK \n");
2858
2859 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
2860 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2861 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2862 if (cdict!=NULL) goto _output_error;
2863 ZSTD_freeCDict(cdict);
2864 }
2865 DISPLAYLEVEL(3, "OK \n");
2866
2867 { char* rawDictBuffer = (char*)malloc(dictSize);
2868 assert(rawDictBuffer);
2869 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
2870 memset(rawDictBuffer + dictSize - 2, 0, 2);
2871 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
2872
2873 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
2874 {
2875 size_t ret;
2876 /* Either operation is allowed to fail, but one must fail. */
2877 ret = ZSTD_CCtx_loadDictionary_advanced(
2878 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
2879 if (!ZSTD_isError(ret)) {
2880 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2881 if (!ZSTD_isError(ret)) goto _output_error;
2882 }
2883 }
2884 DISPLAYLEVEL(3, "OK \n");
2885
2886 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
2887 {
2888 size_t ret;
2889 ret = ZSTD_CCtx_loadDictionary_advanced(
2890 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
2891 if (ZSTD_isError(ret)) goto _output_error;
2892 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2893 if (ZSTD_isError(ret)) goto _output_error;
2894 }
2895 DISPLAYLEVEL(3, "OK \n");
2896
2897 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
2898 { size_t const srcSize = MIN(CNBuffSize, 100);
2899 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2900 /* Force the dictionary to be reloaded in raw content mode */
2901 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
2902 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
2903 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
2904 CHECK_Z(cSize);
2905 }
2906 DISPLAYLEVEL(3, "OK \n");
2907
2908 free(rawDictBuffer);
2909 }
2910
2911 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
2912 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
2913 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2914 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2915 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2916 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2917 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2918 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2919 ZSTD_freeCDict(cdict);
2920 }
2921 DISPLAYLEVEL(3, "OK \n");
2922
2923 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
2924 {
2925 size_t size1, size2;
2926 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2927 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2928 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2929 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2930 if (ZSTD_isError(size1)) goto _output_error;
2931
2932 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2933 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2934 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2935 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2936 if (ZSTD_isError(size2)) goto _output_error;
2937
2938 if (size1 != size2) goto _output_error;
2939 }
2940 DISPLAYLEVEL(3, "OK \n");
2941
2942 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
2943 {
2944 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2945 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2946 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2947 }
2948 DISPLAYLEVEL(3, "OK \n");
2949
2950 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
2951 {
2952 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2953 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2954 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2955 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2956 ZSTD_freeCDict(cdict);
2957 }
2958 DISPLAYLEVEL(3, "OK \n");
2959
2960 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
2961 {
2962 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2963 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2964 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2965 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2966 ZSTD_freeCDict(cdict);
2967 }
2968 DISPLAYLEVEL(3, "OK \n");
2969
2970 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
2971 {
2972 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2973 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2974 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2975 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2976 ZSTD_freeCDict(cdict);
2977 }
2978 DISPLAYLEVEL(3, "OK \n");
2979
2980 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
2981 {
2982 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2983 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2984 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2985 }
2986 DISPLAYLEVEL(3, "OK \n");
2987
2988 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
2989 {
2990 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2991 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2992 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2993 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2994 ZSTD_freeCDict(cdict);
2995 }
2996 DISPLAYLEVEL(3, "OK \n");
2997
2998 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
2999 {
3000 size_t size1, size2;
3001 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3002 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
3003 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3004 if (ZSTD_isError(size1)) goto _output_error;
3005
3006 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
3007 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3008 if (ZSTD_isError(size2)) goto _output_error;
3009
3010 if (size1 != size2) goto _output_error;
3011 }
3012 DISPLAYLEVEL(3, "OK \n");
3013
3014 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
3015 {
3016 size_t size1, size2;
3017 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3018 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
3019 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3020 if (ZSTD_isError(size1)) goto _output_error;
3021
3022 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3023 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3024 if (ZSTD_isError(size2)) goto _output_error;
3025
3026 if (size1 == size2) goto _output_error;
3027 }
3028 DISPLAYLEVEL(3, "OK \n");
3029
3030 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3031 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
3032 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3033 CHECK_Z(cSize);
3034 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
3035 {
3036 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3037 size_t ret;
3038 /* We should fail to decompress without a dictionary. */
3039 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3040 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3041 if (!ZSTD_isError(ret)) goto _output_error;
3042 /* We should succeed to decompress with the dictionary. */
3043 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3044 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
3045 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3046 /* The dictionary should persist across calls. */
3047 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3048 /* When we reset the context the dictionary is cleared. */
3049 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3050 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3051 if (!ZSTD_isError(ret)) goto _output_error;
3052 ZSTD_freeDCtx(dctx);
3053 }
3054 DISPLAYLEVEL(3, "OK \n");
3055
3056 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
3057 {
3058 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3059 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
3060 size_t ret;
3061 /* We should succeed to decompress with the ddict. */
3062 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3063 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
3064 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3065 /* The ddict should persist across calls. */
3066 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3067 /* When we reset the context the ddict is cleared. */
3068 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3069 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3070 if (!ZSTD_isError(ret)) goto _output_error;
3071 ZSTD_freeDCtx(dctx);
3072 ZSTD_freeDDict(ddict);
3073 }
3074 DISPLAYLEVEL(3, "OK \n");
3075
3076 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
3077 {
3078 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3079 size_t ret;
3080 /* We should succeed to decompress with the prefix. */
3081 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3082 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
3083 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3084 /* The prefix should be cleared after the first compression. */
3085 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3086 if (!ZSTD_isError(ret)) goto _output_error;
3087 ZSTD_freeDCtx(dctx);
3088 }
3089 DISPLAYLEVEL(3, "OK \n");
3090
3091 DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3092 {
3093 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3094 ZSTD_customMem customMem = {NULL, NULL, NULL};
3095 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3096 ZSTD_CDict* cdict;
3097 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
3098 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3099 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3100 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3101 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3102 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3103 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3104 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3105 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3106 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3107 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3108 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3109 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3110 CHECK_Z(cSize);
3111 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3112 ZSTD_freeCDict(cdict);
3113 ZSTD_freeDCtx(dctx);
3114 ZSTD_freeCCtxParams(cctxParams);
3115 }
3116 DISPLAYLEVEL(3, "OK \n");
3117
3118 DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3119 {
3120 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3121 ZSTD_customMem customMem = {NULL, NULL, NULL};
3122 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3123 ZSTD_CDict* cdict;
3124 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
3125 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3126 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3127 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3128 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3129 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3130 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3131 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3132 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3133 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3135 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3136 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3137 CHECK_Z(cSize);
3138 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3139 ZSTD_freeCDict(cdict);
3140 ZSTD_freeDCtx(dctx);
3141 ZSTD_freeCCtxParams(cctxParams);
3142 }
3143 DISPLAYLEVEL(3, "OK \n");
3144
3145 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
3146 if (MEM_64bits()) {
3147 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3148 ZSTD_customMem customMem = {NULL, NULL, NULL};
3149 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3150 ZSTD_CDict* cdict;
3151 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
3152 /* Force enable row based match finder, and disable dedicated dict search. */
3153 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
3154 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
3155 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
3156 /* Set windowLog to 29 so hash/chain logs don't get sized down */
3157 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
3158 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
3159 /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
3160 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
3161 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3162 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3163 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3164 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3165 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3166 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3167 CHECK_Z(cSize);
3168 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3169 ZSTD_freeCDict(cdict);
3170 ZSTD_freeDCtx(dctx);
3171 ZSTD_freeCCtxParams(cctxParams);
3172 }
3173 DISPLAYLEVEL(3, "OK \n");
3174
3175 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
3176 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3177 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
3178 CNBuffer, samplesSizes, nbSamples);
3179 if (ZDICT_isError(dictSize)) goto _output_error;
3180 /* Set all the repcodes to non-default */
3181 {
3182 BYTE* dictPtr = (BYTE*)dictBuffer;
3183 BYTE* dictLimit = dictPtr + dictSize - 12;
3184 /* Find the repcodes */
3185 while (dictPtr < dictLimit &&
3186 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
3187 MEM_readLE32(dictPtr + 8) != 8)) {
3188 ++dictPtr;
3189 }
3190 if (dictPtr >= dictLimit) goto _output_error;
3191 MEM_writeLE32(dictPtr + 0, 10);
3192 MEM_writeLE32(dictPtr + 4, 10);
3193 MEM_writeLE32(dictPtr + 8, 10);
3194 /* Set the last 8 bytes to 'x' */
3195 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
3196 }
3197 /* The optimal parser checks all the repcodes.
3198 * Make sure at least one is a match >= targetLength so that it is
3199 * immediately chosen. This will make sure that the compressor and
3200 * decompressor agree on at least one of the repcodes.
3201 */
3202 { size_t dSize;
3203 BYTE data[1024];
3204 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3205 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
3206 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
3207 ZSTD_dlm_byRef, ZSTD_dct_auto,
3208 cParams, ZSTD_defaultCMem);
3209 assert(dctx != NULL); assert(cdict != NULL);
3210 memset(data, 'x', sizeof(data));
3211 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
3212 data, sizeof(data), cdict);
3213 ZSTD_freeCDict(cdict);
3214 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
3215 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
3216 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
3217 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
3218 ZSTD_freeDCtx(dctx);
3219 }
3220 DISPLAYLEVEL(3, "OK \n");
3221
3222 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++);
3223 {
3224 const size_t numDicts = 128;
3225 const size_t numFrames = 4;
3226 size_t i;
3227 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3228 ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts);
3229 ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts);
3230 U32 dictIDSeed = seed;
3231 /* Create new compressed buffer that will hold frames with differing dictIDs */
3232 char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */
3233
3234 ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
3235 /* Create a bunch of DDicts with random dict IDs */
3236 for (i = 0; i < numDicts; ++i) {
3237 U32 currDictID = FUZ_rand(&dictIDSeed);
3238 MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID);
3239 ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize);
3240 cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3);
3241 if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) {
3242 goto _output_error;
3243 }
3244 }
3245 /* Compress a few frames using random CDicts */
3246 {
3247 size_t off = 0;
3248 /* only use the first half so we don't push against size limit of compressedBuffer */
3249 size_t const segSize = (CNBuffSize / 2) / numFrames;
3250 for (i = 0; i < numFrames; i++) {
3251 size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts;
3252 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3253 { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx,
3254 (BYTE*)compressedBuffer + off, CNBuffSize - off,
3255 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
3256 cdictTable[dictIdx]));
3257 off += r;
3258 }
3259 }
3260 cSize = off;
3261 }
3262
3263 /* We should succeed to decompression even though different dicts were used on different frames */
3264 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3265 ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts);
3266 /* Reference every single ddict we made */
3267 for (i = 0; i < numDicts; ++i) {
3268 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
3269 }
3270 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3271 /* Streaming decompression should also work */
3272 {
3273 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
3274 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
3275 while (in.pos < in.size) {
3276 CHECK_Z(ZSTD_decompressStream(dctx, &out, &in));
3277 }
3278 }
3279 ZSTD_freeDCtx(dctx);
3280 for (i = 0; i < numDicts; ++i) {
3281 ZSTD_freeCDict(cdictTable[i]);
3282 ZSTD_freeDDict(ddictTable[i]);
3283 }
3284 free(dictBufferMulti);
3285 free(ddictTable);
3286 free(cdictTable);
3287 }
3288 DISPLAYLEVEL(3, "OK \n");
3289
3290 ZSTD_freeCCtx(cctx);
3291 free(dictBuffer);
3292 free(samplesSizes);
3293 }
3294
3295 /* COVER dictionary builder tests */
3296 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3297 size_t dictSize = 16 KB;
3298 size_t optDictSize = dictSize;
3299 void* dictBuffer = malloc(dictSize);
3300 size_t const totalSampleSize = 1 MB;
3301 size_t const sampleUnitSize = 8 KB;
3302 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3303 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3304 U32 seed32 = seed;
3305 ZDICT_cover_params_t params;
3306 U32 dictID;
3307
3308 if (dictBuffer==NULL || samplesSizes==NULL) {
3309 free(dictBuffer);
3310 free(samplesSizes);
3311 goto _output_error;
3312 }
3313
3314 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
3315 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3316 memset(¶ms, 0, sizeof(params));
3317 params.d = 1 + (FUZ_rand(&seed32) % 16);
3318 params.k = params.d + (FUZ_rand(&seed32) % 256);
3319 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
3320 CNBuffer, samplesSizes, nbSamples,
3321 params);
3322 if (ZDICT_isError(dictSize)) goto _output_error;
3323 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
3324
3325 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3326 dictID = ZDICT_getDictID(dictBuffer, dictSize);
3327 if (dictID==0) goto _output_error;
3328 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3329
3330 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
3331 memset(¶ms, 0, sizeof(params));
3332 params.steps = 4;
3333 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
3334 CNBuffer, samplesSizes,
3335 nbSamples / 4, ¶ms);
3336 if (ZDICT_isError(optDictSize)) goto _output_error;
3337 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize);
3338
3339 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3340 dictID = ZDICT_getDictID(dictBuffer, optDictSize);
3341 if (dictID==0) goto _output_error;
3342 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3343
3344 ZSTD_freeCCtx(cctx);
3345 free(dictBuffer);
3346 free(samplesSizes);
3347 }
3348
3349 /* Decompression defense tests */
3350 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
3351 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
3352 if (!ZSTD_isError(r)) goto _output_error;
3353 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
3354 DISPLAYLEVEL(3, "OK \n");
3355
3356 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
3357 ((char*)(CNBuffer))[0] = 1;
3358 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
3359 if (!ZSTD_isError(r)) goto _output_error; }
3360 DISPLAYLEVEL(3, "OK \n");
3361
3362 /* content size verification test */
3363 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
3364 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3365 size_t const srcSize = 5000;
3366 size_t const wrongSrcSize = (srcSize + 1000);
3367 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
3368 params.fParams.contentSizeFlag = 1;
3369 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
3370 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
3371 if (!ZSTD_isError(result)) goto _output_error;
3372 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
3373 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
3374 }
3375 ZSTD_freeCCtx(cctx);
3376 }
3377
3378 /* negative compression level test : ensure simple API and advanced API produce same result */
3379 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
3380 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3381 size_t const srcSize = CNBuffSize / 5;
3382 int const compressionLevel = -1;
3383
3384 assert(cctx != NULL);
3385 { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize,
3386 CNBuffer, srcSize, compressionLevel);
3387 if (ZSTD_isError(cSize_1pass)) goto _output_error;
3388
3389 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
3390 { size_t const compressionResult = ZSTD_compress2(cctx,
3391 compressedBuffer, compressedBufferSize,
3392 CNBuffer, srcSize);
3393 DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
3394 if (ZSTD_isError(compressionResult)) goto _output_error;
3395 if (compressionResult != cSize_1pass) goto _output_error;
3396 } }
3397 ZSTD_freeCCtx(cctx);
3398 }
3399 DISPLAYLEVEL(3, "OK \n");
3400
3401 /* parameters order test */
3402 { size_t const inputSize = CNBuffSize / 2;
3403 U64 xxh64;
3404
3405 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3406 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
3407 assert(cctx != NULL);
3408 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3409 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3410 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3411 { size_t const compressedSize = ZSTD_compress2(cctx,
3412 compressedBuffer, ZSTD_compressBound(inputSize),
3413 CNBuffer, inputSize);
3414 CHECK_Z(compressedSize);
3415 cSize = compressedSize;
3416 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
3417 }
3418 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3419 ZSTD_freeCCtx(cctx);
3420 }
3421
3422 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3423 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
3424 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3425 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3426 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3427 { size_t const result = ZSTD_compress2(cctx,
3428 compressedBuffer, ZSTD_compressBound(inputSize),
3429 CNBuffer, inputSize);
3430 CHECK_Z(result);
3431 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
3432 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
3433 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
3434 }
3435 ZSTD_freeCCtx(cctx);
3436 }
3437 }
3438
3439 /* advanced parameters for decompression */
3440 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3441 assert(dctx != NULL);
3442
3443 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
3444 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
3445 CHECK_Z(bounds.error);
3446 }
3447 DISPLAYLEVEL(3, "OK \n");
3448
3449 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
3450 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
3451 if (!ZSTD_isError(sr)) goto _output_error;
3452 }
3453 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
3454 if (!ZSTD_isError(bounds.error)) goto _output_error;
3455 }
3456 DISPLAYLEVEL(3, "OK \n");
3457
3458 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
3459 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
3460 if (!ZSTD_isError(sr)) goto _output_error;
3461 }
3462 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
3463 if (!ZSTD_isError(sr)) goto _output_error;
3464 }
3465 DISPLAYLEVEL(3, "OK \n");
3466
3467 ZSTD_freeDCtx(dctx);
3468 }
3469
3470
3471 /* custom formats tests */
3472 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3473 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3474 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
3475 assert(dctx != NULL); assert(cctx != NULL);
3476
3477 /* basic block compression */
3478 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
3479 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3480 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
3481 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
3482 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3483 if (result != 0) goto _output_error;
3484 if (in.pos != in.size) goto _output_error;
3485 cSize = out.pos;
3486 }
3487 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3488
3489 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
3490 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3491 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
3492 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
3493 }
3494
3495 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
3496 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3497 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3498 { ZSTD_frameHeader zfh;
3499 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
3500 if (zfhrt != 0) goto _output_error;
3501 }
3502 /* one shot */
3503 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3504 if (result != inputSize) goto _output_error;
3505 DISPLAYLEVEL(3, "one-shot OK, ");
3506 }
3507 /* streaming */
3508 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3509 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3510 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3511 if (result != 0) goto _output_error;
3512 if (in.pos != in.size) goto _output_error;
3513 if (out.pos != inputSize) goto _output_error;
3514 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3515 }
3516
3517 /* basic block compression */
3518 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
3519 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3520 { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
3521 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
3522 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3523 if (result != 0) goto _output_error;
3524 if (in.pos != in.size) goto _output_error;
3525 cSize = out.pos;
3526 }
3527 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
3528
3529 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
3530 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3531 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3532 /* one shot */
3533 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3534 if (result != 0) goto _output_error;
3535 DISPLAYLEVEL(3, "one-shot OK, ");
3536 }
3537 /* streaming */
3538 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3539 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3540 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3541 if (result != 0) goto _output_error;
3542 if (in.pos != in.size) goto _output_error;
3543 if (out.pos != 0) goto _output_error;
3544 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3545 }
3546
3547 ZSTD_freeCCtx(cctx);
3548 ZSTD_freeDCtx(dctx);
3549 }
3550
3551 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++);
3552 {
3553 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3554 /* Attempt to future proof this to new parameters. */
3555 int const maxParam = 2000;
3556 int param;
3557 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error;
3558 for (param = 0; param < maxParam; ++param) {
3559 ZSTD_dParameter dParam = (ZSTD_dParameter)param;
3560 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam);
3561 int value1;
3562 int value2;
3563 int check;
3564 if (ZSTD_isError(bounds.error))
3565 continue;
3566 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
3567 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
3568 CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
3569 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3570 if (check != value2) goto _output_error;
3571 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
3572 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3573 if (check != value1) goto _output_error;
3574 }
3575 ZSTD_freeDCtx(dctx);
3576 }
3577 DISPLAYLEVEL(3, "OK \n");
3578
3579 /* block API tests */
3580 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3581 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3582 static const size_t dictSize = 65 KB;
3583 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
3584 size_t cSize2;
3585 assert(cctx != NULL); assert(dctx != NULL);
3586
3587 /* basic block compression */
3588 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
3589 CHECK_Z( ZSTD_compressBegin(cctx, 5) );
3590 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3591 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
3592 DISPLAYLEVEL(3, "OK \n");
3593
3594 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
3595 CHECK_Z( ZSTD_decompressBegin(dctx) );
3596 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3597 if (r != blockSize) goto _output_error; }
3598 DISPLAYLEVEL(3, "OK \n");
3599
3600 /* very long stream of block compression */
3601 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
3602 CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
3603 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3604 { U64 const toCompress = 5000000000ULL; /* > 4 GB */
3605 U64 compressed = 0;
3606 while (compressed < toCompress) {
3607 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
3608 assert(blockCSize != 0);
3609 if (ZSTD_isError(blockCSize)) goto _output_error;
3610 compressed += blockCSize;
3611 } }
3612 DISPLAYLEVEL(3, "OK \n");
3613
3614 /* dictionary block compression */
3615 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
3616 CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
3617 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
3618 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
3619 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
3620 assert(r == 0); /* non-compressible block */ }
3621 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
3622 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
3623 (char*)CNBuffer+dictSize+2*blockSize, blockSize));
3624 DISPLAYLEVEL(3, "OK \n");
3625
3626 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
3627 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
3628 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
3629 if (r != blockSize) {
3630 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3631 goto _output_error;
3632 } }
3633 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
3634 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
3635 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
3636 if (r != blockSize) {
3637 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3638 goto _output_error;
3639 } }
3640 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
3641 DISPLAYLEVEL(3, "OK \n");
3642
3643 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
3644 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
3645 if (cdict==NULL) goto _output_error;
3646 CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
3647 CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
3648 ZSTD_freeCDict(cdict);
3649 }
3650 DISPLAYLEVEL(3, "OK \n");
3651
3652 ZSTD_freeCCtx(cctx);
3653 ZSTD_freeDCtx(dctx);
3654 }
3655
3656 /* long rle test */
3657 { size_t sampleSize = 0;
3658 size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
3659 DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
3660 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
3661 sampleSize += 256 KB - 1;
3662 memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
3663 sampleSize += 96 KB;
3664 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
3665 if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
3666 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
3667 if (regenSize!=sampleSize) goto _output_error; }
3668 DISPLAYLEVEL(3, "OK \n");
3669 }
3670
3671 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++);
3672 {
3673 size_t srcSize = 150 KB;
3674 BYTE* src = (BYTE*)CNBuffer;
3675 BYTE* decoded = (BYTE*)compressedBuffer;
3676
3677 ZSTD_CCtx* cctx = ZSTD_createCCtx();
3678 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3679 size_t seqsSize;
3680
3681 if (seqs == NULL) goto _output_error;
3682 assert(cctx != NULL);
3683 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
3684 /* Populate src with random data */
3685 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3686
3687 /* Test with block delimiters roundtrip */
3688 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3689 CHECK_Z(seqsSize);
3690 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters);
3691 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3692
3693 /* Test no block delimiters roundtrip */
3694 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
3695 CHECK_Z(seqsSize);
3696 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters);
3697 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3698
3699 ZSTD_freeCCtx(cctx);
3700 free(seqs);
3701 }
3702 DISPLAYLEVEL(3, "OK \n");
3703
3704 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences too small output buffer : ", testNb++);
3705 {
3706 const size_t seqsCapacity = 10;
3707 const size_t srcSize = 150 KB;
3708 const BYTE* src = (BYTE*)CNBuffer;
3709
3710 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3711 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(seqsCapacity * sizeof(ZSTD_Sequence));
3712
3713 if (seqs == NULL) goto _output_error;
3714 if (cctx == NULL) goto _output_error;
3715 /* Populate src with random data */
3716 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3717
3718 /* Test with block delimiters roundtrip */
3719 {
3720 size_t const seqsSize = ZSTD_generateSequences(cctx, seqs, seqsCapacity, src, srcSize);
3721 if (!ZSTD_isError(seqsSize)) goto _output_error;
3722 }
3723
3724 ZSTD_freeCCtx(cctx);
3725 free(seqs);
3726 }
3727 DISPLAYLEVEL(3, "OK \n");
3728
3729 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
3730 {
3731 const size_t srcSize = 500 KB;
3732 const BYTE* const src = (BYTE*)CNBuffer;
3733 BYTE* const dst = (BYTE*)compressedBuffer;
3734 const size_t dstCapacity = ZSTD_compressBound(srcSize);
3735 const size_t decompressSize = srcSize;
3736 char* const decompressBuffer = (char*)malloc(decompressSize);
3737 size_t compressedSize;
3738
3739 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3740 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3741 size_t nbSeqs;
3742
3743 if (seqs == NULL) goto _output_error;
3744 assert(cctx != NULL);
3745
3746 /* Populate src with random data */
3747 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
3748
3749 /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
3750 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3751 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3752 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
3753 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
3754 if (ZSTD_isError(compressedSize)) {
3755 DISPLAY("Error in sequence compression with block delims\n");
3756 goto _output_error;
3757 }
3758 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3759 if (ZSTD_isError(dSize)) {
3760 DISPLAY("Error in sequence compression roundtrip with block delims\n");
3761 goto _output_error;
3762 } }
3763 assert(!memcmp(decompressBuffer, src, srcSize));
3764
3765 /* Roundtrip Test with no block delimiters */
3766 { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs);
3767 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3768 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
3769 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize);
3770 }
3771 if (ZSTD_isError(compressedSize)) {
3772 DISPLAY("Error in sequence compression with no block delims\n");
3773 goto _output_error;
3774 }
3775 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3776 if (ZSTD_isError(dSize)) {
3777 DISPLAY("Error in sequence compression roundtrip with no block delims\n");
3778 goto _output_error;
3779 } }
3780 assert(!memcmp(decompressBuffer, src, srcSize));
3781
3782 ZSTD_freeCCtx(cctx);
3783 free(decompressBuffer);
3784 free(seqs);
3785 }
3786 DISPLAYLEVEL(3, "OK \n");
3787
3788 /* Multiple blocks of zeros test */
3789 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
3790 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3791 memset(CNBuffer, 0, LONGZEROSLENGTH);
3792 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) );
3793 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100);
3794
3795 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3796 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) );
3797 if (r != LONGZEROSLENGTH) goto _output_error; }
3798 DISPLAYLEVEL(3, "OK \n");
3799
3800 /* All zeroes test (test bug #137) */
3801 #define ZEROESLENGTH 100
3802 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
3803 memset(CNBuffer, 0, ZEROESLENGTH);
3804 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
3805 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
3806
3807 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
3808 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
3809 if (r != ZEROESLENGTH) goto _output_error; }
3810 DISPLAYLEVEL(3, "OK \n");
3811
3812 /* nbSeq limit test */
3813 #define _3BYTESTESTLENGTH 131000
3814 #define NB3BYTESSEQLOG 9
3815 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
3816 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
3817 /* creates a buffer full of 3-bytes sequences */
3818 { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
3819 U32 rSeed = 1;
3820
3821 /* create batch of 3-bytes sequences */
3822 { int i;
3823 for (i=0; i < NB3BYTESSEQ; i++) {
3824 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
3825 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
3826 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
3827 } }
3828
3829 /* randomly fills CNBuffer with prepared 3-bytes sequences */
3830 { int i;
3831 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
3832 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
3833 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
3834 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
3835 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
3836 } } }
3837 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++);
3838 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3839 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3;
3840 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
3841 size_t nbSeq = 1;
3842 while (nbSeq <= maxNbSeq) {
3843 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
3844 /* Check every sequence for the first 100, then skip more rapidly. */
3845 if (nbSeq < 100) {
3846 ++nbSeq;
3847 } else {
3848 nbSeq += (nbSeq >> 2);
3849 }
3850 }
3851 ZSTD_freeCCtx(cctx);
3852 }
3853 DISPLAYLEVEL(3, "OK \n");
3854
3855 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
3856 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
3857 CNBuffer, _3BYTESTESTLENGTH, 19) );
3858 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
3859
3860 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
3861 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
3862 if (r != _3BYTESTESTLENGTH) goto _output_error; }
3863 DISPLAYLEVEL(3, "OK \n");
3864
3865
3866 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++);
3867 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
3868 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3869 size_t const bound = ZSTD_compressBound(CNBuffSize);
3870 size_t size = 1;
3871 while (size <= CNBuffSize) {
3872 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
3873 /* Check every size for the first 100, then skip more rapidly. */
3874 if (size < 100) {
3875 ++size;
3876 } else {
3877 size += (size >> 2);
3878 }
3879 }
3880 ZSTD_freeCCtx(cctx);
3881 }
3882 DISPLAYLEVEL(3, "OK \n");
3883
3884 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
3885 { /* Train a dictionary on low characters */
3886 size_t dictSize = 16 KB;
3887 void* const dictBuffer = malloc(dictSize);
3888 size_t const totalSampleSize = 1 MB;
3889 size_t const sampleUnitSize = 8 KB;
3890 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3891 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3892 if (!dictBuffer || !samplesSizes) goto _output_error;
3893 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3894 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
3895 if (ZDICT_isError(dictSize)) goto _output_error;
3896 /* Reverse the characters to make the dictionary ill suited */
3897 { U32 u;
3898 for (u = 0; u < CNBuffSize; ++u) {
3899 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
3900 } }
3901 { /* Compress the data */
3902 size_t const inputSize = 500;
3903 size_t const outputSize = ZSTD_compressBound(inputSize);
3904 void* const outputBuffer = malloc(outputSize);
3905 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3906 if (!outputBuffer || !cctx) goto _output_error;
3907 CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
3908 free(outputBuffer);
3909 ZSTD_freeCCtx(cctx);
3910 }
3911
3912 free(dictBuffer);
3913 free(samplesSizes);
3914 }
3915 DISPLAYLEVEL(3, "OK \n");
3916
3917
3918 /* findFrameCompressedSize on skippable frames */
3919 DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
3920 { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
3921 size_t const frameSrcSize = 13;
3922 if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
3923 DISPLAYLEVEL(3, "OK \n");
3924
3925 /* error string tests */
3926 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
3927 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
3928 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
3929 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
3930 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
3931 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
3932 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
3933 DISPLAYLEVEL(3, "OK \n");
3934
3935 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
3936 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
3937 {
3938 size_t const size = MIN(128 KB, CNBuffSize);
3939 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3940 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
3941 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
3942 ZSTD_frameHeader lgHeader;
3943 ZSTD_frameHeader smHeader;
3944
3945 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
3946 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
3947 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
3948 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
3949
3950 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
3951
3952 ZSTD_freeCDict(smCDict);
3953 ZSTD_freeCDict(lgCDict);
3954 ZSTD_freeCCtx(cctx);
3955 }
3956 DISPLAYLEVEL(3, "OK \n");
3957
3958 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
3959 {
3960 short norm[32];
3961 unsigned count[32];
3962 unsigned const tableLog = 5;
3963 size_t const nbSeq = 32;
3964 unsigned const maxSymbolValue = 31;
3965 size_t i;
3966
3967 for (i = 0; i < 32; ++i)
3968 count[i] = 1;
3969 /* Calling FSE_normalizeCount() on a uniform distribution should not
3970 * cause a division by zero.
3971 */
3972 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1);
3973 }
3974 DISPLAYLEVEL(3, "OK \n");
3975
3976 DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++);
3977 {
3978 size_t const outBufSize = 9;
3979 short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18};
3980 unsigned const tableLog = 5;
3981 unsigned const maxSymbolValue = 10;
3982 BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE));
3983
3984 /* Ensure that this write doesn't write out of bounds, and that
3985 * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1.
3986 */
3987 FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog);
3988 free(outBuf);
3989 }
3990 DISPLAYLEVEL(3, "OK \n");
3991
3992 DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
3993 {
3994 U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
3995 U32 rand32 = FUZ_rand(&seed_copy);
3996 U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
3997 U32 lowbit_only_32 = 1;
3998 U64 lowbit_only_64 = 1;
3999 U32 highbit_only_32 = (U32)1 << 31;
4000 U64 highbit_only_64 = (U64)1 << 63;
4001 U32 i;
4002 if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
4003 if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
4004
4005 /* Test ZSTD_countTrailingZeros32 */
4006 CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
4007 CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u);
4008 CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32));
4009
4010 /* Test ZSTD_countLeadingZeros32 */
4011 CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u);
4012 CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u);
4013 CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32));
4014
4015 /* Test ZSTD_countTrailingZeros64 */
4016 CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u);
4017 CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u);
4018
4019 /* Test ZSTD_countLeadingZeros64 */
4020 CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u);
4021 CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u);
4022
4023 /* Test ZSTD_highbit32 */
4024 CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u);
4025 CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u);
4026
4027 /* Test ZSTD_NbCommonBytes */
4028 if (MEM_isLittleEndian()) {
4029 if (MEM_64bits()) {
4030 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
4031 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
4032 } else {
4033 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
4034 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
4035 }
4036 } else {
4037 if (MEM_64bits()) {
4038 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u);
4039 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u);
4040 } else {
4041 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u);
4042 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u);
4043 }
4044 }
4045
4046 /* Test MEM_ intrinsics */
4047 CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32));
4048 CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64));
4049
4050 /* Test fallbacks vs intrinsics on a range of small integers */
4051 for (i=1; i <= 1000; i++) {
4052 CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i));
4053 CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i));
4054 CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i));
4055 CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i));
4056 }
4057 }
4058 DISPLAYLEVEL(3, "OK \n");
4059
4060 #ifdef ZSTD_MULTITHREAD
4061 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++);
4062 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4063 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4064 size_t const dstSize = ZSTD_compressBound(srcSize);
4065 void* const src = CNBuffer;
4066 void* const dst = compressedBuffer;
4067 void* dict = (void*)malloc(srcSize);
4068
4069 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4070 RDG_genBuffer(dict, srcSize, compressibility, 0., seed);
4071
4072 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4073 memset(dict, 0, sizeof(U32));
4074
4075 /* something more than 1 */
4076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4077 /* lie and claim this is a full dict */
4078 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict));
4079
4080 { ZSTD_outBuffer out = {dst, dstSize, 0};
4081 ZSTD_inBuffer in = {src, srcSize, 0};
4082 /* should fail because its not a full dict like we said it was */
4083 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)));
4084 }
4085
4086 ZSTD_freeCCtx(cctx);
4087 free(dict);
4088 }
4089 DISPLAYLEVEL(3, "OK \n");
4090
4091 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
4092 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4093 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4094 size_t const dictSize = 10;
4095 size_t const dstSize = ZSTD_compressBound(srcSize);
4096 void* const src = CNBuffer;
4097 void* const dst = compressedBuffer;
4098 void* dict = (void*)malloc(dictSize);
4099
4100 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4101 RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
4102
4103 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4104 memset(dict, 0, sizeof(U32));
4105
4106 /* Enable MT, LDM, and use refPrefix() for a small dict */
4107 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4108 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4109 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
4110
4111 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
4112
4113 ZSTD_freeCCtx(cctx);
4114 free(dict);
4115 }
4116 DISPLAYLEVEL(3, "OK \n");
4117
4118 DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++);
4119 {
4120 ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0);
4121 ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0);
4122 ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400);
4123 ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000);
4124 ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000);
4125
4126 assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict)));
4127 assert(!memcmp(&medium, &mediumDict, sizeof(medium)));
4128 assert(!memcmp(&large, &largeDict, sizeof(large)));
4129 }
4130 DISPLAYLEVEL(3, "OK \n");
4131
4132 DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++);
4133 {
4134 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0);
4135 ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400);
4136 ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400);
4137
4138 assert(smallSrcAndDict.windowLog == 10);
4139 assert(!memcmp(&cParams, &smallDict, sizeof(cParams)));
4140 }
4141 DISPLAYLEVEL(3, "OK \n");
4142
4143 DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
4144 {
4145 int level = 1;
4146 size_t prevSize = 0;
4147 for (; level < ZSTD_maxCLevel(); ++level) {
4148 size_t const currSize = ZSTD_estimateCCtxSize(level);
4149 if (prevSize > currSize) {
4150 DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
4151 prevSize, level-1, currSize, level);
4152 goto _output_error;
4153 }
4154 prevSize = currSize;
4155 }
4156 }
4157 DISPLAYLEVEL(3, "OK \n");
4158
4159 DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
4160 {
4161 size_t const kSizeIncrement = 2 KB;
4162 int level = -3;
4163
4164 for (; level <= ZSTD_maxCLevel(); ++level) {
4165 size_t dictSize = 0;
4166 for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
4167 size_t srcSize = 2 KB;
4168 for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
4169 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
4170 size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
4171 size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
4172 if (cctxSizeUsingLevel < cctxSizeUsingCParams
4173 || ZSTD_isError(cctxSizeUsingCParams)
4174 || ZSTD_isError(cctxSizeUsingLevel)) {
4175 DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
4176 level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
4177 goto _output_error;
4178 } } } } }
4179 DISPLAYLEVEL(3, "OK \n");
4180
4181 DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++)
4182 {
4183 int const threadPoolTestResult = threadPoolTests();
4184 if (threadPoolTestResult) {
4185 goto _output_error;
4186 }
4187 }
4188 DISPLAYLEVEL(3, "thread pool tests OK \n");
4189
4190 #endif /* ZSTD_MULTITHREAD */
4191
4192 _end:
4193 free(CNBuffer);
4194 free(compressedBuffer);
4195 free(decodedBuffer);
4196 return testResult;
4197
4198 _output_error:
4199 testResult = 1;
4200 DISPLAY("Error detected in Unit tests ! \n");
4201 goto _end;
4202 }
4203
longUnitTests(U32 const seed,double compressibility)4204 static int longUnitTests(U32 const seed, double compressibility)
4205 {
4206 size_t const CNBuffSize = 5 MB;
4207 void* const CNBuffer = malloc(CNBuffSize);
4208 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
4209 void* const compressedBuffer = malloc(compressedBufferSize);
4210 void* const decodedBuffer = malloc(CNBuffSize);
4211 int testResult = 0;
4212 unsigned testNb=0;
4213 size_t cSize;
4214
4215 /* Create compressible noise */
4216 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
4217 DISPLAY("Not enough memory, aborting\n");
4218 testResult = 1;
4219 goto _end;
4220 }
4221 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
4222
4223 /* note : this test is rather long, it would be great to find a way to speed up its execution */
4224 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++);
4225 { int cLevel;
4226 size_t approxIndex = 0;
4227 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
4228
4229 /* Provision enough space in a static context so that we can do all
4230 * this without ever reallocating, which would reset the indices. */
4231 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
4232 void* const staticCCtxBuffer = malloc(staticCCtxSize);
4233 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
4234
4235 /* bump the indices so the following compressions happen at high
4236 * indices. */
4237 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4238 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4239 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4240 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4241 while (approxIndex <= (maxIndex / 4) * 3) {
4242 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4243 approxIndex += in.pos;
4244 CHECK_Z(in.pos == in.size);
4245 in.pos = 0;
4246 out.pos = 0;
4247 }
4248 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4249 }
4250
4251 /* spew a bunch of stuff into the table area */
4252 for (cLevel = 1; cLevel <= 22; cLevel++) {
4253 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4254 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4255 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4256 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4257 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4258 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4259 approxIndex += in.pos;
4260 }
4261
4262 /* now crank the indices so we overflow */
4263 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4264 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4265 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4266 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4267 while (approxIndex <= maxIndex) {
4268 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4269 approxIndex += in.pos;
4270 CHECK_Z(in.pos == in.size);
4271 in.pos = 0;
4272 out.pos = 0;
4273 }
4274 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4275 }
4276
4277 /* do a bunch of compressions again in low indices and ensure we don't
4278 * hit untracked invalid indices */
4279 for (cLevel = 1; cLevel <= 22; cLevel++) {
4280 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4281 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4282 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4283 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4284 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4285 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4286 approxIndex += in.pos;
4287 }
4288
4289 free(staticCCtxBuffer);
4290 }
4291 DISPLAYLEVEL(3, "OK \n");
4292
4293 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
4294 { size_t cSizeLdm;
4295 size_t cSizeNoLdm;
4296 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4297
4298 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
4299
4300 /* Enable checksum to verify round trip. */
4301 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4302 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4303 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4304
4305 /* Round trip once with ldm. */
4306 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4307 CHECK_Z(cSizeLdm);
4308 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
4309
4310 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4311 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4312 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
4313 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4314
4315 /* Round trip once without ldm. */
4316 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4317 CHECK_Z(cSizeNoLdm);
4318 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
4319
4320 if (cSizeLdm > cSizeNoLdm) {
4321 DISPLAY("Using long mode should not cause regressions for btopt+\n");
4322 testResult = 1;
4323 goto _end;
4324 }
4325
4326 ZSTD_freeCCtx(cctx);
4327 }
4328 DISPLAYLEVEL(3, "OK \n");
4329
4330 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++);
4331 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4332 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4333 size_t dictSize = CNBuffSize;
4334 void* dict = (void*)malloc(dictSize);
4335 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams();
4336 ZSTD_dictAttachPref_e const attachPrefs[] = {
4337 ZSTD_dictDefaultAttach,
4338 ZSTD_dictForceAttach,
4339 ZSTD_dictForceCopy,
4340 ZSTD_dictForceLoad,
4341 ZSTD_dictDefaultAttach,
4342 ZSTD_dictForceAttach,
4343 ZSTD_dictForceCopy,
4344 ZSTD_dictForceLoad
4345 };
4346 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1};
4347 int cLevel;
4348 int i;
4349
4350 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
4351 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
4352
4353 CHECK_Z(cctx_params != NULL);
4354
4355 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
4356 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
4357 for (cLevel = 4; cLevel < 13; cLevel++) {
4358 for (i = 0; i < 8; ++i) {
4359 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i];
4360 int const enableDDS = enableDedicatedDictSearch[i];
4361 ZSTD_CDict* cdict;
4362
4363 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i);
4364
4365 ZSTD_CCtxParams_init(cctx_params, cLevel);
4366 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS));
4367
4368 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem);
4369 CHECK(cdict != NULL);
4370
4371 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
4372 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref));
4373
4374 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4375 CHECK_Z(cSize);
4376 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
4377
4378 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize);
4379
4380 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
4381 ZSTD_freeCDict(cdict);
4382 } } }
4383
4384 ZSTD_freeCCtx(cctx);
4385 ZSTD_freeDCtx(dctx);
4386 ZSTD_freeCCtxParams(cctx_params);
4387 free(dict);
4388 }
4389 DISPLAYLEVEL(3, "OK \n");
4390
4391 _end:
4392 free(CNBuffer);
4393 free(compressedBuffer);
4394 free(decodedBuffer);
4395 return testResult;
4396 }
4397
4398
findDiff(const void * buf1,const void * buf2,size_t max)4399 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
4400 {
4401 const BYTE* b1 = (const BYTE*)buf1;
4402 const BYTE* b2 = (const BYTE*)buf2;
4403 size_t u;
4404 for (u=0; u<max; u++) {
4405 if (b1[u] != b2[u]) break;
4406 }
4407 return u;
4408 }
4409
4410
FUZ_makeParams(ZSTD_compressionParameters cParams,ZSTD_frameParameters fParams)4411 static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
4412 {
4413 ZSTD_parameters params;
4414 params.cParams = cParams;
4415 params.fParams = fParams;
4416 return params;
4417 }
4418
FUZ_rLogLength(U32 * seed,U32 logLength)4419 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
4420 {
4421 size_t const lengthMask = ((size_t)1 << logLength) - 1;
4422 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
4423 }
4424
FUZ_randomLength(U32 * seed,U32 maxLog)4425 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
4426 {
4427 U32 const logLength = FUZ_rand(seed) % maxLog;
4428 return FUZ_rLogLength(seed, logLength);
4429 }
4430
4431 #undef CHECK
4432 #define CHECK(cond, ...) { \
4433 if (cond) { \
4434 DISPLAY("Error => "); \
4435 DISPLAY(__VA_ARGS__); \
4436 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4437 goto _output_error; \
4438 } }
4439
4440 #undef CHECK_Z
4441 #define CHECK_Z(f) { \
4442 size_t const err = f; \
4443 if (ZSTD_isError(err)) { \
4444 DISPLAY("Error => %s : %s ", \
4445 #f, ZSTD_getErrorName(err)); \
4446 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4447 goto _output_error; \
4448 } }
4449
4450
fuzzerTests(U32 seed,unsigned nbTests,unsigned startTest,U32 const maxDurationS,double compressibility,int bigTests)4451 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
4452 {
4453 static const U32 maxSrcLog = 23;
4454 static const U32 maxSampleLog = 22;
4455 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
4456 size_t const dstBufferSize = (size_t)1<<maxSampleLog;
4457 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
4458 BYTE* cNoiseBuffer[5];
4459 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
4460 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
4461 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
4462 ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
4463 ZSTD_CCtx* const ctx = ZSTD_createCCtx();
4464 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4465 U32 result = 0;
4466 unsigned testNb = 0;
4467 U32 coreSeed = seed;
4468 UTIL_time_t const startClock = UTIL_getTime();
4469 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
4470 int const cLevelLimiter = bigTests ? 3 : 2;
4471
4472 /* allocation */
4473 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
4474 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
4475 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
4476 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
4477 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
4478 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
4479 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
4480 "Not enough memory, fuzzer tests cancelled");
4481
4482 /* Create initial samples */
4483 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
4484 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
4485 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
4486 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
4487 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
4488
4489 /* catch up testNb */
4490 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
4491
4492 /* main test loop */
4493 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
4494 BYTE* srcBuffer; /* jumping pointer */
4495 U32 lseed;
4496 size_t sampleSize, maxTestSize, totalTestSize;
4497 size_t cSize, totalCSize, totalGenSize;
4498 U64 crcOrig;
4499 BYTE* sampleBuffer;
4500 const BYTE* dict;
4501 size_t dictSize;
4502
4503 /* notification */
4504 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
4505 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
4506
4507 FUZ_rand(&coreSeed);
4508 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
4509
4510 /* srcBuffer selection [0-4] */
4511 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
4512 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
4513 else {
4514 buffNb >>= 3;
4515 if (buffNb & 7) {
4516 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
4517 buffNb = tnb[buffNb >> 3];
4518 } else {
4519 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
4520 buffNb = tnb[buffNb >> 3];
4521 } }
4522 srcBuffer = cNoiseBuffer[buffNb];
4523 }
4524
4525 /* select src segment */
4526 sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
4527
4528 /* create sample buffer (to catch read error with valgrind & sanitizers) */
4529 sampleBuffer = (BYTE*)malloc(sampleSize);
4530 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
4531 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
4532 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
4533 crcOrig = XXH64(sampleBuffer, sampleSize, 0);
4534
4535 /* compression tests */
4536 { int const cLevelPositive = (int)
4537 ( FUZ_rand(&lseed) %
4538 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) )
4539 + 1;
4540 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ?
4541 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */
4542 cLevelPositive;
4543 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel);
4544 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
4545 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
4546
4547 /* compression failure test : too small dest buffer */
4548 assert(cSize > 3);
4549 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;
4550 const size_t tooSmallSize = cSize - missing;
4551 const unsigned endMark = 0x4DC2B1A9;
4552 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark));
4553 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
4554 testNb, (unsigned)tooSmallSize, (unsigned)missing);
4555 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
4556 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
4557 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
4558 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); }
4559 } }
4560
4561 /* frame header decompression test */
4562 { ZSTD_frameHeader zfh;
4563 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
4564 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
4565 }
4566
4567 /* Decompressed size test */
4568 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
4569 CHECK(rSize != sampleSize, "decompressed size incorrect");
4570 }
4571
4572 /* successful decompression test */
4573 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
4574 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
4575 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
4576 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize);
4577 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
4578 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize);
4579 } }
4580
4581 free(sampleBuffer); /* no longer useful after this point */
4582
4583 /* truncated src decompression test */
4584 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
4585 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4586 size_t const tooSmallSize = cSize - missing;
4587 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
4588 CHECK(cBufferTooSmall == NULL, "not enough memory !");
4589 memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
4590 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
4591 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
4592 free(cBufferTooSmall);
4593 }
4594
4595 /* too small dst decompression test */
4596 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
4597 if (sampleSize > 3) {
4598 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4599 size_t const tooSmallSize = sampleSize - missing;
4600 static const BYTE token = 0xA9;
4601 dstBuffer[tooSmallSize] = token;
4602 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
4603 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
4604 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
4605 }
4606
4607 /* noisy src decompression test */
4608 if (cSize > 6) {
4609 /* insert noise into src */
4610 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
4611 size_t pos = 4; /* preserve magic number (too easy to detect) */
4612 for (;;) {
4613 /* keep some original src */
4614 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
4615 size_t const mask = (1<<nbBits) - 1;
4616 size_t const skipLength = FUZ_rand(&lseed) & mask;
4617 pos += skipLength;
4618 }
4619 if (pos >= cSize) break;
4620 /* add noise */
4621 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
4622 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
4623 size_t const mask = (1<<nbBits) - 1;
4624 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
4625 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
4626 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
4627 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
4628 pos += noiseLength;
4629 } } }
4630
4631 /* decompress noisy source */
4632 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
4633 { U32 const endMark = 0xA9B1C3D6;
4634 memcpy(dstBuffer+sampleSize, &endMark, 4);
4635 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
4636 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
4637 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
4638 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize);
4639 }
4640 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
4641 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
4642 } } } /* noisy src decompression test */
4643
4644 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
4645 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
4646 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
4647 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
4648 int const cLevel = (int)(FUZ_rand(&lseed) %
4649 ((U32)ZSTD_maxCLevel() -
4650 (MAX(testLog, dictLog) / (U32)cLevelLimiter))) +
4651 1;
4652 maxTestSize = FUZ_rLogLength(&lseed, testLog);
4653 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
4654
4655 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
4656 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
4657
4658 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
4659 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize);
4660
4661 if (FUZ_rand(&lseed) & 0xF) {
4662 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
4663 } else {
4664 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
4665 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
4666 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
4667 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
4668 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
4669 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
4670 }
4671 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
4672 }
4673
4674 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
4675 U32 n;
4676 XXH64_state_t xxhState;
4677 XXH64_reset(&xxhState, 0);
4678 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
4679 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
4680 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
4681
4682 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
4683 if (totalTestSize+segmentSize > maxTestSize) break;
4684
4685 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
4686 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
4687 cSize += compressResult;
4688 }
4689 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
4690 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
4691 totalTestSize += segmentSize;
4692 }
4693
4694 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
4695 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
4696 cSize += flushResult;
4697 }
4698 crcOrig = XXH64_digest(&xxhState);
4699 }
4700
4701 /* streaming decompression test */
4702 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
4703 /* ensure memory requirement is good enough (should always be true) */
4704 { ZSTD_frameHeader zfh;
4705 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
4706 "ZSTD_getFrameHeader(): error retrieving frame information");
4707 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
4708 CHECK_Z(roundBuffSize);
4709 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN),
4710 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
4711 (unsigned)roundBuffSize, (unsigned)totalTestSize );
4712 } }
4713 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
4714 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
4715 totalCSize = 0;
4716 totalGenSize = 0;
4717 while (totalCSize < cSize) {
4718 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
4719 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
4720 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
4721 totalGenSize += genSize;
4722 totalCSize += inSize;
4723 }
4724 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
4725 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
4726 CHECK (totalCSize != cSize, "compressed data should be fully read")
4727 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
4728 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
4729 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize);
4730 }
4731 } /* for ( ; (testNb <= nbTests) */
4732 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
4733
4734 _cleanup:
4735 ZSTD_freeCCtx(refCtx);
4736 ZSTD_freeCCtx(ctx);
4737 ZSTD_freeDCtx(dctx);
4738 free(cNoiseBuffer[0]);
4739 free(cNoiseBuffer[1]);
4740 free(cNoiseBuffer[2]);
4741 free(cNoiseBuffer[3]);
4742 free(cNoiseBuffer[4]);
4743 free(cBuffer);
4744 free(dstBuffer);
4745 free(mirrorBuffer);
4746 return (int)result;
4747
4748 _output_error:
4749 result = 1;
4750 goto _cleanup;
4751 }
4752
4753
4754 /*_*******************************************************
4755 * Command line
4756 *********************************************************/
FUZ_usage(const char * programName)4757 static int FUZ_usage(const char* programName)
4758 {
4759 DISPLAY( "Usage :\n");
4760 DISPLAY( " %s [args]\n", programName);
4761 DISPLAY( "\n");
4762 DISPLAY( "Arguments :\n");
4763 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault);
4764 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
4765 DISPLAY( " -s# : Select seed (default:prompt user)\n");
4766 DISPLAY( " -t# : Select starting test number (default:0)\n");
4767 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);
4768 DISPLAY( " -v : verbose\n");
4769 DISPLAY( " -p : pause at the end\n");
4770 DISPLAY( " -h : display help and exit\n");
4771 return 0;
4772 }
4773
4774 /*! readU32FromChar() :
4775 @return : unsigned integer value read from input in `char` format
4776 allows and interprets K, KB, KiB, M, MB and MiB suffix.
4777 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
4778 Note : function result can overflow if digit string > MAX_UINT */
readU32FromChar(const char ** stringPtr)4779 static unsigned readU32FromChar(const char** stringPtr)
4780 {
4781 unsigned result = 0;
4782 while ((**stringPtr >='0') && (**stringPtr <='9'))
4783 result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ;
4784 if ((**stringPtr=='K') || (**stringPtr=='M')) {
4785 result <<= 10;
4786 if (**stringPtr=='M') result <<= 10;
4787 (*stringPtr)++ ;
4788 if (**stringPtr=='i') (*stringPtr)++;
4789 if (**stringPtr=='B') (*stringPtr)++;
4790 }
4791 return result;
4792 }
4793
4794 /** longCommandWArg() :
4795 * check if *stringPtr is the same as longCommand.
4796 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
4797 * @return 0 and doesn't modify *stringPtr otherwise.
4798 */
longCommandWArg(const char ** stringPtr,const char * longCommand)4799 static int longCommandWArg(const char** stringPtr, const char* longCommand)
4800 {
4801 size_t const comSize = strlen(longCommand);
4802 int const result = !strncmp(*stringPtr, longCommand, comSize);
4803 if (result) *stringPtr += comSize;
4804 return result;
4805 }
4806
main(int argc,const char ** argv)4807 int main(int argc, const char** argv)
4808 {
4809 U32 seed = 0;
4810 int seedset = 0;
4811 int argNb;
4812 int nbTests = nbTestsDefault;
4813 int testNb = 0;
4814 int proba = FUZ_compressibility_default;
4815 double probfloat;
4816 int result = 0;
4817 U32 mainPause = 0;
4818 U32 maxDuration = 0;
4819 int bigTests = 1;
4820 int longTests = 0;
4821 U32 memTestsOnly = 0;
4822 const char* const programName = argv[0];
4823
4824 /* Check command line */
4825 for (argNb=1; argNb<argc; argNb++) {
4826 const char* argument = argv[argNb];
4827 if(!argument) continue; /* Protection if argument empty */
4828
4829 /* Handle commands. Aggregated commands are allowed */
4830 if (argument[0]=='-') {
4831
4832 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
4833
4834 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
4835 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
4836 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; }
4837 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; }
4838
4839 argument++;
4840 while (*argument!=0) {
4841 switch(*argument)
4842 {
4843 case 'h':
4844 return FUZ_usage(programName);
4845
4846 case 'v':
4847 argument++;
4848 g_displayLevel++;
4849 break;
4850
4851 case 'q':
4852 argument++;
4853 g_displayLevel--;
4854 break;
4855
4856 case 'p': /* pause at the end */
4857 argument++;
4858 mainPause = 1;
4859 break;
4860
4861 case 'i':
4862 argument++; maxDuration = 0;
4863 nbTests = (int)readU32FromChar(&argument);
4864 break;
4865
4866 case 'T':
4867 argument++;
4868 nbTests = 0;
4869 maxDuration = readU32FromChar(&argument);
4870 if (*argument=='s') argument++; /* seconds */
4871 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
4872 if (*argument=='n') argument++;
4873 break;
4874
4875 case 's':
4876 argument++;
4877 seedset = 1;
4878 seed = readU32FromChar(&argument);
4879 break;
4880
4881 case 't':
4882 argument++;
4883 testNb = (int)readU32FromChar(&argument);
4884 break;
4885
4886 case 'P': /* compressibility % */
4887 argument++;
4888 proba = (int)readU32FromChar(&argument);
4889 if (proba>100) proba = 100;
4890 break;
4891
4892 default:
4893 return (FUZ_usage(programName), 1);
4894 } } } } /* for (argNb=1; argNb<argc; argNb++) */
4895
4896 /* Get Seed */
4897 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
4898
4899 if (!seedset) {
4900 time_t const t = time(NULL);
4901 U32 const h = XXH32(&t, sizeof(t), 1);
4902 seed = h % 10000;
4903 }
4904
4905 DISPLAY("Seed = %u\n", (unsigned)seed);
4906 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba);
4907
4908 probfloat = ((double)proba) / 100;
4909
4910 if (memTestsOnly) {
4911 g_displayLevel = MAX(3, g_displayLevel);
4912 return FUZ_mallocTests(seed, probfloat, memTestsOnly);
4913 }
4914
4915 if (nbTests < testNb) nbTests = testNb;
4916
4917 if (testNb==0) {
4918 result = basicUnitTests(0, probfloat); /* constant seed for predictability */
4919
4920 if (!result && longTests) {
4921 result = longUnitTests(0, probfloat);
4922 }
4923 }
4924 if (!result)
4925 result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests);
4926 if (mainPause) {
4927 int unused;
4928 DISPLAY("Press Enter \n");
4929 unused = getchar();
4930 (void)unused;
4931 }
4932 return result;
4933 }
4934