1 /*
2 datagen.c - compressible data generator test tool
3 Copyright (C) Yann Collet 2012-2024
4
5 GPL v2 License
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License along
18 with this program; if not, write to the Free Software Foundation, Inc.,
19 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21 You can contact the author at :
22 - LZ4 source repository : https://github.com/lz4/lz4
23 - Public forum : https://groups.google.com/forum/#!forum/lz4c
24 */
25
26 /**************************************
27 * Includes
28 **************************************/
29 #include "platform.h" /* Compiler options, SET_BINARY_MODE */
30 #include "util.h" /* U32 */
31 #include <stdlib.h> /* malloc */
32 #include <stdio.h> /* FILE, fwrite */
33 #include <string.h> /* memcpy */
34 #include <assert.h>
35
36
37 /**************************************
38 * Constants
39 **************************************/
40 #define KB *(1 <<10)
41
42 #define PRIME1 2654435761U
43 #define PRIME2 2246822519U
44
45
46 /**************************************
47 * Local types
48 **************************************/
49 #define LTLOG 13
50 #define LTSIZE (1<<LTLOG)
51 #define LTMASK (LTSIZE-1)
52 typedef BYTE litDistribTable[LTSIZE];
53
54
55 /*********************************************************
56 * Local Functions
57 *********************************************************/
58 #define MIN(a,b) ( (a) < (b) ? (a) :(b) )
59 #define RDG_rotl32(x,r) ((x << r) | (x >> (32 - r)))
RDG_rand(U32 * src)60 static unsigned int RDG_rand(U32* src)
61 {
62 U32 rand32 = *src;
63 rand32 *= PRIME1;
64 rand32 ^= PRIME2;
65 rand32 = RDG_rotl32(rand32, 13);
66 *src = rand32;
67 return rand32;
68 }
69
70
RDG_fillLiteralDistrib(litDistribTable lt,double ld)71 static void RDG_fillLiteralDistrib(litDistribTable lt, double ld)
72 {
73 BYTE const firstChar = ld <= 0.0 ? 0 : '(';
74 BYTE const lastChar = ld <= 0.0 ? 255 : '}';
75 BYTE character = ld <= 0.0 ? 0 : '0';
76 U32 u = 0;
77
78 while (u<LTSIZE) {
79 U32 const weight = (U32)((double)(LTSIZE - u) * ld) + 1;
80 U32 const end = MIN(u+weight, LTSIZE);
81 while (u < end) {
82 assert(u<LTSIZE); /* try to ease static analyzer. u < end <= LTSIZE */
83 lt[u++] = character;
84 }
85 character++;
86 if (character > lastChar) character = firstChar;
87 }
88 }
89
90
RDG_genChar(U32 * seed,const litDistribTable lt)91 static BYTE RDG_genChar(U32* seed, const litDistribTable lt)
92 {
93 U32 id = RDG_rand(seed) & LTMASK;
94 return (lt[id]);
95 }
96
97
98 #define RDG_DICTSIZE (32 KB)
99 #define RDG_RAND15BITS ((RDG_rand(seed) >> 3) & 32767)
100 #define RDG_RANDLENGTH ( ((RDG_rand(seed) >> 7) & 7) ? (RDG_rand(seed) & 15) : (RDG_rand(seed) & 511) + 15)
RDG_genBlock(void * buffer,size_t buffSize,size_t prefixSize,double matchProba,litDistribTable lt,unsigned * seedPtr)101 void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, litDistribTable lt, unsigned* seedPtr)
102 {
103 BYTE* buffPtr = (BYTE*)buffer;
104 const U32 matchProba32 = (U32)(32768 * matchProba);
105 size_t pos = prefixSize;
106 U32* seed = seedPtr;
107
108 /* special case */
109 while (matchProba >= 1.0) {
110 size_t size0 = RDG_rand(seed) & 3;
111 size0 = (size_t)1 << (16 + size0 * 2);
112 size0 += RDG_rand(seed) & (size0-1); /* because size0 is power of 2*/
113 if (buffSize < pos + size0) {
114 memset(buffPtr+pos, 0, buffSize-pos);
115 return;
116 }
117 memset(buffPtr+pos, 0, size0);
118 pos += size0;
119 buffPtr[pos-1] = RDG_genChar(seed, lt);
120 }
121
122 /* init */
123 if (pos==0) {
124 buffPtr[0] = RDG_genChar(seed, lt);
125 pos=1;
126 }
127
128 /* Generate compressible data */
129 while (pos < buffSize) {
130 /* Select : Literal (char) or Match (within 32K) */
131 if (RDG_RAND15BITS < matchProba32) {
132 /* Copy (within 32K) */
133 size_t match;
134 size_t d;
135 size_t length = RDG_RANDLENGTH + 4;
136 U32 offset = RDG_RAND15BITS + 1;
137 if (offset > pos) offset = (U32)pos;
138 match = pos - offset;
139 d = pos + length;
140 if (d > buffSize) d = buffSize;
141 while (pos < d) buffPtr[pos++] = buffPtr[match++];
142 } else {
143 /* Literal (noise) */
144 size_t d;
145 size_t length = RDG_RANDLENGTH;
146 d = pos + length;
147 if (d > buffSize) d = buffSize;
148 while (pos < d) buffPtr[pos++] = RDG_genChar(seed, lt);
149 }
150 }
151 }
152
153
RDG_genBuffer(void * buffer,size_t size,double matchProba,double litProba,unsigned seed)154 void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed)
155 {
156 litDistribTable lt;
157 if (litProba==0.0) litProba = matchProba / 4.5;
158 RDG_fillLiteralDistrib(lt, litProba);
159 RDG_genBlock(buffer, size, 0, matchProba, lt, &seed);
160 }
161
162
163 #define RDG_BLOCKSIZE (128 KB)
RDG_genOut(unsigned long long size,double matchProba,double litProba,unsigned seed)164 void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed)
165 {
166 BYTE buff[RDG_DICTSIZE + RDG_BLOCKSIZE];
167 U64 total = 0;
168 size_t genBlockSize = RDG_BLOCKSIZE;
169 litDistribTable lt;
170
171 /* init */
172 if (litProba==0.0) litProba = matchProba / 4.5;
173 RDG_fillLiteralDistrib(lt, litProba);
174 SET_BINARY_MODE(stdout);
175
176 /* Generate dict */
177 RDG_genBlock(buff, RDG_DICTSIZE, 0, matchProba, lt, &seed);
178
179 /* Generate compressible data */
180 while (total < size) {
181 RDG_genBlock(buff, RDG_DICTSIZE+RDG_BLOCKSIZE, RDG_DICTSIZE, matchProba, lt, &seed);
182 if (size-total < RDG_BLOCKSIZE) genBlockSize = (size_t)(size-total);
183 total += genBlockSize;
184 fwrite(buff, 1, genBlockSize, stdout); /* should check potential write error */
185 /* update dict */
186 memcpy(buff, buff + RDG_BLOCKSIZE, RDG_DICTSIZE);
187 }
188 }
189