1 // ZstdDecoder.cpp
2
3 #include "StdAfx.h"
4
5 // #include <stdio.h>
6
7 #include "../../../C/Alloc.h"
8
9 #include "../Common/CWrappers.h"
10 #include "../Common/StreamUtils.h"
11
12 #include "ZstdDecoder.h"
13
14 namespace NCompress {
15 namespace NZstd {
16
17 static const size_t k_Zstd_BlockSizeMax = 1 << 17;
18 /*
19 we set _outStepMask as (k_Zstd_BlockSizeMax - 1), because:
20 - cycSize in zstd decoder for isCyclicMode is aligned for (1 << 17) only.
21 So some write sizes will be multiple of ((1 << 17) * n).
22 - Also it can be optimal to flush data after each block decoding.
23 */
24
CDecoder()25 CDecoder::CDecoder():
26 _outStepMask(k_Zstd_BlockSizeMax - 1) // must be = (1 << x) - 1
27 , _dec(NULL)
28 , _inProcessed(0)
29 , _inBufSize(1u << 19) // larger value will reduce the number of memcpy() calls in CZstdDec code
30 , _inBuf(NULL)
31 , FinishMode(false)
32 , DisableHash(False)
33 // , DisableHash(True) // for debug : fast decoding without hash calculation
34 {
35 // ZstdDecInfo_Clear(&ResInfo);
36 }
37
~CDecoder()38 CDecoder::~CDecoder()
39 {
40 if (_dec)
41 ZstdDec_Destroy(_dec);
42 MidFree(_inBuf);
43 }
44
45
Z7_COM7F_IMF(CDecoder::SetInBufSize (UInt32,UInt32 size))46 Z7_COM7F_IMF(CDecoder::SetInBufSize(UInt32 , UInt32 size))
47 { _inBufSize = size; return S_OK; }
Z7_COM7F_IMF(CDecoder::SetOutBufSize (UInt32,UInt32 size))48 Z7_COM7F_IMF(CDecoder::SetOutBufSize(UInt32 , UInt32 size))
49 {
50 // we round it down:
51 size >>= 1;
52 size |= size >> (1 << 0);
53 size |= size >> (1 << 1);
54 size |= size >> (1 << 2);
55 size |= size >> (1 << 3);
56 size |= size >> (1 << 4);
57 _outStepMask = size; // it's (1 << x) - 1 now
58 return S_OK;
59 }
60
Z7_COM7F_IMF(CDecoder::SetDecoderProperties2 (const Byte *,UInt32))61 Z7_COM7F_IMF(CDecoder::SetDecoderProperties2(const Byte * /* prop */, UInt32 /* size */))
62 {
63 // if (size != 3 && size != 5) return E_NOTIMPL;
64 return S_OK;
65 }
66
67
Z7_COM7F_IMF(CDecoder::SetFinishMode (UInt32 finishMode))68 Z7_COM7F_IMF(CDecoder::SetFinishMode(UInt32 finishMode))
69 {
70 FinishMode = (finishMode != 0);
71 // FinishMode = false; // for debug
72 return S_OK;
73 }
74
75
Z7_COM7F_IMF(CDecoder::ReadUnusedFromInBuf (void * data,UInt32 size,UInt32 * processedSize))76 Z7_COM7F_IMF(CDecoder::ReadUnusedFromInBuf(void *data, UInt32 size, UInt32 *processedSize))
77 {
78 size_t cur = ZstdDec_ReadUnusedFromInBuf(_dec, _afterDecoding_tempPos, data, size);
79 _afterDecoding_tempPos += cur;
80 size -= (UInt32)cur;
81 if (size)
82 {
83 const size_t rem = _state.inLim - _state.inPos;
84 if (size > rem)
85 size = (UInt32)rem;
86 if (size)
87 {
88 memcpy((Byte *)data + cur, _state.inBuf + _state.inPos, size);
89 _state.inPos += size;
90 cur += size;
91 }
92 }
93 *processedSize = (UInt32)cur;
94 return S_OK;
95 }
96
97
98
Prepare(const UInt64 * outSize)99 HRESULT CDecoder::Prepare(const UInt64 *outSize)
100 {
101 _inProcessed = 0;
102 _afterDecoding_tempPos = 0;
103 ZstdDecState_Clear(&_state);
104 ZstdDecInfo_CLEAR(&ResInfo)
105 // _state.outStep = _outStepMask + 1; // must be = (1 << x)
106 _state.disableHash = DisableHash;
107 if (outSize)
108 {
109 _state.outSize_Defined = True;
110 _state.outSize = *outSize;
111 // _state.outSize = 0; // for debug
112 }
113 if (!_dec)
114 {
115 _dec = ZstdDec_Create(&g_AlignedAlloc, &g_BigAlloc);
116 if (!_dec)
117 return E_OUTOFMEMORY;
118 }
119 if (!_inBuf || _inBufSize != _inBufSize_Allocated)
120 {
121 MidFree(_inBuf);
122 _inBuf = NULL;
123 _inBufSize_Allocated = 0;
124 _inBuf = (Byte *)MidAlloc(_inBufSize);
125 if (!_inBuf)
126 return E_OUTOFMEMORY;
127 _inBufSize_Allocated = _inBufSize;
128 }
129 _state.inBuf = _inBuf;
130 ZstdDec_Init(_dec);
131 return S_OK;
132 }
133
134
Z7_COM7F_IMF(CDecoder::Code (ISequentialInStream * inStream,ISequentialOutStream * outStream,const UInt64 * inSize,const UInt64 * outSize,ICompressProgressInfo * progress))135 Z7_COM7F_IMF(CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
136 const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress))
137 {
138 RINOK(Prepare(outSize))
139
140 UInt64 inPrev = 0;
141 UInt64 outPrev = 0;
142 UInt64 writtenSize = 0;
143 bool readWasFinished = false;
144 SRes sres = SZ_OK;
145 HRESULT hres = S_OK;
146 HRESULT hres_Read = S_OK;
147
148 for (;;)
149 {
150 if (_state.inPos == _state.inLim && !readWasFinished)
151 {
152 _state.inPos = 0;
153 _state.inLim = _inBufSize;
154 hres_Read = ReadStream(inStream, _inBuf, &_state.inLim);
155 // _state.inLim -= 5; readWasFinished = True; // for debug
156 if (_state.inLim != _inBufSize || hres_Read != S_OK)
157 {
158 // hres_Read = 99; // for debug
159 readWasFinished = True;
160 }
161 }
162 {
163 const size_t inPos_Start = _state.inPos;
164 sres = ZstdDec_Decode(_dec, &_state);
165 _inProcessed += _state.inPos - inPos_Start;
166 }
167 /*
168 if (_state.status == ZSTD_STATUS_FINISHED_FRAME)
169 printf("\nfinished frame pos=%8x, checksum=%08x\n", (unsigned)_state.outProcessed, (unsigned)_state.info.checksum);
170 */
171 const bool needStop = (sres != SZ_OK)
172 || _state.status == ZSTD_STATUS_OUT_REACHED
173 || (outSize && *outSize < _state.outProcessed)
174 || (readWasFinished && _state.inPos == _state.inLim
175 && ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(&_state));
176
177 size_t size = _state.winPos - _state.wrPos; // full write size
178 if (size)
179 {
180 if (!needStop)
181 {
182 // we try to flush on aligned positions, if possible
183 size = _state.needWrite_Size; // minimal required write size
184 const size_t alignedPos = _state.winPos & ~(size_t)_outStepMask;
185 if (alignedPos > _state.wrPos)
186 {
187 const size_t size2 = alignedPos - _state.wrPos; // optimized aligned size
188 if (size < size2)
189 size = size2;
190 }
191 }
192 if (size)
193 {
194 {
195 size_t curSize = size;
196 if (outSize)
197 {
198 const UInt64 rem = *outSize - writtenSize;
199 if (curSize > rem)
200 curSize = (size_t)rem;
201 }
202 if (curSize)
203 {
204 // printf("Write wrPos=%8x, size=%8x\n", (unsigned)_state.wrPos, (unsigned)size);
205 hres = WriteStream(outStream, _state.win + _state.wrPos, curSize);
206 if (hres != S_OK)
207 break;
208 writtenSize += curSize; // it's real size of data that was written to stream
209 }
210 }
211 _state.wrPos += size; // virtual written size, that will be reported to CZstdDec
212 // _state.needWrite_Size = 0; // optional
213 }
214 }
215
216 if (needStop)
217 break;
218 if (progress)
219 if (_inProcessed - inPrev >= (1 << 27)
220 || _state.outProcessed - outPrev >= (1 << 28))
221 {
222 inPrev = _inProcessed;
223 outPrev = _state.outProcessed;
224 RINOK(progress->SetRatioInfo(&inPrev, &outPrev))
225 }
226 }
227
228 if (hres == S_OK)
229 {
230 ZstdDec_GetResInfo(_dec, &_state, sres, &ResInfo);
231 sres = ResInfo.decode_SRes;
232 /* now (ResInfo.decode_SRes) can contain 2 extra error codes:
233 - SZ_ERROR_NO_ARCHIVE : if no frames
234 - SZ_ERROR_INPUT_EOF : if ZSTD_STATUS_NEEDS_MORE_INPUT
235 */
236 _inProcessed -= ResInfo.extraSize;
237 if (hres_Read != S_OK && _state.inLim == _state.inPos && readWasFinished)
238 {
239 /* if (there is stream reading error,
240 and decoding was stopped because of end of input stream),
241 then we use reading error as main error code */
242 if (sres == SZ_OK ||
243 sres == SZ_ERROR_INPUT_EOF ||
244 sres == SZ_ERROR_NO_ARCHIVE)
245 hres = hres_Read;
246 }
247 if (sres == SZ_ERROR_INPUT_EOF && !FinishMode)
248 {
249 /* SZ_ERROR_INPUT_EOF case is allowed case for (!FinishMode) mode.
250 So we restore SZ_OK result for that case: */
251 ResInfo.decode_SRes = sres = SZ_OK;
252 }
253 if (hres == S_OK)
254 {
255 hres = SResToHRESULT(sres);
256 if (hres == S_OK && FinishMode)
257 {
258 if ((inSize && *inSize != _inProcessed)
259 || ResInfo.is_NonFinishedFrame
260 || (outSize && (*outSize != writtenSize || writtenSize != _state.outProcessed)))
261 hres = S_FALSE;
262 }
263 }
264 }
265 return hres;
266 }
267
268
Z7_COM7F_IMF(CDecoder::GetInStreamProcessedSize (UInt64 * value))269 Z7_COM7F_IMF(CDecoder::GetInStreamProcessedSize(UInt64 *value))
270 {
271 *value = _inProcessed;
272 return S_OK;
273 }
274
275
276 #ifndef Z7_NO_READ_FROM_CODER_ZSTD
277
Z7_COM7F_IMF(CDecoder::SetOutStreamSize (const UInt64 * outSize))278 Z7_COM7F_IMF(CDecoder::SetOutStreamSize(const UInt64 *outSize))
279 {
280 _inProcessed = 0;
281 _hres_Read = S_OK;
282 _hres_Decode = S_OK;
283 _writtenSize = 0;
284 _readWasFinished = false;
285 _wasFinished = false;
286 return Prepare(outSize);
287 }
288
289
Z7_COM7F_IMF(CDecoder::SetInStream (ISequentialInStream * inStream))290 Z7_COM7F_IMF(CDecoder::SetInStream(ISequentialInStream *inStream))
291 { _inStream = inStream; return S_OK; }
Z7_COM7F_IMF(CDecoder::ReleaseInStream ())292 Z7_COM7F_IMF(CDecoder::ReleaseInStream())
293 { _inStream.Release(); return S_OK; }
294
295
296 // if SetInStream() mode: the caller must call GetFinishResult() after full decoding
297 // to check that there decoding was finished correctly
298
GetFinishResult()299 HRESULT CDecoder::GetFinishResult()
300 {
301 if (_state.winPos != _state.wrPos || !_wasFinished)
302 return FinishMode ? S_FALSE : S_OK;
303 // _state.winPos == _state.wrPos
304 // _wasFinished == true
305 if (FinishMode && _hres_Decode == S_OK && _state.outSize_Defined && _state.outSize != _writtenSize)
306 _hres_Decode = S_FALSE;
307 return _hres_Decode;
308 }
309
310
Z7_COM7F_IMF(CDecoder::Read (void * data,UInt32 size,UInt32 * processedSize))311 Z7_COM7F_IMF(CDecoder::Read(void *data, UInt32 size, UInt32 *processedSize))
312 {
313 if (processedSize)
314 *processedSize = 0;
315
316 for (;;)
317 {
318 if (_state.outSize_Defined)
319 {
320 // _writtenSize <= _state.outSize
321 const UInt64 rem = _state.outSize - _writtenSize;
322 if (size > rem)
323 size = (UInt32)rem;
324 }
325 {
326 size_t cur = _state.winPos - _state.wrPos;
327 if (cur)
328 {
329 // _state.winPos != _state.wrPos;
330 // so there is some decoded data that was not written still
331 if (size == 0)
332 {
333 // if (FinishMode) and we are not allowed to write more, then it's data error
334 if (FinishMode && _state.outSize_Defined && _state.outSize == _writtenSize)
335 return S_FALSE;
336 return S_OK;
337 }
338 if (cur > size)
339 cur = (size_t)size;
340 // cur != 0
341 memcpy(data, _state.win + _state.wrPos, cur);
342 _state.wrPos += cur;
343 _writtenSize += cur;
344 data = (void *)((Byte *)data + cur);
345 if (processedSize)
346 *processedSize += (UInt32)cur;
347 size -= (UInt32)cur;
348 continue;
349 }
350 }
351
352 // _state.winPos == _state.wrPos
353 if (_wasFinished)
354 {
355 if (_hres_Decode == S_OK && FinishMode
356 && _state.outSize_Defined && _state.outSize != _writtenSize)
357 _hres_Decode = S_FALSE;
358 return _hres_Decode;
359 }
360
361 // _wasFinished == false
362 if (size == 0 && _state.outSize_Defined && _state.outSize != _state.outProcessed)
363 {
364 /* size == 0 : so the caller don't need more data now.
365 _state.outSize > _state.outProcessed : so more data will be requested
366 later by caller for full processing.
367 So we exit without ZstdDec_Decode() call, because we don't want
368 ZstdDec_Decode() to start new block decoding
369 */
370 return S_OK;
371 }
372 // size != 0 || !_state.outSize_Defined || _state.outSize == _state.outProcessed)
373
374 if (_state.inPos == _state.inLim && !_readWasFinished)
375 {
376 _state.inPos = 0;
377 _state.inLim = _inBufSize;
378 _hres_Read = ReadStream(_inStream, _inBuf, &_state.inLim);
379 if (_state.inLim != _inBufSize || _hres_Read != S_OK)
380 {
381 // _hres_Read = 99; // for debug
382 _readWasFinished = True;
383 }
384 }
385
386 SRes sres;
387 {
388 const SizeT inPos_Start = _state.inPos;
389 sres = ZstdDec_Decode(_dec, &_state);
390 _inProcessed += _state.inPos - inPos_Start;
391 }
392
393 const bool inFinished = (_state.inPos == _state.inLim) && _readWasFinished;
394
395 _wasFinished = (sres != SZ_OK)
396 || _state.status == ZSTD_STATUS_OUT_REACHED
397 || (_state.outSize_Defined && _state.outSize < _state.outProcessed)
398 || (inFinished
399 && ZstdDecState_DOES_NEED_MORE_INPUT_OR_FINISHED_FRAME(&_state));
400
401 if (!_wasFinished)
402 continue;
403
404 // _wasFinished == true
405 /* (_state.winPos != _state.wrPos) is possible here.
406 So we still can have some data to flush,
407 but we must all result variables .
408 */
409 HRESULT hres = S_OK;
410 ZstdDec_GetResInfo(_dec, &_state, sres, &ResInfo);
411 sres = ResInfo.decode_SRes;
412 _inProcessed -= ResInfo.extraSize;
413 if (_hres_Read != S_OK && inFinished)
414 {
415 if (sres == SZ_OK ||
416 sres == SZ_ERROR_INPUT_EOF ||
417 sres == SZ_ERROR_NO_ARCHIVE)
418 hres = _hres_Read;
419 }
420 if (sres == SZ_ERROR_INPUT_EOF && !FinishMode)
421 ResInfo.decode_SRes = sres = SZ_OK;
422 if (hres == S_OK)
423 {
424 hres = SResToHRESULT(sres);
425 if (hres == S_OK && FinishMode)
426 if (!inFinished
427 || ResInfo.is_NonFinishedFrame
428 || (_state.outSize_Defined && _state.outSize != _state.outProcessed))
429 hres = S_FALSE;
430 }
431 _hres_Decode = hres;
432 }
433 }
434
435 #endif
436
437 }}
438