1 // 7zUpdate.cpp
2
3 #include "StdAfx.h"
4
5 #include "../../../../C/CpuArch.h"
6
7 #include "../../../Common/MyLinux.h"
8 #include "../../../Common/StringToInt.h"
9 #include "../../../Common/Wildcard.h"
10
11 #include "../../Common/CreateCoder.h"
12 #include "../../Common/LimitedStreams.h"
13 #include "../../Common/ProgressUtils.h"
14
15 #include "../../Compress/CopyCoder.h"
16
17 #include "../Common/ItemNameUtils.h"
18
19 #include "7zDecode.h"
20 #include "7zEncode.h"
21 #include "7zFolderInStream.h"
22 #include "7zHandler.h"
23 #include "7zOut.h"
24 #include "7zUpdate.h"
25
26 namespace NArchive {
27 namespace N7z {
28
29 #define k_X86 k_BCJ
30
31 struct CFilterMode
32 {
33 UInt32 Id;
34 UInt32 Delta; // required File Size alignment, if Id is not k_Delta.
35 // (Delta == 0) means unknown alignment
36 UInt32 Offset; // for k_ARM64 / k_RISCV
37 // UInt32 AlignSizeOpt; // for k_ARM64
38
CFilterModeNArchive::N7z::CFilterMode39 CFilterMode():
40 Id(0),
41 Delta(0),
42 Offset(0)
43 // , AlignSizeOpt(0)
44 {}
45
ClearFilterModeNArchive::N7z::CFilterMode46 void ClearFilterMode()
47 {
48 Id = 0;
49 Delta = 0;
50 Offset = 0;
51 // AlignSizeOpt = 0;
52 }
53
54 // it sets Delta as Align value, if Id is exe filter
55 // in another cases it sets Delta = 0, that
SetDeltaNArchive::N7z::CFilterMode56 void SetDelta()
57 {
58 if (Id == k_IA64)
59 Delta = 16;
60 else if (Id == k_ARM64 || Id == k_ARM || Id == k_PPC || Id == k_SPARC)
61 Delta = 4;
62 else if (Id == k_ARMT || Id == k_RISCV)
63 Delta = 2;
64 else if (Id == k_BCJ || Id == k_BCJ2)
65 Delta = 1; // do we need it?
66 else
67 Delta = 0;
68 }
69 };
70
71
72 /* ---------- PE ---------- */
73
74 #define MZ_SIG 0x5A4D
75
76 #define PE_SIG 0x00004550
77 #define PE_OptHeader_Magic_32 0x10B
78 #define PE_OptHeader_Magic_64 0x20B
79 // #define PE_SectHeaderSize 40
80 // #define PE_SECT_EXECUTE 0x20000000
81
Parse_EXE(const Byte * buf,size_t size,CFilterMode * filterMode)82 static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode)
83 {
84 if (size < 512 || GetUi16(buf) != MZ_SIG)
85 return 0;
86
87 const Byte *p;
88 UInt32 peOffset, optHeaderSize, filterId;
89
90 peOffset = GetUi32(buf + 0x3C);
91 if (peOffset >= 0x1000 || peOffset + 512 > size || (peOffset & 7) != 0)
92 return 0;
93 p = buf + peOffset;
94 if (GetUi32(p) != PE_SIG)
95 return 0;
96 p += 4;
97
98 const unsigned machine = GetUi16(p);
99
100 switch (machine)
101 {
102 case 0x014C:
103 case 0x8664: filterId = k_X86; break;
104 case 0xAA64: filterId = k_ARM64; break;
105
106 /*
107 IMAGE_FILE_MACHINE_ARM 0x01C0 // ARM LE
108 IMAGE_FILE_MACHINE_THUMB 0x01C2 // ARM Thumb / Thumb-2 LE
109 IMAGE_FILE_MACHINE_ARMNT 0x01C4 // ARM Thumb-2, LE
110 Note: We use ARM filter for 0x01C2. (WinCE 5 - 0x01C2) files mostly contain ARM code (not Thumb/Thumb-2).
111 */
112
113 case 0x01C0: // WinCE old
114 case 0x01C2: filterId = k_ARM; break; // WinCE new
115 case 0x01C4: filterId = k_ARMT; break; // WinRT
116
117 case 0x5032: // RISCV32
118 case 0x5064: // RISCV64
119 // case 0x5128: // RISCV128
120 filterId = k_RISCV; break;
121
122 case 0x0200: filterId = k_IA64; break;
123 default: return 0;
124 }
125
126 const UInt32 numSections = GetUi16(p + 2);
127 optHeaderSize = GetUi16(p + 16);
128 if (optHeaderSize > (1 << 10))
129 return 0;
130
131 p += 20; /* headerSize */
132
133 switch (GetUi16(p))
134 {
135 case PE_OptHeader_Magic_32:
136 case PE_OptHeader_Magic_64:
137 break;
138 default:
139 return 0;
140 }
141
142 // Windows exe file sizes are not aligned for 4 KiB.
143 // So we can't use (CFilterMode::Offset != 0) in solid archives.
144 // So we just don't set Offset here.
145 #define NUM_SCAN_SECTIONS_MAX (1 << 6)
146 // #define EXE_SECTION_OFFSET_MAX (1 << 27)
147 // #define EXE_SECTION_SIZE_MIN (1 << 8)
148 // #define EXE_SECTION_SIZE_MAX (1 << 27)
149 #define PE_SectHeaderSize 40
150 // #define PE_SECT_EXECUTE 0x20000000
151
152 /*
153 if (numSections > NUM_SCAN_SECTIONS_MAX)
154 return 0;
155 */
156
157 if ((size_t)(p - buf) + optHeaderSize <= size)
158 {
159 p += optHeaderSize;
160 /*
161 // UInt32 numExeSections = 0;
162 // bool execute_finded = false;
163 // UInt32 sect_va = 0;
164 // UInt32 sect_size = 0;
165 // UInt32 sect_offset = 0;
166 */
167 if (numSections <= NUM_SCAN_SECTIONS_MAX)
168 if (machine == 0x8664)
169 for (UInt32 i = 0; i < numSections
170 ; i++, p += PE_SectHeaderSize)
171 {
172 // UInt32 characts, rawSize, offset;
173 if ((UInt32)(p - buf) + PE_SectHeaderSize > size)
174 {
175 // return 0;
176 break;
177 }
178 if (memcmp(p, ".a64xrm", 8) == 0)
179 {
180 // ARM64EC
181 filterId = k_ARM64;
182 break;
183 }
184 /*
185 rawSize = GetUi32(p + 16);
186 offset = GetUi32(p + 20);
187 characts = GetUi32(p + 36);
188 if (rawSize >= EXE_SECTION_SIZE_MIN &&
189 rawSize <= EXE_SECTION_SIZE_MAX &&
190 offset <= EXE_SECTION_OFFSET_MAX &&
191 // offset < limit &&
192 offset > 0)
193 {
194 if ((characts & PE_SECT_EXECUTE) != 0)
195 {
196 // execute_finded = true;
197 // sect_va = GetUi32(p + 12);
198 // sect_size = rawSize;
199 // sect_offset = offset;
200 break;
201 }
202 }
203 */
204 }
205 }
206
207 /*
208 filterMode->Offset = 0;
209 if (filterId == k_ARM64)
210 {
211 // filterMode->AlignSizeOpt = (1 << 12);
212 // const UInt32 offs = (sect_va - sect_offset) & 0xFFF;
213 // if (offs != 0)
214 // filterMode->Offset = offs; // change it
215 }
216 */
217 filterMode->Id = filterId;
218 return 1;
219 }
220
221
222 /*
223 Filters don't improve the compression ratio for relocatable object files (".o").
224 But we can get compression ratio gain, if we compress object
225 files and executables in same solid block.
226 So we use filters for relocatable object files (".o"):
227 */
228 // #define Z7_7Z_CREATE_ARC_DISABLE_FILTER_FOR_OBJ
229
230 /* ---------- ELF ---------- */
231
232 #define ELF_SIG 0x464C457F
233
234 #define ELF_CLASS_32 1
235 #define ELF_CLASS_64 2
236
237 #define ELF_DATA_2LSB 1
238 #define ELF_DATA_2MSB 2
239
Get16(const Byte * p,BoolInt be)240 static UInt16 Get16(const Byte *p, BoolInt be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); }
Get32(const Byte * p,BoolInt be)241 static UInt32 Get32(const Byte *p, BoolInt be) { if (be) return GetBe32(p); return GetUi32(p); }
242 // static UInt64 Get64(const Byte *p, BoolInt be) { if (be) return GetBe64(p); return GetUi64(p); }
243
Parse_ELF(const Byte * buf,size_t size,CFilterMode * filterMode)244 static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode)
245 {
246 BoolInt /* is32, */ be;
247 UInt32 filterId;
248
249 if (size < 512 || buf[6] != 1) /* ver */
250 return 0;
251
252 if (GetUi32(buf) != ELF_SIG)
253 return 0;
254
255 switch (buf[4])
256 {
257 case ELF_CLASS_32: /* is32 = True; */ break;
258 case ELF_CLASS_64: /* is32 = False; */ break;
259 default: return 0;
260 }
261
262 switch (buf[5])
263 {
264 case ELF_DATA_2LSB: be = False; break;
265 case ELF_DATA_2MSB: be = True; break;
266 default: return 0;
267 }
268
269 #ifdef Z7_7Z_CREATE_ARC_DISABLE_FILTER_FOR_OBJ
270 #define ELF_ET_REL 1
271 if (Get16(buf + 0x10, be) == ELF_ET_REL)
272 return 0;
273 #endif
274
275 switch (Get16(buf + 0x12, be))
276 {
277 case 3:
278 case 6:
279 case 62: filterId = k_X86; break;
280 case 2:
281 case 18:
282 case 43: filterId = k_SPARC; break;
283 case 20:
284 case 21: if (!be) return 0; filterId = k_PPC; break;
285 case 40: if (be) return 0; filterId = k_ARM; break;
286 case 183: if (be) return 0; filterId = k_ARM64; break;
287 case 243: if (be) return 0; filterId = k_RISCV; break;
288
289 /* Some IA-64 ELF executables have size that is not aligned for 16 bytes.
290 So we don't use IA-64 filter for IA-64 ELF */
291 // case 50: if ( be) return 0; filterId = k_IA64; break;
292
293 default: return 0;
294 }
295
296 filterMode->Id = filterId;
297 return 1;
298 }
299
300
301
302 /* ---------- Mach-O ---------- */
303
304 #define MACH_SIG_BE_32 0xCEFAEDFE
305 #define MACH_SIG_BE_64 0xCFFAEDFE
306 #define MACH_SIG_LE_32 0xFEEDFACE
307 #define MACH_SIG_LE_64 0xFEEDFACF
308
309 #define MACH_ARCH_ABI64 (1 << 24)
310 #define MACH_MACHINE_386 7
311 #define MACH_MACHINE_ARM 12
312 #define MACH_MACHINE_SPARC 14
313 #define MACH_MACHINE_PPC 18
314 #define MACH_MACHINE_PPC64 (MACH_ARCH_ABI64 | MACH_MACHINE_PPC)
315 #define MACH_MACHINE_AMD64 (MACH_ARCH_ABI64 | MACH_MACHINE_386)
316 #define MACH_MACHINE_ARM64 (MACH_ARCH_ABI64 | MACH_MACHINE_ARM)
317
Parse_MACH(const Byte * buf,size_t size,CFilterMode * filterMode)318 static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode)
319 {
320 UInt32 filterId, numCommands, commandsSize;
321
322 if (size < 512)
323 return 0;
324
325 BoolInt /* mode64, */ be;
326 switch (GetUi32(buf))
327 {
328 case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break;
329 case MACH_SIG_BE_64: /* mode64 = True; */ be = True; break;
330 case MACH_SIG_LE_32: /* mode64 = False; */ be = False; break;
331 case MACH_SIG_LE_64: /* mode64 = True; */ be = False; break;
332 default: return 0;
333 }
334
335 #ifdef Z7_7Z_CREATE_ARC_DISABLE_FILTER_FOR_OBJ
336 #define MACH_TYPE_OBJECT 1
337 if (Get32(buf + 0xC, be) == MACH_TYPE_OBJECT)
338 return 0;
339 #endif
340
341 switch (Get32(buf + 4, be))
342 {
343 case MACH_MACHINE_386:
344 case MACH_MACHINE_AMD64: filterId = k_X86; break;
345 case MACH_MACHINE_ARM: if ( be) return 0; filterId = k_ARM; break;
346 case MACH_MACHINE_SPARC: if (!be) return 0; filterId = k_SPARC; break;
347 case MACH_MACHINE_PPC:
348 case MACH_MACHINE_PPC64: if (!be) return 0; filterId = k_PPC; break;
349 case MACH_MACHINE_ARM64: if ( be) return 0; filterId = k_ARM64; break;
350 default: return 0;
351 }
352
353 numCommands = Get32(buf + 0x10, be);
354 commandsSize = Get32(buf + 0x14, be);
355
356 if (commandsSize > (1 << 24) || numCommands > (1 << 18))
357 return 0;
358
359 filterMode->Id = filterId;
360 return 1;
361 }
362
363
364 /* ---------- WAV ---------- */
365
366 #define WAV_SUBCHUNK_fmt 0x20746D66
367 #define WAV_SUBCHUNK_data 0x61746164
368
369 #define RIFF_SIG 0x46464952
370
Parse_WAV(const Byte * buf,size_t size,CFilterMode * filterMode)371 static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode)
372 {
373 UInt32 subChunkSize, pos;
374 if (size < 0x2C)
375 return False;
376
377 if (GetUi32(buf + 0) != RIFF_SIG ||
378 GetUi32(buf + 8) != 0x45564157 || // WAVE
379 GetUi32(buf + 0xC) != WAV_SUBCHUNK_fmt)
380 return False;
381 subChunkSize = GetUi32(buf + 0x10);
382 /* [0x14 = format] = 1 (PCM) */
383 if (subChunkSize < 0x10 || subChunkSize > 0x12 || GetUi16(buf + 0x14) != 1)
384 return False;
385
386 const unsigned numChannels = GetUi16(buf + 0x16);
387 const unsigned bitsPerSample = GetUi16(buf + 0x22);
388 if ((bitsPerSample & 0x7) != 0)
389 return False;
390 const UInt32 delta = (UInt32)numChannels * (bitsPerSample >> 3);
391 if (delta == 0 || delta > 256)
392 return False;
393
394 pos = 0x14 + subChunkSize;
395
396 const int kNumSubChunksTests = 10;
397 // Do we need to scan more than 3 sub-chunks?
398 for (int i = 0; i < kNumSubChunksTests; i++)
399 {
400 if (pos + 8 > size)
401 return False;
402 subChunkSize = GetUi32(buf + pos + 4);
403 if (GetUi32(buf + pos) == WAV_SUBCHUNK_data)
404 {
405 filterMode->Id = k_Delta;
406 filterMode->Delta = delta;
407 return True;
408 }
409 if (subChunkSize > (1 << 16))
410 return False;
411 pos += subChunkSize + 8;
412 }
413 return False;
414 }
415
416
417 /*
418 filterMode->Delta will be set as:
419 = delta value : [1, 256] : for k_Delta
420 = 0 for another filters (branch filters)
421 */
ParseFile(const Byte * buf,size_t size,CFilterMode * filterMode)422 static BoolInt ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode)
423 {
424 filterMode->ClearFilterMode();
425
426 if (Parse_EXE(buf, size, filterMode)) return True;
427 if (Parse_ELF(buf, size, filterMode)) return True;
428 if (Parse_MACH(buf, size, filterMode)) return True;
429 return Parse_WAV(buf, size, filterMode);
430 }
431
432
433
434
435 struct CFilterMode2: public CFilterMode
436 {
437 bool Encrypted;
438 unsigned GroupIndex;
439
CFilterMode2NArchive::N7z::CFilterMode2440 CFilterMode2(): Encrypted(false) {}
441
CompareNArchive::N7z::CFilterMode2442 int Compare(const CFilterMode2 &m) const
443 {
444 if (!Encrypted)
445 {
446 if (m.Encrypted)
447 return -1;
448 }
449 else if (!m.Encrypted)
450 return 1;
451
452 const UInt32 id1 = Id;
453 const UInt32 id2 = m.Id;
454 /*
455 // we can change the order to place k_ARM64 files close to another exe files
456 if (id1 <= k_SPARC &&
457 id2 <= k_SPARC)
458 {
459 #define k_ARM64_FOR_SORT 0x3030901
460 if (id1 == k_ARM64) id1 = k_ARM64_FOR_SORT;
461 if (id2 == k_ARM64) id2 = k_ARM64_FOR_SORT;
462 }
463 */
464 if (id1 < id2) return -1;
465 if (id1 > id2) return 1;
466
467 if (Delta < m.Delta) return -1;
468 if (Delta > m.Delta) return 1;
469
470 if (Offset < m.Offset) return -1;
471 if (Offset > m.Offset) return 1;
472
473 /* we don't go here, because GetGroup()
474 and operator ==(const CFilterMode2 &m)
475 add only unique CFilterMode2:: { Id, Delta, Offset, Encrypted } items.
476 */
477 /*
478 if (GroupIndex < m.GroupIndex) return -1;
479 if (GroupIndex > m.GroupIndex) return 1;
480 */
481 return 0;
482 }
483
operator ==NArchive::N7z::CFilterMode2484 bool operator ==(const CFilterMode2 &m) const
485 {
486 return Id == m.Id
487 && Delta == m.Delta
488 && Offset == m.Offset
489 && Encrypted == m.Encrypted;
490 }
491 };
492
GetGroup(CRecordVector<CFilterMode2> & filters,const CFilterMode2 & m)493 static unsigned GetGroup(CRecordVector<CFilterMode2> &filters, const CFilterMode2 &m)
494 {
495 unsigned i;
496 for (i = 0; i < filters.Size(); i++)
497 {
498 const CFilterMode2 &m2 = filters[i];
499 if (m == m2)
500 return i;
501 /*
502 if (m.Encrypted != m2.Encrypted)
503 {
504 if (!m.Encrypted)
505 break;
506 continue;
507 }
508
509 if (m.Id < m2.Id) break;
510 if (m.Id != m2.Id) continue;
511
512 if (m.Delta < m2.Delta) break;
513 if (m.Delta != m2.Delta) continue;
514 */
515 }
516 // filters.Insert(i, m);
517 // return i;
518 return filters.Add(m);
519 }
520
Is86Filter(CMethodId m)521 static inline bool Is86Filter(CMethodId m)
522 {
523 return (m == k_BCJ || m == k_BCJ2);
524 }
525
IsExeFilter(CMethodId m)526 static inline bool IsExeFilter(CMethodId m)
527 {
528 switch (m)
529 {
530 case k_ARM64:
531 case k_RISCV:
532 case k_BCJ:
533 case k_BCJ2:
534 case k_ARM:
535 case k_ARMT:
536 case k_PPC:
537 case k_SPARC:
538 case k_IA64:
539 return true;
540 default: break;
541 }
542 return false;
543 }
544
Get_FilterGroup_for_Folder(CRecordVector<CFilterMode2> & filters,const CFolderEx & f,bool extractFilter)545 static unsigned Get_FilterGroup_for_Folder(
546 CRecordVector<CFilterMode2> &filters, const CFolderEx &f, bool extractFilter)
547 {
548 CFilterMode2 m;
549 // m.Id = 0;
550 // m.Delta = 0;
551 // m.Offset = 0;
552 m.Encrypted = f.IsEncrypted();
553
554 if (extractFilter)
555 {
556 const CCoderInfo &coder = f.Coders[f.UnpackCoder];
557
558 if (coder.MethodID == k_Delta)
559 {
560 if (coder.Props.Size() == 1)
561 {
562 m.Delta = (unsigned)coder.Props[0] + 1;
563 m.Id = k_Delta;
564 }
565 }
566 else if (IsExeFilter(coder.MethodID))
567 {
568 m.Id = (UInt32)coder.MethodID;
569 if (m.Id == k_BCJ2)
570 m.Id = k_BCJ;
571 m.SetDelta();
572 if (m.Id == k_ARM64 ||
573 m.Id == k_RISCV)
574 if (coder.Props.Size() == 4)
575 m.Offset = GetUi32(coder.Props);
576 }
577 }
578
579 return GetGroup(filters, m);
580 }
581
582
583
584
WriteRange(IInStream * inStream,ISequentialOutStream * outStream,UInt64 position,UInt64 size,ICompressProgressInfo * progress)585 static HRESULT WriteRange(IInStream *inStream, ISequentialOutStream *outStream,
586 UInt64 position, UInt64 size, ICompressProgressInfo *progress)
587 {
588 RINOK(InStream_SeekSet(inStream, position))
589 CMyComPtr2_Create<ISequentialInStream, CLimitedSequentialInStream> streamSpec;
590 streamSpec->SetStream(inStream);
591 streamSpec->Init(size);
592 CMyComPtr2_Create<ICompressCoder, NCompress::CCopyCoder> copyCoder;
593 RINOK(copyCoder.Interface()->Code(streamSpec, outStream, NULL, NULL, progress))
594 return (copyCoder->TotalSize == size ? S_OK : E_FAIL);
595 }
596
597 /*
598 unsigned CUpdateItem::GetExtensionPos() const
599 {
600 int slashPos = Name.ReverseFind_PathSepar();
601 int dotPos = Name.ReverseFind_Dot();
602 if (dotPos <= slashPos)
603 return Name.Len();
604 return dotPos + 1;
605 }
606
607 UString CUpdateItem::GetExtension() const
608 {
609 return Name.Ptr(GetExtensionPos());
610 }
611 */
612
613 #define RINOZ(x) { const int _t_ = (x); if (_t_ != 0) return _t_; }
614
615 #define RINOZ_COMP(a, b) RINOZ(MyCompare(a, b))
616
617 /*
618 static int CompareBuffers(const CByteBuffer &a1, const CByteBuffer &a2)
619 {
620 size_t c1 = a1.GetCapacity();
621 size_t c2 = a2.GetCapacity();
622 RINOZ_COMP(c1, c2);
623 for (size_t i = 0; i < c1; i++)
624 RINOZ_COMP(a1[i], a2[i]);
625 return 0;
626 }
627
628 static int CompareCoders(const CCoderInfo &c1, const CCoderInfo &c2)
629 {
630 RINOZ_COMP(c1.NumInStreams, c2.NumInStreams);
631 RINOZ_COMP(c1.NumOutStreams, c2.NumOutStreams);
632 RINOZ_COMP(c1.MethodID, c2.MethodID);
633 return CompareBuffers(c1.Props, c2.Props);
634 }
635
636 static int CompareBonds(const CBond &b1, const CBond &b2)
637 {
638 RINOZ_COMP(b1.InIndex, b2.InIndex);
639 return MyCompare(b1.OutIndex, b2.OutIndex);
640 }
641
642 static int CompareFolders(const CFolder &f1, const CFolder &f2)
643 {
644 int s1 = f1.Coders.Size();
645 int s2 = f2.Coders.Size();
646 RINOZ_COMP(s1, s2);
647 int i;
648 for (i = 0; i < s1; i++)
649 RINOZ(CompareCoders(f1.Coders[i], f2.Coders[i]));
650 s1 = f1.Bonds.Size();
651 s2 = f2.Bonds.Size();
652 RINOZ_COMP(s1, s2);
653 for (i = 0; i < s1; i++)
654 RINOZ(CompareBonds(f1.Bonds[i], f2.Bonds[i]));
655 return 0;
656 }
657 */
658
659 /*
660 static int CompareFiles(const CFileItem &f1, const CFileItem &f2)
661 {
662 return CompareFileNames(f1.Name, f2.Name);
663 }
664 */
665
666 struct CFolderRepack
667 {
668 unsigned FolderIndex;
669 CNum NumCopyFiles;
670 };
671
672 /*
673 static int CompareFolderRepacks(const CFolderRepack *p1, const CFolderRepack *p2, void *)
674 {
675 int i1 = p1->FolderIndex;
676 int i2 = p2->FolderIndex;
677 // In that version we don't want to parse folders here, so we don't compare folders
678 // probably it must be improved in future
679 // const CDbEx &db = *(const CDbEx *)param;
680 // RINOZ(CompareFolders(
681 // db.Folders[i1],
682 // db.Folders[i2]));
683
684 return MyCompare(i1, i2);
685
686 // RINOZ_COMP(
687 // db.NumUnpackStreamsVector[i1],
688 // db.NumUnpackStreamsVector[i2]);
689 // if (db.NumUnpackStreamsVector[i1] == 0)
690 // return 0;
691 // return CompareFiles(
692 // db.Files[db.FolderStartFileIndex[i1]],
693 // db.Files[db.FolderStartFileIndex[i2]]);
694 }
695 */
696
697 /*
698 we sort empty files and dirs in such order:
699 - Dir.NonAnti (name sorted)
700 - File.NonAnti (name sorted)
701 - File.Anti (name sorted)
702 - Dir.Anti (reverse name sorted)
703 */
704
CompareEmptyItems(const unsigned * p1,const unsigned * p2,void * param)705 static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param)
706 {
707 const CObjectVector<CUpdateItem> &updateItems = *(const CObjectVector<CUpdateItem> *)param;
708 const CUpdateItem &u1 = updateItems[*p1];
709 const CUpdateItem &u2 = updateItems[*p2];
710 // NonAnti < Anti
711 if (u1.IsAnti != u2.IsAnti)
712 return (u1.IsAnti ? 1 : -1);
713 if (u1.IsDir != u2.IsDir)
714 {
715 // Dir.NonAnti < File < Dir.Anti
716 if (u1.IsDir)
717 return (u1.IsAnti ? 1 : -1);
718 return (u2.IsAnti ? -1 : 1);
719 }
720 int n = CompareFileNames(u1.Name, u2.Name);
721 return (u1.IsDir && u1.IsAnti) ? -n : n;
722 }
723
724 static const char *g_Exts =
725 " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
726 " zip jar ear war msi"
727 " 3gp avi mov mpeg mpg mpe wmv"
728 " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"
729 " swf"
730 " chm hxi hxs"
731 " gif jpeg jpg jp2 png tiff bmp ico psd psp"
732 " awg ps eps cgm dxf svg vrml wmf emf ai md"
733 " cad dwg pps key sxi"
734 " max 3ds"
735 " iso bin nrg mdf img pdi tar cpio xpi"
736 " vfd vhd vud vmc vsv"
737 " vmdk dsk nvram vmem vmsd vmsn vmss vmtm"
738 " inl inc idl acf asa"
739 " h hpp hxx c cpp cxx m mm go swift"
740 " rc java cs rs pas bas vb cls ctl frm dlg def"
741 " f77 f f90 f95"
742 " asm s"
743 " sql manifest dep"
744 " mak clw csproj vcproj sln dsp dsw"
745 " class"
746 " bat cmd bash sh"
747 " xml xsd xsl xslt hxk hxc htm html xhtml xht mht mhtml htw asp aspx css cgi jsp shtml"
748 " awk sed hta js json php php3 php4 php5 phptml pl pm py pyo rb tcl ts vbs"
749 " text txt tex ans asc srt reg ini doc docx mcw dot rtf hlp xls xlr xlt xlw ppt pdf"
750 " sxc sxd sxi sxg sxw stc sti stw stm odt ott odg otg odp otp ods ots odf"
751 " abw afp cwk lwp wpd wps wpt wrf wri"
752 " abf afm bdf fon mgf otf pcf pfa snf ttf"
753 " dbf mdb nsf ntf wdb db fdb gdb"
754 " exe dll ocx vbx sfx sys tlb awx com obj lib out o so"
755 " pdb pch idb ncb opt";
756
GetExtIndex(const char * ext)757 static unsigned GetExtIndex(const char *ext)
758 {
759 unsigned extIndex = 1;
760 const char *p = g_Exts;
761 for (;;)
762 {
763 char c = *p++;
764 if (c == 0)
765 return extIndex;
766 if (c == ' ')
767 continue;
768 unsigned pos = 0;
769 for (;;)
770 {
771 char c2 = ext[pos++];
772 if (c2 == 0 && (c == 0 || c == ' '))
773 return extIndex;
774 if (c != c2)
775 break;
776 c = *p++;
777 }
778 extIndex++;
779 for (;;)
780 {
781 if (c == 0)
782 return extIndex;
783 if (c == ' ')
784 break;
785 c = *p++;
786 }
787 }
788 }
789
790 struct CRefItem
791 {
792 const CUpdateItem *UpdateItem;
793 UInt32 Index;
794 unsigned ExtensionPos;
795 unsigned NamePos;
796 unsigned ExtensionIndex;
797
CRefItemNArchive::N7z::CRefItem798 CRefItem() {}
CRefItemNArchive::N7z::CRefItem799 CRefItem(UInt32 index, const CUpdateItem &ui, bool sortByType):
800 UpdateItem(&ui),
801 Index(index),
802 ExtensionPos(0),
803 NamePos(0),
804 ExtensionIndex(0)
805 {
806 if (sortByType)
807 {
808 const int slashPos = ui.Name.ReverseFind_PathSepar();
809 NamePos = (unsigned)(slashPos + 1);
810 const int dotPos = ui.Name.ReverseFind_Dot();
811 if (dotPos <= slashPos)
812 ExtensionPos = ui.Name.Len();
813 else
814 {
815 ExtensionPos = (unsigned)(dotPos + 1);
816 if (ExtensionPos != ui.Name.Len())
817 {
818 AString s;
819 for (unsigned pos = ExtensionPos;; pos++)
820 {
821 const wchar_t c = ui.Name[pos];
822 if (c >= 0x80)
823 break;
824 if (c == 0)
825 {
826 ExtensionIndex = GetExtIndex(s);
827 break;
828 }
829 s.Add_Char((char)MyCharLower_Ascii((char)c));
830 }
831 }
832 }
833 }
834 }
835 };
836
837 struct CSortParam
838 {
839 // const CObjectVector<CTreeFolder> *TreeFolders;
840 bool SortByType;
841 };
842
843 /*
844 we sort files in such order:
845 - Dir.NonAnti (name sorted)
846 - alt streams
847 - Dirs
848 - Dir.Anti (reverse name sorted)
849 */
850
851
CompareUpdateItems(const CRefItem * p1,const CRefItem * p2,void * param)852 static int CompareUpdateItems(const CRefItem *p1, const CRefItem *p2, void *param)
853 {
854 const CRefItem &a1 = *p1;
855 const CRefItem &a2 = *p2;
856 const CUpdateItem &u1 = *a1.UpdateItem;
857 const CUpdateItem &u2 = *a2.UpdateItem;
858
859 /*
860 if (u1.IsAltStream != u2.IsAltStream)
861 return u1.IsAltStream ? 1 : -1;
862 */
863
864 // Actually there are no dirs that time. They were stored in other steps
865 // So that code is unused?
866 if (u1.IsDir != u2.IsDir)
867 return u1.IsDir ? 1 : -1;
868 if (u1.IsDir)
869 {
870 if (u1.IsAnti != u2.IsAnti)
871 return (u1.IsAnti ? 1 : -1);
872 int n = CompareFileNames(u1.Name, u2.Name);
873 return -n;
874 }
875
876 // bool sortByType = *(bool *)param;
877 const CSortParam *sortParam = (const CSortParam *)param;
878 const bool sortByType = sortParam->SortByType;
879 if (sortByType)
880 {
881 RINOZ_COMP(a1.ExtensionIndex, a2.ExtensionIndex)
882 RINOZ(CompareFileNames(u1.Name.Ptr(a1.ExtensionPos), u2.Name.Ptr(a2.ExtensionPos)))
883 RINOZ(CompareFileNames(u1.Name.Ptr(a1.NamePos), u2.Name.Ptr(a2.NamePos)))
884 if (!u1.MTimeDefined && u2.MTimeDefined) return 1;
885 if (u1.MTimeDefined && !u2.MTimeDefined) return -1;
886 if (u1.MTimeDefined && u2.MTimeDefined) RINOZ_COMP(u1.MTime, u2.MTime)
887 RINOZ_COMP(u1.Size, u2.Size)
888 }
889 /*
890 int par1 = a1.UpdateItem->ParentFolderIndex;
891 int par2 = a2.UpdateItem->ParentFolderIndex;
892 const CTreeFolder &tf1 = (*sortParam->TreeFolders)[par1];
893 const CTreeFolder &tf2 = (*sortParam->TreeFolders)[par2];
894
895 int b1 = tf1.SortIndex, e1 = tf1.SortIndexEnd;
896 int b2 = tf2.SortIndex, e2 = tf2.SortIndexEnd;
897 if (b1 < b2)
898 {
899 if (e1 <= b2)
900 return -1;
901 // p2 in p1
902 int par = par2;
903 for (;;)
904 {
905 const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
906 par = tf.Parent;
907 if (par == par1)
908 {
909 RINOZ(CompareFileNames(u1.Name, tf.Name));
910 break;
911 }
912 }
913 }
914 else if (b2 < b1)
915 {
916 if (e2 <= b1)
917 return 1;
918 // p1 in p2
919 int par = par1;
920 for (;;)
921 {
922 const CTreeFolder &tf = (*sortParam->TreeFolders)[par];
923 par = tf.Parent;
924 if (par == par2)
925 {
926 RINOZ(CompareFileNames(tf.Name, u2.Name));
927 break;
928 }
929 }
930 }
931 */
932 // RINOZ_COMP(a1.UpdateItem->ParentSortIndex, a2.UpdateItem->ParentSortIndex);
933 RINOK(CompareFileNames(u1.Name, u2.Name))
934 RINOZ_COMP(a1.UpdateItem->IndexInClient, a2.UpdateItem->IndexInClient)
935 RINOZ_COMP(a1.UpdateItem->IndexInArchive, a2.UpdateItem->IndexInArchive)
936 return 0;
937 }
938
939 struct CSolidGroup
940 {
941 CRecordVector<UInt32> Indices;
942
943 CRecordVector<CFolderRepack> folderRefs;
944 };
945
946 static const char * const g_Exe_Exts[] =
947 {
948 "dll"
949 , "exe"
950 , "ocx"
951 , "sfx"
952 , "sys"
953 };
954
955 static const char * const g_ExeUnix_Exts[] =
956 {
957 "so"
958 , "dylib"
959 };
960
IsExt_Exe(const wchar_t * ext)961 static bool IsExt_Exe(const wchar_t *ext)
962 {
963 for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_Exe_Exts); i++)
964 if (StringsAreEqualNoCase_Ascii(ext, g_Exe_Exts[i]))
965 return true;
966 return false;
967 }
968
969 /*
970 static bool IsExt_ExeUnix(const wchar_t *ext)
971 {
972 for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_ExeUnix_Exts); i++)
973 if (StringsAreEqualNoCase_Ascii(ext, g_ExeUnix_Exts[i]))
974 return true;
975 return false;
976 }
977 */
978
979 // we try to find "so" extension in such name: libstdc++.so.6.0.29
IsExt_ExeUnix_NumericAllowed(const UString & path)980 static bool IsExt_ExeUnix_NumericAllowed(const UString &path)
981 {
982 unsigned pos = path.Len();
983 unsigned dotPos = pos;
984 for (;;)
985 {
986 if (pos == 0)
987 return false;
988 const wchar_t c = path[--pos];
989 if (IS_PATH_SEPAR(c))
990 return false;
991 if (c == '.')
992 {
993 const unsigned num = (dotPos - pos) - 1;
994 if (num < 1)
995 return false;
996 const wchar_t *cur = path.Ptr(pos + 1);
997 for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_ExeUnix_Exts); i++)
998 {
999 const char *ext = g_ExeUnix_Exts[i];
1000 if (num == MyStringLen(ext))
1001 if (IsString1PrefixedByString2_NoCase_Ascii(cur, ext))
1002 return true;
1003 }
1004 const wchar_t *end;
1005 ConvertStringToUInt32(cur, &end);
1006 if ((size_t)(end - cur) != num)
1007 return false;
1008 dotPos = pos;
1009 }
1010 }
1011 }
1012
1013
1014 struct CAnalysis
1015 {
1016 CMyComPtr<IArchiveUpdateCallbackFile> Callback;
1017 CByteBuffer Buffer;
1018
1019 bool ParseWav;
1020 bool ParseExe;
1021 bool ParseExeUnix;
1022 bool ParseNoExt;
1023 bool ParseAll;
1024
1025 /*
1026 bool Need_ATime;
1027 bool ATime_Defined;
1028 FILETIME ATime;
1029 */
1030
CAnalysisNArchive::N7z::CAnalysis1031 CAnalysis():
1032 ParseWav(false),
1033 ParseExe(false),
1034 ParseExeUnix(false),
1035 ParseNoExt(false),
1036 ParseAll(false)
1037 /*
1038 , Need_ATime(false)
1039 , ATime_Defined(false)
1040 */
1041 {}
1042
1043 HRESULT GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode);
1044 };
1045
1046 static const size_t kAnalysisBufSize = 1 << 14;
1047
GetFilterGroup(UInt32 index,const CUpdateItem & ui,CFilterMode & filterMode)1048 HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMode &filterMode)
1049 {
1050 filterMode.Id = 0;
1051 filterMode.Delta = 0;
1052 filterMode.Offset = 0;
1053
1054 CFilterMode filterModeTemp = filterMode;
1055
1056 const int slashPos = ui.Name.ReverseFind_PathSepar();
1057 const int dotPos = ui.Name.ReverseFind_Dot();
1058
1059 // if (dotPos > slashPos)
1060 {
1061 bool needReadFile = ParseAll;
1062 /* if (Callback) is not supported by client,
1063 we still try to use file name extension to detect executable file */
1064 bool probablyIsSameIsa = false;
1065
1066 if (!needReadFile || !Callback)
1067 {
1068 const wchar_t *ext = NULL;
1069 if (dotPos > slashPos)
1070 ext = ui.Name.Ptr((unsigned)(dotPos + 1));
1071 // 7-zip stores posix attributes in high 16 bits and sets (0x8000) flag
1072 if (ui.Attrib & 0x8000)
1073 {
1074 const unsigned st_mode = ui.Attrib >> 16;
1075 /* note: executable ".so" can be without execute permission,
1076 and symbolic link to such ".so" file is possible */
1077 // st_mode = 00111; // for debug
1078 /* in Linux we expect such permissions:
1079 0755 : for most executables
1080 0644 : for some ".so" files
1081 0777 : in WSL for all files.
1082 We can try to exclude some such 0777 cases from analysis,
1083 if there is non-executable extension.
1084 */
1085
1086 if ((st_mode & (
1087 MY_LIN_S_IXUSR |
1088 MY_LIN_S_IXGRP |
1089 MY_LIN_S_IXOTH)) != 0
1090 && MY_LIN_S_ISREG(st_mode)
1091 && (ui.Size >= (1u << 11)))
1092 {
1093 #ifndef _WIN32
1094 probablyIsSameIsa = true;
1095 #endif
1096 needReadFile = true;
1097 }
1098 }
1099
1100 if (!needReadFile)
1101 {
1102 if (!ext)
1103 needReadFile = ParseNoExt;
1104 else
1105 {
1106 bool isUnixExt = false;
1107 if (ParseExeUnix)
1108 isUnixExt = IsExt_ExeUnix_NumericAllowed(ui.Name);
1109 if (isUnixExt)
1110 {
1111 needReadFile = true;
1112 #ifndef _WIN32
1113 probablyIsSameIsa = true;
1114 #endif
1115 }
1116 else if (IsExt_Exe(ext))
1117 {
1118 needReadFile = ParseExe;
1119 #ifdef _WIN32
1120 probablyIsSameIsa = true;
1121 #endif
1122 }
1123 else if (StringsAreEqualNoCase_Ascii(ext, "wav"))
1124 {
1125 if (!needReadFile)
1126 needReadFile = ParseWav;
1127 }
1128 }
1129 }
1130 }
1131
1132 if (needReadFile)
1133 {
1134 BoolInt parseRes = false;
1135 if (Callback)
1136 {
1137 if (Buffer.Size() != kAnalysisBufSize)
1138 Buffer.Alloc(kAnalysisBufSize);
1139 CMyComPtr<ISequentialInStream> stream;
1140 HRESULT result = Callback->GetStream2(index, &stream, NUpdateNotifyOp::kAnalyze);
1141 if (result == S_OK && stream)
1142 {
1143 /*
1144 if (Need_ATime)
1145 {
1146 // access time could be changed in analysis pass
1147 CMyComPtr<IStreamGetProps> getProps;
1148 stream.QueryInterface(IID_IStreamGetProps, (void **)&getProps);
1149 if (getProps)
1150 if (getProps->GetProps(NULL, NULL, &ATime, NULL, NULL) == S_OK)
1151 ATime_Defined = true;
1152 }
1153 */
1154 size_t size = kAnalysisBufSize;
1155 result = ReadStream(stream, Buffer, &size);
1156 stream.Release();
1157 // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK));
1158 if (result == S_OK)
1159 {
1160 parseRes = ParseFile(Buffer, size, &filterModeTemp);
1161 }
1162 }
1163 } // Callback
1164 else if (probablyIsSameIsa)
1165 {
1166 #ifdef MY_CPU_X86_OR_AMD64
1167 filterModeTemp.Id = k_X86;
1168 #endif
1169 #ifdef MY_CPU_ARM64
1170 filterModeTemp.Id = k_ARM64;
1171 #endif
1172 #ifdef MY_CPU_RISCV
1173 filterModeTemp.Id = k_RISCV;
1174 #endif
1175 #ifdef MY_CPU_SPARC
1176 filterModeTemp.Id = k_SPARC;
1177 #endif
1178 parseRes = true;
1179 }
1180
1181 if (parseRes
1182 && filterModeTemp.Id != k_Delta
1183 && filterModeTemp.Delta == 0)
1184 {
1185 /* ParseFile() sets (filterModeTemp.Delta == 0) for all
1186 methods except of k_Delta. */
1187 // it's not k_Delta
1188 // So we call SetDelta() to set Delta
1189 filterModeTemp.SetDelta();
1190 if (filterModeTemp.Delta > 1)
1191 {
1192 /* If file Size is not aligned, then branch filter
1193 will not work for next file in solid block.
1194 Maybe we should allow filter for non-aligned-size file in non-solid archives ?
1195 */
1196 if (ui.Size % filterModeTemp.Delta != 0)
1197 parseRes = false;
1198 // windows exe files are not aligned for 4 KiB.
1199 /*
1200 else if (filterModeTemp.Id == k_ARM64 && filterModeTemp.Offset != 0)
1201 {
1202 if (ui.Size % (1 << 12) != 0)
1203 {
1204 // If Size is not aligned for 4 KiB, then Offset will not work for next file in solid block.
1205 // so we place such file in group with (Offset==0).
1206 filterModeTemp.Offset = 0;
1207 }
1208 }
1209 */
1210 }
1211 }
1212 if (!parseRes)
1213 filterModeTemp.ClearFilterMode();
1214 }
1215 }
1216
1217 filterMode = filterModeTemp;
1218 return S_OK;
1219 }
1220
GetMethodFull(UInt64 methodID,UInt32 numStreams,CMethodFull & m)1221 static inline void GetMethodFull(UInt64 methodID, UInt32 numStreams, CMethodFull &m)
1222 {
1223 m.Id = methodID;
1224 m.NumStreams = numStreams;
1225 }
1226
1227
1228 // we add bond for mode.Methods[0] that is filter
AddBondForFilter(CCompressionMethodMode & mode)1229 static HRESULT AddBondForFilter(CCompressionMethodMode &mode)
1230 {
1231 for (unsigned c = 1; c < mode.Methods.Size(); c++)
1232 {
1233 if (!mode.IsThereBond_to_Coder(c))
1234 {
1235 CBond2 bond;
1236 bond.OutCoder = 0;
1237 bond.OutStream = 0;
1238 bond.InCoder = c;
1239 mode.Bonds.Add(bond);
1240 return S_OK;
1241 }
1242 }
1243 return E_INVALIDARG;
1244 }
1245
1246 /*
1247 static HRESULT AddBondForFilter_if_ThereAreBonds(CCompressionMethodMode &mode)
1248 {
1249 if (!mode.Bonds.IsEmpty())
1250 return AddBondForFilter(mode);
1251 return S_OK;
1252 }
1253 */
1254
AddBcj2Methods(CCompressionMethodMode & mode)1255 static HRESULT AddBcj2Methods(CCompressionMethodMode &mode)
1256 {
1257 // mode.Methods[0] must be k_BCJ2 method !
1258 // mode.Methods[1] : we expect that there is at least one method after BCJ2
1259
1260 CMethodFull m;
1261 GetMethodFull(k_LZMA, 1, m);
1262
1263 m.AddProp32(NCoderPropID::kDictionarySize, 1 << 20);
1264 m.AddProp32(NCoderPropID::kNumFastBytes, 128);
1265 m.AddProp32(NCoderPropID::kNumThreads, 1);
1266 m.AddProp32(NCoderPropID::kLitPosBits, 2);
1267 m.AddProp32(NCoderPropID::kLitContextBits, 0);
1268 // m.AddProp_Ascii(NCoderPropID::kMatchFinder, "BT2");
1269
1270 const unsigned methodIndex = mode.Methods.Size();
1271
1272 if (mode.Bonds.IsEmpty())
1273 {
1274 for (unsigned i = 1; i + 1 < mode.Methods.Size(); i++)
1275 {
1276 CBond2 bond;
1277 bond.OutCoder = i;
1278 bond.OutStream = 0;
1279 bond.InCoder = i + 1;
1280 mode.Bonds.Add(bond);
1281 }
1282 }
1283
1284 mode.Methods.Add(m);
1285 mode.Methods.Add(m);
1286
1287 RINOK(AddBondForFilter(mode))
1288 CBond2 bond;
1289 bond.OutCoder = 0; // index of BCJ2 coder
1290 bond.InCoder = methodIndex; bond.OutStream = 1; mode.Bonds.Add(bond);
1291 bond.InCoder = methodIndex + 1; bond.OutStream = 2; mode.Bonds.Add(bond);
1292 return S_OK;
1293 }
1294
1295
MakeExeMethod(CCompressionMethodMode & mode,const CFilterMode & filterMode,const bool bcj2_IsAllowed,const CUIntVector & disabledFilterIDs)1296 static HRESULT MakeExeMethod(CCompressionMethodMode &mode,
1297 const CFilterMode &filterMode,
1298 const bool bcj2_IsAllowed,
1299 const CUIntVector &disabledFilterIDs)
1300 {
1301 if (mode.Filter_was_Inserted)
1302 {
1303 // filter was inserted, but bond for that filter was not added still.
1304 const CMethodFull &m = mode.Methods[0];
1305 if (m.Id == k_BCJ2)
1306 return AddBcj2Methods(mode);
1307 if (!m.IsSimpleCoder())
1308 return E_NOTIMPL;
1309 if (mode.Bonds.IsEmpty())
1310 return S_OK;
1311 return AddBondForFilter(mode);
1312 }
1313
1314 if (filterMode.Id == 0)
1315 return S_OK;
1316
1317 unsigned nextCoder;
1318
1319 const bool useBcj2 = bcj2_IsAllowed
1320 && Is86Filter(filterMode.Id)
1321 && disabledFilterIDs.FindInSorted(k_BCJ2) < 0;
1322
1323 if (!useBcj2 && disabledFilterIDs.FindInSorted(filterMode.Id) >= 0)
1324 {
1325 // required filter is disabled,
1326 // but we still can use information about data alignment.
1327 #if 0 // 1 for debug
1328 // we can return here, if we want default lzma properties
1329 return S_OK;
1330 #else
1331 // we will try to change lzma/lzma2 properties
1332 nextCoder = 0;
1333 if (!mode.Bonds.IsEmpty())
1334 for (unsigned c = 0;; c++)
1335 {
1336 if (c == mode.Methods.Size())
1337 return S_OK;
1338 if (!mode.IsThereBond_to_Coder(c))
1339 {
1340 nextCoder = c;
1341 break;
1342 }
1343 }
1344 #endif
1345 }
1346 else
1347 {
1348 // we insert new filter method:
1349 CMethodFull &m = mode.Methods.InsertNew(0); // 0 == index of new inserted item
1350 {
1351 // we move all coder indexes in bonds up for 1 position:
1352 FOR_VECTOR (k, mode.Bonds)
1353 {
1354 CBond2 &bond = mode.Bonds[k];
1355 bond.InCoder++;
1356 bond.OutCoder++;
1357 }
1358 }
1359 if (useBcj2)
1360 {
1361 GetMethodFull(k_BCJ2, 4, m);
1362 return AddBcj2Methods(mode);
1363 }
1364
1365 GetMethodFull(filterMode.Id, 1, m);
1366
1367 if (filterMode.Id == k_Delta)
1368 m.AddProp32(NCoderPropID::kDefaultProp, filterMode.Delta);
1369 else if (filterMode.Id == k_ARM64
1370 || filterMode.Id == k_RISCV)
1371 {
1372 // if (filterMode.Offset != 0)
1373 m.AddProp32(
1374 NCoderPropID::kDefaultProp,
1375 // NCoderPropID::kBranchOffset,
1376 filterMode.Offset);
1377 }
1378
1379 nextCoder = 1;
1380 if (!mode.Bonds.IsEmpty())
1381 {
1382 RINOK(AddBondForFilter(mode))
1383 nextCoder = mode.Bonds.Back().InCoder;
1384 }
1385 }
1386
1387 if (nextCoder >= mode.Methods.Size())
1388 {
1389 // we don't expect that case, if there was non-filter method.
1390 // but we return S_OK to support filter-only case.
1391 return S_OK;
1392 }
1393
1394 int alignBits = -1;
1395 {
1396 const UInt32 delta = filterMode.Delta;
1397 if (delta == 0 || delta > 16)
1398 {
1399 // if (delta == 0) alignBits = GetAlignForFilterMethod(filterMode.Id);
1400 }
1401 else if ((delta & ((1 << 4) - 1)) == 0) alignBits = 4;
1402 else if ((delta & ((1 << 3) - 1)) == 0) alignBits = 3;
1403 else if ((delta & ((1 << 2) - 1)) == 0) alignBits = 2;
1404 else if ((delta & ((1 << 1) - 1)) == 0) alignBits = 1;
1405 // else alignBits = 0;
1406 /* alignBits=0 is default mode for lzma/lzma2.
1407 So we don't set alignBits=0 here. */
1408 }
1409 if (alignBits <= 0)
1410 return S_OK;
1411 // (alignBits > 0)
1412 CMethodFull &nextMethod = mode.Methods[nextCoder];
1413 if (nextMethod.Id == k_LZMA || nextMethod.Id == k_LZMA2)
1414 if (!nextMethod.Are_Lzma_Model_Props_Defined())
1415 {
1416 if (alignBits > 2 || filterMode.Id == k_Delta)
1417 nextMethod.AddProp32(NCoderPropID::kPosStateBits, (unsigned)alignBits);
1418 const unsigned lc = (alignBits < 3) ? (unsigned)(3 - alignBits) : 0u;
1419 nextMethod.AddProp32(NCoderPropID::kLitContextBits, lc);
1420 nextMethod.AddProp32(NCoderPropID::kLitPosBits, (unsigned)alignBits);
1421 }
1422 return S_OK;
1423 }
1424
1425
UpdateItem_To_FileItem2(const CUpdateItem & ui,CFileItem2 & file2)1426 static void UpdateItem_To_FileItem2(const CUpdateItem &ui, CFileItem2 &file2)
1427 {
1428 file2.Attrib = ui.Attrib; file2.AttribDefined = ui.AttribDefined;
1429 file2.CTime = ui.CTime; file2.CTimeDefined = ui.CTimeDefined;
1430 file2.ATime = ui.ATime; file2.ATimeDefined = ui.ATimeDefined;
1431 file2.MTime = ui.MTime; file2.MTimeDefined = ui.MTimeDefined;
1432 file2.IsAnti = ui.IsAnti;
1433 // file2.IsAux = false;
1434 file2.StartPosDefined = false;
1435 // file2.StartPos = 0;
1436 }
1437
1438
UpdateItem_To_FileItem(const CUpdateItem & ui,CFileItem & file,CFileItem2 & file2)1439 static void UpdateItem_To_FileItem(const CUpdateItem &ui,
1440 CFileItem &file, CFileItem2 &file2)
1441 {
1442 UpdateItem_To_FileItem2(ui, file2);
1443
1444 file.Size = ui.Size;
1445 file.IsDir = ui.IsDir;
1446 file.HasStream = ui.HasStream();
1447 // file.IsAltStream = ui.IsAltStream;
1448 }
1449
1450
1451
1452 Z7_CLASS_IMP_COM_2(
1453 CRepackInStreamWithSizes
1454 , ISequentialInStream
1455 , ICompressGetSubStreamSize
1456 )
1457 CMyComPtr<ISequentialInStream> _stream;
1458 UInt64 _size;
1459 const CBoolVector *_extractStatuses;
1460 UInt32 _startIndex;
1461 public:
1462 const CDbEx *_db;
1463
1464 void Init(ISequentialInStream *stream, UInt32 startIndex, const CBoolVector *extractStatuses)
1465 {
1466 _startIndex = startIndex;
1467 _extractStatuses = extractStatuses;
1468 _size = 0;
1469 _stream = stream;
1470 }
1471 UInt64 GetSize() const { return _size; }
1472 };
1473
1474 Z7_COM7F_IMF(CRepackInStreamWithSizes::Read(void *data, UInt32 size, UInt32 *processedSize))
1475 {
1476 UInt32 realProcessedSize;
1477 const HRESULT result = _stream->Read(data, size, &realProcessedSize);
1478 _size += realProcessedSize;
1479 if (processedSize)
1480 *processedSize = realProcessedSize;
1481 return result;
1482 }
1483
1484 Z7_COM7F_IMF(CRepackInStreamWithSizes::GetSubStreamSize(UInt64 subStream, UInt64 *value))
1485 {
1486 *value = 0;
1487 if (subStream >= _extractStatuses->Size())
1488 return S_FALSE; // E_FAIL;
1489 const unsigned index = (unsigned)subStream;
1490 if ((*_extractStatuses)[index])
1491 {
1492 const CFileItem &fi = _db->Files[_startIndex + index];
1493 if (fi.HasStream)
1494 *value = fi.Size;
1495 }
1496 return S_OK;
1497 }
1498
1499
1500 class CRepackStreamBase
1501 {
1502 protected:
1503 bool _needWrite;
1504 bool _fileIsOpen;
1505 bool _calcCrc;
1506 UInt32 _crc;
1507 UInt64 _rem;
1508
1509 const CBoolVector *_extractStatuses;
1510 UInt32 _startIndex;
1511 unsigned _currentIndex;
1512
1513 HRESULT OpenFile();
1514 HRESULT CloseFile();
1515 HRESULT ProcessEmptyFiles();
1516
1517 public:
1518 const CDbEx *_db;
1519 CMyComPtr<IArchiveUpdateCallbackFile> _opCallback;
1520 CMyComPtr<IArchiveExtractCallbackMessage2> _extractCallback;
1521
1522 HRESULT Init(UInt32 startIndex, const CBoolVector *extractStatuses);
1523 HRESULT CheckFinishedState() const { return (_currentIndex == _extractStatuses->Size()) ? S_OK: E_FAIL; }
1524 };
1525
1526 HRESULT CRepackStreamBase::Init(UInt32 startIndex, const CBoolVector *extractStatuses)
1527 {
1528 _startIndex = startIndex;
1529 _extractStatuses = extractStatuses;
1530
1531 _currentIndex = 0;
1532 _fileIsOpen = false;
1533
1534 return ProcessEmptyFiles();
1535 }
1536
1537 HRESULT CRepackStreamBase::OpenFile()
1538 {
1539 UInt32 arcIndex = _startIndex + _currentIndex;
1540 const CFileItem &fi = _db->Files[arcIndex];
1541
1542 _needWrite = (*_extractStatuses)[_currentIndex];
1543 if (_opCallback)
1544 {
1545 RINOK(_opCallback->ReportOperation(
1546 NEventIndexType::kInArcIndex, arcIndex,
1547 _needWrite ?
1548 NUpdateNotifyOp::kRepack :
1549 NUpdateNotifyOp::kSkip))
1550 }
1551
1552 _crc = CRC_INIT_VAL;
1553 _calcCrc = (fi.CrcDefined && !fi.IsDir);
1554
1555 _fileIsOpen = true;
1556 _rem = fi.Size;
1557 return S_OK;
1558 }
1559
1560 const HRESULT k_My_HRESULT_CRC_ERROR = 0x20000002;
1561
1562 HRESULT CRepackStreamBase::CloseFile()
1563 {
1564 UInt32 arcIndex = _startIndex + _currentIndex;
1565 const CFileItem &fi = _db->Files[arcIndex];
1566 _fileIsOpen = false;
1567 _currentIndex++;
1568 if (!_calcCrc || fi.Crc == CRC_GET_DIGEST(_crc))
1569 return S_OK;
1570
1571 if (_extractCallback)
1572 {
1573 RINOK(_extractCallback->ReportExtractResult(
1574 NEventIndexType::kInArcIndex, arcIndex,
1575 NExtract::NOperationResult::kCRCError))
1576 }
1577 // return S_FALSE;
1578 return k_My_HRESULT_CRC_ERROR;
1579 }
1580
1581 HRESULT CRepackStreamBase::ProcessEmptyFiles()
1582 {
1583 while (_currentIndex < _extractStatuses->Size() && _db->Files[_startIndex + _currentIndex].Size == 0)
1584 {
1585 RINOK(OpenFile())
1586 RINOK(CloseFile())
1587 }
1588 return S_OK;
1589 }
1590
1591
1592
1593 #ifndef Z7_ST
1594
1595 class CFolderOutStream2 Z7_final:
1596 public CRepackStreamBase,
1597 public ISequentialOutStream,
1598 public CMyUnknownImp
1599 {
1600 Z7_COM_UNKNOWN_IMP_0
1601 Z7_IFACE_COM7_IMP(ISequentialOutStream)
1602 public:
1603 CMyComPtr<ISequentialOutStream> _stream;
1604 };
1605
1606 Z7_COM7F_IMF(CFolderOutStream2::Write(const void *data, UInt32 size, UInt32 *processedSize))
1607 {
1608 if (processedSize)
1609 *processedSize = 0;
1610
1611 while (size != 0)
1612 {
1613 if (_fileIsOpen)
1614 {
1615 UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1616 HRESULT result = S_OK;
1617 if (_needWrite)
1618 result = _stream->Write(data, cur, &cur);
1619 if (_calcCrc)
1620 _crc = CrcUpdate(_crc, data, cur);
1621 if (processedSize)
1622 *processedSize += cur;
1623 data = (const Byte *)data + cur;
1624 size -= cur;
1625 _rem -= cur;
1626 if (_rem == 0)
1627 {
1628 RINOK(CloseFile())
1629 RINOK(ProcessEmptyFiles())
1630 }
1631 RINOK(result)
1632 if (cur == 0)
1633 break;
1634 continue;
1635 }
1636
1637 RINOK(ProcessEmptyFiles())
1638 if (_currentIndex == _extractStatuses->Size())
1639 {
1640 // we don't support write cut here
1641 return E_FAIL;
1642 }
1643 RINOK(OpenFile())
1644 }
1645
1646 return S_OK;
1647 }
1648
1649 #endif
1650
1651
1652
1653 static const UInt32 kTempBufSize = 1 << 16;
1654
1655 class CFolderInStream2 Z7_final:
1656 public CRepackStreamBase,
1657 public ISequentialInStream,
1658 public CMyUnknownImp
1659 {
1660 Z7_COM_UNKNOWN_IMP_0
1661 Z7_IFACE_COM7_IMP(ISequentialInStream)
1662
1663 Byte *_buf;
1664 public:
1665 CMyComPtr<ISequentialInStream> _inStream;
1666 HRESULT Result;
1667
1668 CFolderInStream2():
1669 Result(S_OK)
1670 {
1671 _buf = new Byte[kTempBufSize];
1672 }
1673
1674 ~CFolderInStream2()
1675 {
1676 delete []_buf;
1677 }
1678
1679 void Init() { Result = S_OK; }
1680 };
1681
1682 Z7_COM7F_IMF(CFolderInStream2::Read(void *data, UInt32 size, UInt32 *processedSize))
1683 {
1684 if (processedSize)
1685 *processedSize = 0;
1686
1687 while (size != 0)
1688 {
1689 if (_fileIsOpen)
1690 {
1691 UInt32 cur = (size < _rem ? size : (UInt32)_rem);
1692
1693 void *buf;
1694 if (_needWrite)
1695 buf = data;
1696 else
1697 {
1698 buf = _buf;
1699 if (cur > kTempBufSize)
1700 cur = kTempBufSize;
1701 }
1702
1703 const HRESULT result = _inStream->Read(buf, cur, &cur);
1704 _crc = CrcUpdate(_crc, buf, cur);
1705 _rem -= cur;
1706
1707 if (_needWrite)
1708 {
1709 data = (Byte *)data + cur;
1710 size -= cur;
1711 if (processedSize)
1712 *processedSize += cur;
1713 }
1714
1715 if (result != S_OK)
1716 Result = result;
1717
1718 if (_rem == 0)
1719 {
1720 RINOK(CloseFile())
1721 RINOK(ProcessEmptyFiles())
1722 }
1723
1724 RINOK(result)
1725
1726 if (cur == 0)
1727 return E_FAIL;
1728
1729 continue;
1730 }
1731
1732 RINOK(ProcessEmptyFiles())
1733 if (_currentIndex == _extractStatuses->Size())
1734 {
1735 return S_OK;
1736 }
1737 RINOK(OpenFile())
1738 }
1739
1740 return S_OK;
1741 }
1742
1743
1744 class CThreadDecoder Z7_final
1745 #ifndef Z7_ST
1746 : public CVirtThread
1747 #endif
1748 {
1749 public:
1750 CDecoder Decoder;
1751
1752 CThreadDecoder(bool multiThreadMixer):
1753 Decoder(multiThreadMixer)
1754 {
1755 #ifndef Z7_ST
1756 if (multiThreadMixer)
1757 {
1758 MtMode = false;
1759 NumThreads = 1;
1760 FosSpec = new CFolderOutStream2;
1761 Fos = FosSpec;
1762 Result = E_FAIL;
1763 }
1764 #endif
1765 // UnpackSize = 0;
1766 // send_UnpackSize = false;
1767 }
1768
1769 #ifndef Z7_ST
1770
1771 bool dataAfterEnd_Error;
1772 HRESULT Result;
1773 CMyComPtr<IInStream> InStream;
1774
1775 CFolderOutStream2 *FosSpec;
1776 CMyComPtr<ISequentialOutStream> Fos;
1777
1778 UInt64 StartPos;
1779 const CFolders *Folders;
1780 unsigned FolderIndex;
1781
1782 // bool send_UnpackSize;
1783 // UInt64 UnpackSize;
1784
1785 #ifndef Z7_NO_CRYPTO
1786 CMyComPtr<ICryptoGetTextPassword> getTextPassword;
1787 #endif
1788
1789 DECL_EXTERNAL_CODECS_LOC_VARS_DECL
1790
1791 #ifndef Z7_ST
1792 bool MtMode;
1793 UInt32 NumThreads;
1794 #endif
1795
1796
1797 ~CThreadDecoder() Z7_DESTRUCTOR_override
1798 {
1799 /* WaitThreadFinish() will be called in ~CVirtThread().
1800 But we need WaitThreadFinish() call before
1801 destructors of this class members.
1802 */
1803 CVirtThread::WaitThreadFinish();
1804 }
1805 private:
1806 virtual void Execute() Z7_override;
1807
1808 #endif
1809 };
1810
1811 #ifndef Z7_ST
1812
1813 void CThreadDecoder::Execute()
1814 {
1815 try
1816 {
1817 #ifndef Z7_NO_CRYPTO
1818 bool isEncrypted = false;
1819 bool passwordIsDefined = false;
1820 UString password;
1821 #endif
1822
1823 dataAfterEnd_Error = false;
1824
1825 Result = Decoder.Decode(
1826 EXTERNAL_CODECS_LOC_VARS
1827 InStream,
1828 StartPos,
1829 *Folders, FolderIndex,
1830
1831 // send_UnpackSize ? &UnpackSize : NULL,
1832 NULL, // unpackSize : FULL unpack
1833
1834 Fos,
1835 NULL, // compressProgress
1836
1837 NULL // *inStreamMainRes
1838 , dataAfterEnd_Error
1839
1840 Z7_7Z_DECODER_CRYPRO_VARS
1841 #ifndef Z7_ST
1842 , MtMode, NumThreads,
1843 0 // MemUsage
1844 #endif
1845
1846 );
1847 }
1848 catch(...)
1849 {
1850 Result = E_FAIL;
1851 }
1852
1853 /*
1854 if (Result == S_OK)
1855 Result = FosSpec->CheckFinishedState();
1856 */
1857 FosSpec->_stream.Release();
1858 }
1859
1860 #endif
1861
1862 #ifndef Z7_NO_CRYPTO
1863
1864 Z7_CLASS_IMP_NOQIB_1(
1865 CCryptoGetTextPassword
1866 , ICryptoGetTextPassword
1867 )
1868 public:
1869 UString Password;
1870 };
1871
1872 Z7_COM7F_IMF(CCryptoGetTextPassword::CryptoGetTextPassword(BSTR *password))
1873 {
1874 return StringToBstr(Password, password);
1875 }
1876
1877 #endif
1878
1879
1880 static void GetFile(const CDatabase &inDb, unsigned index, CFileItem &file, CFileItem2 &file2)
1881 {
1882 file = inDb.Files[index];
1883 file2.CTimeDefined = inDb.CTime.GetItem(index, file2.CTime);
1884 file2.ATimeDefined = inDb.ATime.GetItem(index, file2.ATime);
1885 file2.MTimeDefined = inDb.MTime.GetItem(index, file2.MTime);
1886 file2.StartPosDefined = inDb.StartPos.GetItem(index, file2.StartPos);
1887 file2.AttribDefined = inDb.Attrib.GetItem(index, file2.Attrib);
1888 file2.IsAnti = inDb.IsItemAnti(index);
1889 // file2.IsAux = inDb.IsItemAux(index);
1890 }
1891
1892 HRESULT Update(
1893 DECL_EXTERNAL_CODECS_LOC_VARS
1894 IInStream *inStream,
1895 const CDbEx *db,
1896 CObjectVector<CUpdateItem> &updateItems,
1897 // const CObjectVector<CTreeFolder> &treeFolders,
1898 // const CUniqBlocks &secureBlocks,
1899 ISequentialOutStream *seqOutStream,
1900 IArchiveUpdateCallback *updateCallback,
1901 const CUpdateOptions &options)
1902 {
1903 UInt64 numSolidFiles = options.NumSolidFiles;
1904 if (numSolidFiles == 0)
1905 numSolidFiles = 1;
1906
1907 Z7_DECL_CMyComPtr_QI_FROM(
1908 IArchiveUpdateCallbackFile,
1909 opCallback, updateCallback)
1910
1911 Z7_DECL_CMyComPtr_QI_FROM(
1912 IArchiveExtractCallbackMessage2,
1913 extractCallback, updateCallback)
1914
1915 /*
1916 Z7_DECL_CMyComPtr_QI_FROM(
1917 IArchiveUpdateCallbackArcProp,
1918 reportArcProp, updateCallback)
1919 */
1920
1921 // size_t totalSecureDataSize = (size_t)secureBlocks.GetTotalSizeInBytes();
1922
1923 CMyComPtr<IStreamSetRestriction> v_StreamSetRestriction;
1924 {
1925 Z7_DECL_CMyComPtr_QI_FROM(
1926 IOutStream,
1927 outStream, seqOutStream)
1928 if (!outStream)
1929 return E_NOTIMPL;
1930 const UInt64 sfxBlockSize = (db && !options.RemoveSfxBlock) ?
1931 db->ArcInfo.StartPosition: 0;
1932 seqOutStream->QueryInterface(IID_IStreamSetRestriction, (void **)&v_StreamSetRestriction);
1933 if (v_StreamSetRestriction)
1934 {
1935 UInt64 offset = 0;
1936 RINOK(outStream->Seek(0, STREAM_SEEK_CUR, &offset))
1937 RINOK(v_StreamSetRestriction->SetRestriction(
1938 outStream ? offset + sfxBlockSize : 0,
1939 outStream ? offset + sfxBlockSize + k_StartHeadersRewriteSize : 0))
1940 }
1941 outStream.Release();
1942 if (sfxBlockSize != 0)
1943 {
1944 RINOK(WriteRange(inStream, seqOutStream, 0, sfxBlockSize, NULL))
1945 }
1946 }
1947
1948 CIntArr fileIndexToUpdateIndexMap;
1949 UInt64 complexity = 0;
1950 bool isThere_UnknownSize = false;
1951 UInt64 inSizeForReduce2 = 0;
1952
1953 #ifndef Z7_NO_CRYPTO
1954 bool needEncryptedRepack = false;
1955 #endif
1956
1957 CRecordVector<CFilterMode2> filters;
1958 CObjectVector<CSolidGroup> groups;
1959
1960 #ifndef Z7_ST
1961 bool thereAreRepacks = false;
1962 #endif
1963
1964 bool useFilters = options.UseFilters;
1965 if (useFilters)
1966 {
1967 const CCompressionMethodMode &method = *options.Method;
1968
1969 FOR_VECTOR (i, method.Methods)
1970 {
1971 /* IsFilterMethod() knows only built-in codecs
1972 FIXME: we should check IsFilter status for external filters too */
1973 if (IsFilterMethod(method.Methods[i].Id))
1974 {
1975 useFilters = false;
1976 break;
1977 }
1978 }
1979 }
1980
1981 if (db)
1982 {
1983 fileIndexToUpdateIndexMap.Alloc(db->Files.Size());
1984 unsigned i;
1985
1986 for (i = 0; i < db->Files.Size(); i++)
1987 fileIndexToUpdateIndexMap[i] = -1;
1988
1989 for (i = 0; i < updateItems.Size(); i++)
1990 {
1991 int index = updateItems[i].IndexInArchive;
1992 if (index != -1)
1993 fileIndexToUpdateIndexMap[(unsigned)index] = (int)i;
1994 }
1995
1996 for (i = 0; i < db->NumFolders; i++)
1997 {
1998 CNum indexInFolder = 0;
1999 CNum numCopyItems = 0;
2000 const CNum numUnpackStreams = db->NumUnpackStreamsVector[i];
2001 UInt64 repackSize = 0;
2002
2003 for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++)
2004 {
2005 if (fi >= db->Files.Size())
2006 return E_FAIL;
2007
2008 const CFileItem &file = db->Files[fi];
2009 if (file.HasStream)
2010 {
2011 indexInFolder++;
2012 const int updateIndex = fileIndexToUpdateIndexMap[fi];
2013 if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData)
2014 {
2015 numCopyItems++;
2016 repackSize += file.Size;
2017 }
2018 }
2019 }
2020
2021 if (numCopyItems == 0)
2022 continue;
2023
2024 CFolderRepack rep;
2025 rep.FolderIndex = i;
2026 rep.NumCopyFiles = numCopyItems;
2027 CFolderEx f;
2028 db->ParseFolderEx(i, f);
2029
2030 #ifndef Z7_NO_CRYPTO
2031 const bool isEncrypted = f.IsEncrypted();
2032 #endif
2033 const bool needCopy = (numCopyItems == numUnpackStreams);
2034 const bool extractFilter = (useFilters || needCopy);
2035
2036 const unsigned groupIndex = Get_FilterGroup_for_Folder(filters, f, extractFilter);
2037
2038 while (groupIndex >= groups.Size())
2039 groups.AddNew();
2040
2041 groups[groupIndex].folderRefs.Add(rep);
2042
2043 if (needCopy)
2044 complexity += db->GetFolderFullPackSize(i);
2045 else
2046 {
2047 #ifndef Z7_ST
2048 thereAreRepacks = true;
2049 #endif
2050 complexity += repackSize;
2051 if (inSizeForReduce2 < repackSize)
2052 inSizeForReduce2 = repackSize;
2053 #ifndef Z7_NO_CRYPTO
2054 if (isEncrypted)
2055 needEncryptedRepack = true;
2056 #endif
2057 }
2058 }
2059 }
2060
2061 UInt64 inSizeForReduce = 0;
2062 {
2063 const bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0);
2064 FOR_VECTOR (i, updateItems)
2065 {
2066 const CUpdateItem &ui = updateItems[i];
2067 if (ui.NewData)
2068 {
2069 if (ui.Size == (UInt64)(Int64)-1)
2070 isThere_UnknownSize = true;
2071 else
2072 {
2073 complexity += ui.Size;
2074 if (isSolid)
2075 inSizeForReduce += ui.Size;
2076 else if (inSizeForReduce < ui.Size)
2077 inSizeForReduce = ui.Size;
2078 }
2079 }
2080 }
2081 }
2082
2083 if (isThere_UnknownSize)
2084 inSizeForReduce = (UInt64)(Int64)-1;
2085 else
2086 RINOK(updateCallback->SetTotal(complexity))
2087
2088 if (inSizeForReduce < inSizeForReduce2)
2089 inSizeForReduce = inSizeForReduce2;
2090
2091
2092 CMyComPtr2_Create<ICompressProgressInfo, CLocalProgress> lps;
2093 lps->Init(updateCallback, true);
2094
2095 #ifndef Z7_ST
2096
2097 CStreamBinder sb;
2098 /*
2099 if (options.MultiThreadMixer)
2100 {
2101 RINOK(sb.CreateEvents());
2102 }
2103 */
2104
2105 #endif
2106
2107 CThreadDecoder threadDecoder(options.MultiThreadMixer);
2108
2109 #ifndef Z7_ST
2110 if (options.MultiThreadMixer && thereAreRepacks)
2111 {
2112 #ifdef Z7_EXTERNAL_CODECS
2113 threadDecoder._externalCodecs = _externalCodecs;
2114 #endif
2115 const WRes wres = threadDecoder.Create();
2116 if (wres != 0)
2117 return HRESULT_FROM_WIN32(wres);
2118 }
2119 #endif
2120
2121 {
2122 CAnalysis analysis;
2123 // analysis.Need_ATime = options.Need_ATime;
2124 int analysisLevel = options.AnalysisLevel;
2125 // (analysisLevel < 0) means default level (5)
2126 if (analysisLevel < 0)
2127 analysisLevel = 5;
2128 if (analysisLevel != 0)
2129 {
2130 analysis.Callback = opCallback;
2131 analysis.ParseWav = true;
2132 if (analysisLevel >= 5)
2133 {
2134 analysis.ParseExe = true;
2135 analysis.ParseExeUnix = true;
2136 // analysis.ParseNoExt = true;
2137 if (analysisLevel >= 7)
2138 {
2139 analysis.ParseNoExt = true;
2140 if (analysisLevel >= 9)
2141 analysis.ParseAll = true;
2142 }
2143 }
2144 }
2145
2146 // ---------- Split files to groups ----------
2147
2148 const CCompressionMethodMode &method = *options.Method;
2149
2150 FOR_VECTOR (i, updateItems)
2151 {
2152 const CUpdateItem &ui = updateItems[i];
2153 if (!ui.NewData || !ui.HasStream())
2154 continue;
2155
2156 CFilterMode2 fm;
2157 if (useFilters)
2158 {
2159 // analysis.ATime_Defined = false;
2160 RINOK(analysis.GetFilterGroup(i, ui, fm))
2161 /*
2162 if (analysis.ATime_Defined)
2163 {
2164 ui.ATime = FILETIME_To_UInt64(analysis.ATime);
2165 ui.ATime_WasReadByAnalysis = true;
2166 }
2167 */
2168 }
2169 fm.Encrypted = method.PasswordIsDefined;
2170
2171 const unsigned groupIndex = GetGroup(filters, fm);
2172 while (groupIndex >= groups.Size())
2173 groups.AddNew();
2174 groups[groupIndex].Indices.Add(i);
2175 }
2176 }
2177
2178
2179 #ifndef Z7_NO_CRYPTO
2180
2181 CCryptoGetTextPassword *getPasswordSpec = NULL;
2182 CMyComPtr<ICryptoGetTextPassword> getTextPassword;
2183 if (needEncryptedRepack)
2184 {
2185 getPasswordSpec = new CCryptoGetTextPassword;
2186 getTextPassword = getPasswordSpec;
2187
2188 #ifndef Z7_ST
2189 threadDecoder.getTextPassword = getPasswordSpec;
2190 #endif
2191
2192 if (options.Method->PasswordIsDefined)
2193 getPasswordSpec->Password = options.Method->Password;
2194 else
2195 {
2196 Z7_DECL_CMyComPtr_QI_FROM(
2197 ICryptoGetTextPassword,
2198 getDecoderPassword, updateCallback)
2199 if (!getDecoderPassword)
2200 return E_NOTIMPL;
2201 CMyComBSTR password;
2202 RINOK(getDecoderPassword->CryptoGetTextPassword(&password))
2203 if (password)
2204 getPasswordSpec->Password = password;
2205 }
2206 }
2207
2208 #endif
2209
2210 // ---------- Compress ----------
2211
2212 COutArchive archive;
2213 CArchiveDatabaseOut newDatabase;
2214
2215 RINOK(archive.Create_and_WriteStartPrefix(seqOutStream))
2216
2217 /*
2218 CIntVector treeFolderToArcIndex;
2219 treeFolderToArcIndex.Reserve(treeFolders.Size());
2220 for (i = 0; i < treeFolders.Size(); i++)
2221 treeFolderToArcIndex.Add(-1);
2222 // ---------- Write Tree (only AUX dirs) ----------
2223 for (i = 1; i < treeFolders.Size(); i++)
2224 {
2225 const CTreeFolder &treeFolder = treeFolders[i];
2226 CFileItem file;
2227 CFileItem2 file2;
2228 file2.Init();
2229 int secureID = 0;
2230 if (treeFolder.UpdateItemIndex < 0)
2231 {
2232 // we can store virtual dir item wuthout attrib, but we want all items have attrib.
2233 file.SetAttrib(FILE_ATTRIBUTE_DIRECTORY);
2234 file2.IsAux = true;
2235 }
2236 else
2237 {
2238 const CUpdateItem &ui = updateItems[treeFolder.UpdateItemIndex];
2239 // if item is not dir, then it's parent for alt streams.
2240 // we will write such items later
2241 if (!ui.IsDir)
2242 continue;
2243 secureID = ui.SecureIndex;
2244 if (ui.NewProps)
2245 UpdateItem_To_FileItem(ui, file, file2);
2246 else
2247 GetFile(*db, ui.IndexInArchive, file, file2);
2248 }
2249 file.Size = 0;
2250 file.HasStream = false;
2251 file.IsDir = true;
2252 file.Parent = treeFolder.Parent;
2253
2254 treeFolderToArcIndex[i] = newDatabase.Files.Size();
2255 newDatabase.AddFile(file, file2, treeFolder.Name);
2256
2257 if (totalSecureDataSize != 0)
2258 newDatabase.SecureIDs.Add(secureID);
2259 }
2260 */
2261
2262 {
2263 /* ---------- Write non-AUX dirs and Empty files ---------- */
2264 CUIntVector emptyRefs;
2265
2266 unsigned i;
2267
2268 for (i = 0; i < updateItems.Size(); i++)
2269 {
2270 const CUpdateItem &ui = updateItems[i];
2271 if (ui.NewData)
2272 {
2273 if (ui.HasStream())
2274 continue;
2275 }
2276 else if (ui.IndexInArchive != -1 && db->Files[(unsigned)ui.IndexInArchive].HasStream)
2277 continue;
2278 /*
2279 if (ui.TreeFolderIndex >= 0)
2280 continue;
2281 */
2282 emptyRefs.Add(i);
2283 }
2284
2285 emptyRefs.Sort(CompareEmptyItems, (void *)&updateItems);
2286
2287 for (i = 0; i < emptyRefs.Size(); i++)
2288 {
2289 const CUpdateItem &ui = updateItems[emptyRefs[i]];
2290 CFileItem file;
2291 CFileItem2 file2;
2292 UString name;
2293 if (ui.NewProps)
2294 {
2295 UpdateItem_To_FileItem(ui, file, file2);
2296 file.CrcDefined = false;
2297 name = ui.Name;
2298 }
2299 else
2300 {
2301 GetFile(*db, (unsigned)ui.IndexInArchive, file, file2);
2302 db->GetPath((unsigned)ui.IndexInArchive, name);
2303 }
2304
2305 /*
2306 if (totalSecureDataSize != 0)
2307 newDatabase.SecureIDs.Add(ui.SecureIndex);
2308 file.Parent = ui.ParentFolderIndex;
2309 */
2310 newDatabase.AddFile(file, file2, name);
2311 }
2312 }
2313
2314 lps->ProgressOffset = 0;
2315
2316 {
2317 // ---------- Sort Filters ----------
2318 FOR_VECTOR (i, filters)
2319 {
2320 filters[i].GroupIndex = i;
2321 }
2322 filters.Sort2();
2323 }
2324
2325 for (unsigned groupIndex = 0; groupIndex < filters.Size(); groupIndex++)
2326 {
2327 const CFilterMode2 &filterMode = filters[groupIndex];
2328
2329 CCompressionMethodMode method = *options.Method;
2330 {
2331 const HRESULT res = MakeExeMethod(method, filterMode,
2332 // bcj2_IsAllowed:
2333 #ifdef Z7_ST
2334 false
2335 #else
2336 options.MaxFilter && options.MultiThreadMixer
2337 #endif
2338 , options.DisabledFilterIDs);
2339
2340 RINOK(res)
2341 }
2342
2343 if (filterMode.Encrypted)
2344 {
2345 if (!method.PasswordIsDefined)
2346 {
2347 #ifndef Z7_NO_CRYPTO
2348 if (getPasswordSpec)
2349 method.Password = getPasswordSpec->Password;
2350 #endif
2351 method.PasswordIsDefined = true;
2352 }
2353 }
2354 else
2355 {
2356 method.PasswordIsDefined = false;
2357 method.Password.Empty();
2358 }
2359
2360 CEncoder encoder(method);
2361
2362 // ---------- Repack and copy old solid blocks ----------
2363
2364 const CSolidGroup &group = groups[filterMode.GroupIndex];
2365
2366 FOR_VECTOR (folderRefIndex, group.folderRefs)
2367 {
2368 const CFolderRepack &rep = group.folderRefs[folderRefIndex];
2369
2370 const unsigned folderIndex = rep.FolderIndex;
2371
2372 const CNum numUnpackStreams = db->NumUnpackStreamsVector[folderIndex];
2373
2374 if (rep.NumCopyFiles == numUnpackStreams)
2375 {
2376 if (opCallback)
2377 {
2378 RINOK(opCallback->ReportOperation(
2379 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2380 NUpdateNotifyOp::kReplicate))
2381
2382 // ---------- Copy old solid block ----------
2383 {
2384 CNum indexInFolder = 0;
2385 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2386 {
2387 if (db->Files[fi].HasStream)
2388 {
2389 indexInFolder++;
2390 RINOK(opCallback->ReportOperation(
2391 NEventIndexType::kInArcIndex, (UInt32)fi,
2392 NUpdateNotifyOp::kReplicate))
2393 }
2394 }
2395 }
2396 }
2397
2398 const UInt64 packSize = db->GetFolderFullPackSize(folderIndex);
2399 RINOK(WriteRange(inStream, archive.SeqStream,
2400 db->GetFolderStreamPos(folderIndex, 0), packSize, lps))
2401 lps->ProgressOffset += packSize;
2402
2403 const unsigned folderIndex_New = newDatabase.Folders.Size();
2404 CFolder &folder = newDatabase.Folders.AddNew();
2405 // v23.01: we copy FolderCrc, if FolderCrc was used
2406 if (db->FolderCRCs.ValidAndDefined(folderIndex))
2407 newDatabase.FolderUnpackCRCs.SetItem(folderIndex_New,
2408 true, db->FolderCRCs.Vals[folderIndex]);
2409
2410 db->ParseFolderInfo(folderIndex, folder);
2411 const CNum startIndex = db->FoStartPackStreamIndex[folderIndex];
2412 FOR_VECTOR (j, folder.PackStreams)
2413 {
2414 newDatabase.PackSizes.Add(db->GetStreamPackSize(startIndex + j));
2415 // newDatabase.PackCRCsDefined.Add(db.PackCRCsDefined[startIndex + j]);
2416 // newDatabase.PackCRCs.Add(db.PackCRCs[startIndex + j]);
2417 }
2418
2419 size_t indexStart = db->FoToCoderUnpackSizes[folderIndex];
2420 const size_t indexEnd = db->FoToCoderUnpackSizes[folderIndex + 1];
2421 for (; indexStart < indexEnd; indexStart++)
2422 newDatabase.CoderUnpackSizes.Add(db->CoderUnpackSizes.ConstData()[indexStart]);
2423 }
2424 else
2425 {
2426 // ---------- Repack old solid block ----------
2427
2428 CBoolVector extractStatuses;
2429
2430 CNum indexInFolder = 0;
2431
2432 if (opCallback)
2433 {
2434 RINOK(opCallback->ReportOperation(
2435 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2436 NUpdateNotifyOp::kRepack))
2437 }
2438
2439 /* We could reduce data size of decoded folder, if we don't need to repack
2440 last files in folder. But the gain in speed is small in most cases.
2441 So we unpack full folder. */
2442
2443 UInt64 sizeToEncode = 0;
2444
2445 /*
2446 UInt64 importantUnpackSize = 0;
2447 unsigned numImportantFiles = 0;
2448 UInt64 decodeSize = 0;
2449 */
2450
2451 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2452 {
2453 bool needExtract = false;
2454 const CFileItem &file = db->Files[fi];
2455
2456 if (file.HasStream)
2457 {
2458 indexInFolder++;
2459 const int updateIndex = fileIndexToUpdateIndexMap[fi];
2460 if (updateIndex >= 0 && !updateItems[(unsigned)updateIndex].NewData)
2461 needExtract = true;
2462 // decodeSize += file.Size;
2463 }
2464
2465 extractStatuses.Add(needExtract);
2466 if (needExtract)
2467 {
2468 sizeToEncode += file.Size;
2469 /*
2470 numImportantFiles = extractStatuses.Size();
2471 importantUnpackSize = decodeSize;
2472 */
2473 }
2474 }
2475
2476 // extractStatuses.DeleteFrom(numImportantFiles);
2477
2478 unsigned startPackIndex = newDatabase.PackSizes.Size();
2479 UInt64 curUnpackSize;
2480 {
2481 CMyComPtr<ISequentialInStream> sbInStream;
2482 CRepackStreamBase *repackBase;
2483 CFolderInStream2 *FosSpec2 = NULL;
2484
2485 CRepackInStreamWithSizes *inStreamSizeCountSpec = new CRepackInStreamWithSizes;
2486 CMyComPtr<ISequentialInStream> inStreamSizeCount = inStreamSizeCountSpec;
2487 {
2488 #ifndef Z7_ST
2489 if (options.MultiThreadMixer)
2490 {
2491 repackBase = threadDecoder.FosSpec;
2492 CMyComPtr<ISequentialOutStream> sbOutStream;
2493 sb.CreateStreams2(sbInStream, sbOutStream);
2494 RINOK(sb.Create_ReInit())
2495
2496 threadDecoder.FosSpec->_stream = sbOutStream;
2497
2498 threadDecoder.InStream = inStream;
2499 threadDecoder.StartPos = db->ArcInfo.DataStartPosition; // db->GetFolderStreamPos(folderIndex, 0);
2500 threadDecoder.Folders = (const CFolders *)db;
2501 threadDecoder.FolderIndex = folderIndex;
2502
2503 // threadDecoder.UnpackSize = importantUnpackSize;
2504 // threadDecoder.send_UnpackSize = true;
2505 }
2506 else
2507 #endif
2508 {
2509 FosSpec2 = new CFolderInStream2;
2510 FosSpec2->Init();
2511 sbInStream = FosSpec2;
2512 repackBase = FosSpec2;
2513
2514 #ifndef Z7_NO_CRYPTO
2515 bool isEncrypted = false;
2516 bool passwordIsDefined = false;
2517 UString password;
2518 #endif
2519
2520 CMyComPtr<ISequentialInStream> decodedStream;
2521 bool dataAfterEnd_Error = false;
2522
2523 const HRESULT res = threadDecoder.Decoder.Decode(
2524 EXTERNAL_CODECS_LOC_VARS
2525 inStream,
2526 db->ArcInfo.DataStartPosition, // db->GetFolderStreamPos(folderIndex, 0);,
2527 *db, folderIndex,
2528 // &importantUnpackSize, // *unpackSize
2529 NULL, // *unpackSize : FULL unpack
2530
2531 NULL, // *outStream
2532 NULL, // *compressProgress
2533
2534 &decodedStream
2535 , dataAfterEnd_Error
2536
2537 Z7_7Z_DECODER_CRYPRO_VARS
2538 #ifndef Z7_ST
2539 , false // mtMode
2540 , 1 // numThreads
2541 , 0 // memUsage
2542 #endif
2543 );
2544
2545 RINOK(res)
2546 if (!decodedStream)
2547 return E_FAIL;
2548
2549 FosSpec2->_inStream = decodedStream;
2550 }
2551
2552 repackBase->_db = db;
2553 repackBase->_opCallback = opCallback;
2554 repackBase->_extractCallback = extractCallback;
2555
2556 UInt32 startIndex = db->FolderStartFileIndex[folderIndex];
2557 RINOK(repackBase->Init(startIndex, &extractStatuses))
2558
2559 inStreamSizeCountSpec->_db = db;
2560 inStreamSizeCountSpec->Init(sbInStream, startIndex, &extractStatuses);
2561
2562 #ifndef Z7_ST
2563 if (options.MultiThreadMixer)
2564 {
2565 WRes wres = threadDecoder.Start();
2566 if (wres != 0)
2567 return HRESULT_FROM_WIN32(wres);
2568 }
2569 #endif
2570 }
2571
2572 // curUnpackSize = sizeToEncode;
2573
2574 HRESULT encodeRes = encoder.Encode1(
2575 EXTERNAL_CODECS_LOC_VARS
2576 inStreamSizeCount,
2577 // NULL,
2578 &inSizeForReduce,
2579 sizeToEncode, // expectedDataSize
2580 newDatabase.Folders.AddNew(),
2581 // newDatabase.CoderUnpackSizes, curUnpackSize,
2582 archive.SeqStream, newDatabase.PackSizes, lps);
2583
2584 if (encodeRes == k_My_HRESULT_CRC_ERROR)
2585 return E_FAIL;
2586
2587 curUnpackSize = inStreamSizeCountSpec->GetSize();
2588
2589 if (encodeRes == S_OK)
2590 {
2591 encoder.Encode_Post(curUnpackSize, newDatabase.CoderUnpackSizes);
2592 }
2593
2594 #ifndef Z7_ST
2595 if (options.MultiThreadMixer)
2596 {
2597 // 16.00: hang was fixed : for case if decoding was not finished.
2598 // We close CBinderInStream and it calls CStreamBinder::CloseRead()
2599 inStreamSizeCount.Release();
2600 sbInStream.Release();
2601
2602 {
2603 const WRes wres = threadDecoder.WaitExecuteFinish();
2604 if (wres != 0)
2605 return HRESULT_FROM_WIN32(wres);
2606 }
2607
2608 const HRESULT decodeRes = threadDecoder.Result;
2609 // if (res == k_My_HRESULT_CRC_ERROR)
2610 if (decodeRes == S_FALSE || threadDecoder.dataAfterEnd_Error)
2611 {
2612 if (extractCallback)
2613 {
2614 RINOK(extractCallback->ReportExtractResult(
2615 NEventIndexType::kInArcIndex, db->FolderStartFileIndex[folderIndex],
2616 // NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2617 (decodeRes != S_OK ?
2618 NExtract::NOperationResult::kDataError :
2619 NExtract::NOperationResult::kDataAfterEnd)))
2620 }
2621 if (decodeRes != S_OK)
2622 return E_FAIL;
2623 }
2624 RINOK(decodeRes)
2625 if (encodeRes == S_OK)
2626 if (sb.ProcessedSize != sizeToEncode)
2627 encodeRes = E_FAIL;
2628 }
2629 else
2630 #endif
2631 {
2632 if (FosSpec2->Result == S_FALSE)
2633 {
2634 if (extractCallback)
2635 {
2636 RINOK(extractCallback->ReportExtractResult(
2637 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2638 NExtract::NOperationResult::kDataError))
2639 }
2640 return E_FAIL;
2641 }
2642 RINOK(FosSpec2->Result)
2643 }
2644
2645 RINOK(encodeRes)
2646 RINOK(repackBase->CheckFinishedState())
2647
2648 if (curUnpackSize != sizeToEncode)
2649 return E_FAIL;
2650 }
2651
2652 for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2653 lps->OutSize += newDatabase.PackSizes[startPackIndex];
2654 lps->InSize += curUnpackSize;
2655 }
2656
2657 newDatabase.NumUnpackStreamsVector.Add(rep.NumCopyFiles);
2658
2659 CNum indexInFolder = 0;
2660 for (CNum fi = db->FolderStartFileIndex[folderIndex]; indexInFolder < numUnpackStreams; fi++)
2661 {
2662 if (db->Files[fi].HasStream)
2663 {
2664 indexInFolder++;
2665 const int updateIndex = fileIndexToUpdateIndexMap[fi];
2666 if (updateIndex >= 0)
2667 {
2668 const CUpdateItem &ui = updateItems[(unsigned)updateIndex];
2669 if (ui.NewData)
2670 continue;
2671
2672 UString name;
2673 CFileItem file;
2674 CFileItem2 file2;
2675 GetFile(*db, fi, file, file2);
2676
2677 if (ui.NewProps)
2678 {
2679 UpdateItem_To_FileItem2(ui, file2);
2680 file.IsDir = ui.IsDir;
2681 name = ui.Name;
2682 }
2683 else
2684 db->GetPath(fi, name);
2685
2686 /*
2687 file.Parent = ui.ParentFolderIndex;
2688 if (ui.TreeFolderIndex >= 0)
2689 treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2690 if (totalSecureDataSize != 0)
2691 newDatabase.SecureIDs.Add(ui.SecureIndex);
2692 */
2693 newDatabase.AddFile(file, file2, name);
2694 }
2695 }
2696 }
2697 }
2698
2699
2700 // ---------- Compress files to new solid blocks ----------
2701
2702 const unsigned numFiles = group.Indices.Size();
2703 if (numFiles == 0)
2704 continue;
2705 CRecordVector<CRefItem> refItems;
2706 refItems.ClearAndSetSize(numFiles);
2707 // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1
2708 const bool sortByType = options.UseTypeSorting;
2709
2710 unsigned i;
2711
2712 for (i = 0; i < numFiles; i++)
2713 refItems[i] = CRefItem(group.Indices[i], updateItems[group.Indices[i]], sortByType);
2714
2715 CSortParam sortParam;
2716 // sortParam.TreeFolders = &treeFolders;
2717 sortParam.SortByType = sortByType;
2718 refItems.Sort(CompareUpdateItems, (void *)&sortParam);
2719
2720 CObjArray<UInt32> indices(numFiles);
2721
2722 for (i = 0; i < numFiles; i++)
2723 {
2724 const UInt32 index = refItems[i].Index;
2725 indices[i] = index;
2726 /*
2727 const CUpdateItem &ui = updateItems[index];
2728 CFileItem file;
2729 if (ui.NewProps)
2730 UpdateItem_To_FileItem(ui, file);
2731 else
2732 file = db.Files[ui.IndexInArchive];
2733 if (file.IsAnti || file.IsDir)
2734 return E_FAIL;
2735 newDatabase.Files.Add(file);
2736 */
2737 }
2738
2739 for (i = 0; i < numFiles;)
2740 {
2741 UInt64 totalSize = 0;
2742 unsigned numSubFiles;
2743
2744 const wchar_t *prevExtension = NULL;
2745
2746 for (numSubFiles = 0; i + numSubFiles < numFiles && numSubFiles < numSolidFiles; numSubFiles++)
2747 {
2748 const CUpdateItem &ui = updateItems[indices[i + numSubFiles]];
2749 totalSize += ui.Size;
2750 if (totalSize > options.NumSolidBytes)
2751 break;
2752 if (options.SolidExtension)
2753 {
2754 const int slashPos = ui.Name.ReverseFind_PathSepar();
2755 const int dotPos = ui.Name.ReverseFind_Dot();
2756 const wchar_t *ext = ui.Name.Ptr(dotPos <= slashPos ? ui.Name.Len() : (unsigned)(dotPos + 1));
2757 if (numSubFiles == 0)
2758 prevExtension = ext;
2759 else if (!StringsAreEqualNoCase(ext, prevExtension))
2760 break;
2761 }
2762 }
2763
2764 if (numSubFiles < 1)
2765 numSubFiles = 1;
2766
2767 RINOK(lps->SetCur())
2768
2769 /*
2770 const unsigned folderIndex = newDatabase.NumUnpackStreamsVector.Size();
2771
2772 if (opCallback)
2773 {
2774 RINOK(opCallback->ReportOperation(
2775 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
2776 NUpdateNotifyOp::kAdd));
2777 }
2778 */
2779
2780
2781 CMyComPtr2_Create<ISequentialInStream, CFolderInStream> inStreamSpec; // solidInStream;
2782
2783 // inStreamSpec->_reportArcProp = reportArcProp;
2784
2785 inStreamSpec->Need_CTime = options.Need_CTime;
2786 inStreamSpec->Need_ATime = options.Need_ATime;
2787 inStreamSpec->Need_MTime = options.Need_MTime;
2788 inStreamSpec->Need_Attrib = options.Need_Attrib;
2789 // inStreamSpec->Need_Crc = options.Need_Crc;
2790
2791 inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
2792
2793 unsigned startPackIndex = newDatabase.PackSizes.Size();
2794 // UInt64 curFolderUnpackSize = totalSize;
2795 // curFolderUnpackSize = (UInt64)(Int64)-1; // for debug
2796 const UInt64 expectedDataSize = totalSize;
2797
2798 // const unsigned folderIndex_New = newDatabase.Folders.Size();
2799
2800 RINOK(encoder.Encode1(
2801 EXTERNAL_CODECS_LOC_VARS
2802 inStreamSpec,
2803 // NULL,
2804 &inSizeForReduce,
2805 expectedDataSize, // expected size
2806 newDatabase.Folders.AddNew(),
2807 // newDatabase.CoderUnpackSizes, curFolderUnpackSize,
2808 archive.SeqStream, newDatabase.PackSizes, lps))
2809
2810 if (!inStreamSpec->WasFinished())
2811 return E_FAIL;
2812
2813 /*
2814 if (inStreamSpec->Need_FolderCrc)
2815 newDatabase.FolderUnpackCRCs.SetItem(folderIndex_New,
2816 true, inStreamSpec->GetFolderCrc());
2817 */
2818
2819 const UInt64 curFolderUnpackSize = inStreamSpec->Get_TotalSize_for_Coder();
2820 encoder.Encode_Post(curFolderUnpackSize, newDatabase.CoderUnpackSizes);
2821
2822 UInt64 packSize = 0;
2823 // const UInt32 numStreams = newDatabase.PackSizes.Size() - startPackIndex;
2824 for (; startPackIndex < newDatabase.PackSizes.Size(); startPackIndex++)
2825 packSize += newDatabase.PackSizes[startPackIndex];
2826 lps->OutSize += packSize;
2827
2828 // for ()
2829 // newDatabase.PackCRCsDefined.Add(false);
2830 // newDatabase.PackCRCs.Add(0);
2831
2832 CNum numUnpackStreams = 0;
2833 UInt64 skippedSize = 0;
2834 UInt64 procSize = 0;
2835 // unsigned numProcessedFiles = 0;
2836
2837 for (unsigned subIndex = 0; subIndex < numSubFiles; subIndex++)
2838 {
2839 const CUpdateItem &ui = updateItems[indices[i + subIndex]];
2840 CFileItem file;
2841 CFileItem2 file2;
2842 UString name;
2843 if (ui.NewProps)
2844 {
2845 UpdateItem_To_FileItem(ui, file, file2);
2846 name = ui.Name;
2847 }
2848 else
2849 {
2850 GetFile(*db, (unsigned)ui.IndexInArchive, file, file2);
2851 db->GetPath((unsigned)ui.IndexInArchive, name);
2852 }
2853 if (file2.IsAnti || file.IsDir)
2854 return E_FAIL;
2855
2856 /*
2857 CFileItem &file = newDatabase.Files[
2858 startFileIndexInDatabase + i + subIndex];
2859 */
2860 if (!inStreamSpec->Processed[subIndex])
2861 {
2862 // we don't add file here
2863 skippedSize += ui.Size;
2864 continue; // comment it for debug
2865 // name += ".locked"; // for debug
2866 }
2867
2868 // if (inStreamSpec->Need_Crc)
2869 file.Crc = inStreamSpec->CRCs[subIndex];
2870 file.Size = inStreamSpec->Sizes[subIndex];
2871
2872 procSize += file.Size;
2873 // if (file.Size >= 0) // for debug: test purposes
2874 if (file.Size != 0)
2875 {
2876 file.CrcDefined = true; // inStreamSpec->Need_Crc;
2877 file.HasStream = true;
2878 numUnpackStreams++;
2879 }
2880 else
2881 {
2882 file.CrcDefined = false;
2883 file.HasStream = false;
2884 }
2885
2886 if (inStreamSpec->TimesDefined[subIndex])
2887 {
2888 if (inStreamSpec->Need_CTime)
2889 { file2.CTimeDefined = true; file2.CTime = inStreamSpec->CTimes[subIndex]; }
2890 if (inStreamSpec->Need_ATime
2891 // && !ui.ATime_WasReadByAnalysis
2892 )
2893 { file2.ATimeDefined = true; file2.ATime = inStreamSpec->ATimes[subIndex]; }
2894 if (inStreamSpec->Need_MTime)
2895 { file2.MTimeDefined = true; file2.MTime = inStreamSpec->MTimes[subIndex]; }
2896 if (inStreamSpec->Need_Attrib)
2897 {
2898 file2.AttribDefined = true;
2899 file2.Attrib = inStreamSpec->Attribs[subIndex];
2900 }
2901 }
2902
2903 /*
2904 file.Parent = ui.ParentFolderIndex;
2905 if (ui.TreeFolderIndex >= 0)
2906 treeFolderToArcIndex[ui.TreeFolderIndex] = newDatabase.Files.Size();
2907 if (totalSecureDataSize != 0)
2908 newDatabase.SecureIDs.Add(ui.SecureIndex);
2909 */
2910 /*
2911 if (reportArcProp)
2912 {
2913 RINOK(ReportItemProps(reportArcProp, ui.IndexInClient, file.Size,
2914 file.CrcDefined ? &file.Crc : NULL))
2915 }
2916 */
2917
2918 // numProcessedFiles++;
2919 newDatabase.AddFile(file, file2, name);
2920 }
2921
2922 /*
2923 // for debug:
2924 // we can write crc to folders area, if folder contains only one file
2925 if (numUnpackStreams == 1 && numSubFiles == 1)
2926 {
2927 const CFileItem &file = newDatabase.Files.Back();
2928 if (file.CrcDefined)
2929 newDatabase.FolderUnpackCRCs.SetItem(folderIndex_New, true, file.Crc);
2930 }
2931 */
2932
2933 /*
2934 // it's optional check to ensure that sizes are correct
2935 if (inStreamSpec->TotalSize_for_Coder != curFolderUnpackSize)
2936 return E_FAIL;
2937 */
2938 // if (inStreamSpec->AlignLog == 0)
2939 {
2940 if (procSize != curFolderUnpackSize)
2941 return E_FAIL;
2942 }
2943 // else
2944 {
2945 /*
2946 {
2947 const CFolder &old = newDatabase.Folders.Back();
2948 CFolder &folder = newDatabase.Folders.AddNew();
2949 {
2950 const unsigned numBonds = old.Bonds.Size();
2951 folder.Bonds.SetSize(numBonds + 1);
2952 for (unsigned k = 0; k < numBonds; k++)
2953 folder.Bonds[k] = old.Bonds[k];
2954 CBond &bond = folder.Bonds[numBonds];
2955 bond.PackIndex = 0;
2956 bond.UnpackIndex = 0;
2957 }
2958 {
2959 const unsigned numCoders = old.Coders.Size();
2960 folder.Coders.SetSize(numCoders + 1);
2961 for (unsigned k = 0; k < numCoders; k++)
2962 folder.Coders[k] = old.Coders[k];
2963 CCoderInfo &cod = folder.Coders[numCoders];
2964 cod.Props.Alloc(1);
2965 cod.Props[0] = (Byte)inStreamSpec->AlignLog;
2966 cod.NumStreams = 1;
2967 }
2968 {
2969 const unsigned numPackStreams = old.Coders.Size();
2970 folder.Coders.SetSize(numPackStreams);
2971 for (unsigned k = 0; k < numPackStreams; k++)
2972 folder.PackStreams[k] = old.PackStreams[k];
2973 }
2974 }
2975 newDatabase.Folders.Delete(newDatabase.Folders.Size() - 2);
2976 */
2977 }
2978
2979
2980 lps->InSize += procSize;
2981 // lps->InSize += curFolderUnpackSize;
2982
2983 // numUnpackStreams = 0 is very bad case for locked files
2984 // v3.13 doesn't understand it.
2985 newDatabase.NumUnpackStreamsVector.Add(numUnpackStreams);
2986 i += numSubFiles;
2987
2988 if (skippedSize != 0 && complexity >= skippedSize)
2989 {
2990 complexity -= skippedSize;
2991 RINOK(updateCallback->SetTotal(complexity))
2992 }
2993
2994 /*
2995 if (reportArcProp)
2996 {
2997 PROPVARIANT prop;
2998 prop.vt = VT_EMPTY;
2999 prop.wReserved1 = 0;
3000 {
3001 NWindows::NCOM::PropVarEm_Set_UInt32(&prop, numProcessedFiles);
3002 RINOK(reportArcProp->ReportProp(
3003 NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidNumSubFiles, &prop));
3004 }
3005 {
3006 NWindows::NCOM::PropVarEm_Set_UInt64(&prop, curFolderUnpackSize);
3007 RINOK(reportArcProp->ReportProp(
3008 NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidSize, &prop));
3009 }
3010 {
3011 NWindows::NCOM::PropVarEm_Set_UInt64(&prop, packSize);
3012 RINOK(reportArcProp->ReportProp(
3013 NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidPackSize, &prop));
3014 }
3015 {
3016 NWindows::NCOM::PropVarEm_Set_UInt32(&prop, numStreams);
3017 RINOK(reportArcProp->ReportProp(
3018 NEventIndexType::kBlockIndex, (UInt32)folderIndex, kpidNumStreams, &prop));
3019 }
3020 RINOK(reportArcProp->ReportFinished(NEventIndexType::kBlockIndex, (UInt32)folderIndex, NUpdate::NOperationResult::kOK));
3021 }
3022 */
3023 /*
3024 if (opCallback)
3025 {
3026 RINOK(opCallback->ReportOperation(
3027 NEventIndexType::kBlockIndex, (UInt32)folderIndex,
3028 NUpdateNotifyOp::kOpFinished));
3029 }
3030 */
3031 }
3032 }
3033
3034 RINOK(lps->SetCur())
3035
3036 /*
3037 fileIndexToUpdateIndexMap.ClearAndFree();
3038 groups.ClearAndFree();
3039 */
3040
3041 /*
3042 for (i = 0; i < newDatabase.Files.Size(); i++)
3043 {
3044 CFileItem &file = newDatabase.Files[i];
3045 file.Parent = treeFolderToArcIndex[file.Parent];
3046 }
3047
3048 if (totalSecureDataSize != 0)
3049 {
3050 newDatabase.SecureBuf.SetCapacity(totalSecureDataSize);
3051 size_t pos = 0;
3052 newDatabase.SecureSizes.Reserve(secureBlocks.Sorted.Size());
3053 for (i = 0; i < secureBlocks.Sorted.Size(); i++)
3054 {
3055 const CByteBuffer &buf = secureBlocks.Bufs[secureBlocks.Sorted[i]];
3056 size_t size = buf.GetCapacity();
3057 if (size != 0)
3058 memcpy(newDatabase.SecureBuf + pos, buf, size);
3059 newDatabase.SecureSizes.Add((UInt32)size);
3060 pos += size;
3061 }
3062 }
3063 */
3064
3065 {
3066 const unsigned numFolders = newDatabase.Folders.Size();
3067 if (newDatabase.NumUnpackStreamsVector.Size() != numFolders
3068 || newDatabase.FolderUnpackCRCs.Defs.Size() > numFolders)
3069 return E_FAIL;
3070 newDatabase.FolderUnpackCRCs.if_NonEmpty_FillResidue_with_false(numFolders);
3071 }
3072
3073 updateItems.ClearAndFree();
3074 newDatabase.ReserveDown();
3075
3076 if (opCallback)
3077 RINOK(opCallback->ReportOperation(NEventIndexType::kNoIndex, (UInt32)(Int32)-1, NUpdateNotifyOp::kHeader))
3078
3079 RINOK(archive.WriteDatabase(EXTERNAL_CODECS_LOC_VARS
3080 newDatabase, options.HeaderMethod, options.HeaderOptions))
3081
3082 if (v_StreamSetRestriction)
3083 RINOK(v_StreamSetRestriction->SetRestriction(0, 0))
3084
3085 return S_OK;
3086 }
3087
3088 }}
3089