1 /* adler32_sse42.c -- compute the Adler-32 checksum of a data stream
2 * Copyright (C) 1995-2011 Mark Adler
3 * Authors:
4 * Adam Stylinski <[email protected]>
5 * Brian Bockelman <[email protected]>
6 * For conditions of distribution and use, see copyright notice in zlib.h
7 */
8
9 #include "../../zbuild.h"
10 #include "../../adler32_p.h"
11 #include "../../adler32_fold.h"
12 #include "adler32_ssse3_p.h"
13 #include <immintrin.h>
14
15 #ifdef X86_SSE42_ADLER32
16
adler32_fold_copy_sse42(uint32_t adler,uint8_t * dst,const uint8_t * src,size_t len)17 Z_INTERNAL uint32_t adler32_fold_copy_sse42(uint32_t adler, uint8_t *dst, const uint8_t *src, size_t len) {
18 uint32_t adler0, adler1;
19 adler1 = (adler >> 16) & 0xffff;
20 adler0 = adler & 0xffff;
21
22 rem_peel:
23 if (len < 16) {
24 return adler32_copy_len_16(adler0, src, dst, len, adler1);
25 }
26
27 __m128i vbuf, vbuf_0;
28 __m128i vs1_0, vs3, vs1, vs2, vs2_0, v_sad_sum1, v_short_sum2, v_short_sum2_0,
29 v_sad_sum2, vsum2, vsum2_0;
30 __m128i zero = _mm_setzero_si128();
31 const __m128i dot2v = _mm_setr_epi8(32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17);
32 const __m128i dot2v_0 = _mm_setr_epi8(16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1);
33 const __m128i dot3v = _mm_set1_epi16(1);
34 size_t k;
35
36 while (len >= 16) {
37
38 k = MIN(len, NMAX);
39 k -= k % 16;
40 len -= k;
41
42 vs1 = _mm_cvtsi32_si128(adler0);
43 vs2 = _mm_cvtsi32_si128(adler1);
44
45 vs3 = _mm_setzero_si128();
46 vs2_0 = _mm_setzero_si128();
47 vs1_0 = vs1;
48
49 while (k >= 32) {
50 /*
51 vs1 = adler + sum(c[i])
52 vs2 = sum2 + 16 vs1 + sum( (16-i+1) c[i] )
53 */
54 vbuf = _mm_loadu_si128((__m128i*)src);
55 vbuf_0 = _mm_loadu_si128((__m128i*)(src + 16));
56 src += 32;
57 k -= 32;
58
59 v_sad_sum1 = _mm_sad_epu8(vbuf, zero);
60 v_sad_sum2 = _mm_sad_epu8(vbuf_0, zero);
61 _mm_storeu_si128((__m128i*)dst, vbuf);
62 _mm_storeu_si128((__m128i*)(dst + 16), vbuf_0);
63 dst += 32;
64
65 v_short_sum2 = _mm_maddubs_epi16(vbuf, dot2v);
66 v_short_sum2_0 = _mm_maddubs_epi16(vbuf_0, dot2v_0);
67
68 vs1 = _mm_add_epi32(v_sad_sum1, vs1);
69 vs3 = _mm_add_epi32(vs1_0, vs3);
70
71 vsum2 = _mm_madd_epi16(v_short_sum2, dot3v);
72 vsum2_0 = _mm_madd_epi16(v_short_sum2_0, dot3v);
73 vs1 = _mm_add_epi32(v_sad_sum2, vs1);
74 vs2 = _mm_add_epi32(vsum2, vs2);
75 vs2_0 = _mm_add_epi32(vsum2_0, vs2_0);
76 vs1_0 = vs1;
77 }
78
79 vs2 = _mm_add_epi32(vs2_0, vs2);
80 vs3 = _mm_slli_epi32(vs3, 5);
81 vs2 = _mm_add_epi32(vs3, vs2);
82 vs3 = _mm_setzero_si128();
83
84 while (k >= 16) {
85 /*
86 vs1 = adler + sum(c[i])
87 vs2 = sum2 + 16 vs1 + sum( (16-i+1) c[i] )
88 */
89 vbuf = _mm_loadu_si128((__m128i*)src);
90 src += 16;
91 k -= 16;
92
93 v_sad_sum1 = _mm_sad_epu8(vbuf, zero);
94 v_short_sum2 = _mm_maddubs_epi16(vbuf, dot2v_0);
95
96 vs1 = _mm_add_epi32(v_sad_sum1, vs1);
97 vs3 = _mm_add_epi32(vs1_0, vs3);
98 vsum2 = _mm_madd_epi16(v_short_sum2, dot3v);
99 vs2 = _mm_add_epi32(vsum2, vs2);
100 vs1_0 = vs1;
101
102 _mm_storeu_si128((__m128i*)dst, vbuf);
103 dst += 16;
104 }
105
106 vs3 = _mm_slli_epi32(vs3, 4);
107 vs2 = _mm_add_epi32(vs2, vs3);
108
109 adler0 = partial_hsum(vs1) % BASE;
110 adler1 = hsum(vs2) % BASE;
111 }
112
113 /* If this is true, there's fewer than 16 elements remaining */
114 if (len) {
115 goto rem_peel;
116 }
117
118 return adler0 | (adler1 << 16);
119 }
120
121 #endif
122